diff --git a/Makefile b/Makefile index f1c3f50cf..ca29111c5 100644 --- a/Makefile +++ b/Makefile @@ -136,8 +136,10 @@ include third_party/third_party.mk include libc/testlib/testlib.mk include tool/viz/lib/vizlib.mk include third_party/lua/lua.mk +include third_party/sqlite3/sqlite3.mk include third_party/quickjs/quickjs.mk include third_party/lz4cli/lz4cli.mk +include third_party/infozip/infozip.mk include tool/build/lib/buildlib.mk include third_party/chibicc/chibicc.mk include third_party/chibicc/test/test.mk diff --git a/README.md b/README.md index 09c149fd2..a3f6f7654 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ If you're doing your development work on Linux or BSD then you need just five files to get started. Here's what you do on Linux: ```sh -wget https://justine.lol/cosmopolitan/cosmopolitan-amalgamation-0.3.zip -unzip cosmopolitan-amalgamation-0.3.zip +wget https://justine.lol/cosmopolitan/cosmopolitan-amalgamation-1.0.zip +unzip cosmopolitan-amalgamation-1.0.zip printf 'main() { printf("hello world\\n"); }\n' >hello.c gcc -g -Os -static -nostdlib -nostdinc -fno-pie -no-pie -mno-red-zone \ -fno-omit-frame-pointer -pg -mnop-mcount \ @@ -38,23 +38,42 @@ first run, so it can be fast and efficient for subsequent executions. ```sh ./hello.com -bash -c './hello.com' # zsh/fish workaround (we upstreamed a patch) +bash -c './hello.com' # zsh/fish workaround (we upstreamed patches) ``` So if you intend to copy the binary to Windows or Mac then please do that before you run it, not after. -If you're developing on Windows or MacOS then you need to download an +### MacOS + +If you're developing on MacOS you can install the GNU compiler +collection for x86_64-elf via homebrew: + +```sh +brew install x86_64-elf-gcc +``` + +Then in the above scripts just replace `gcc` and `objcopy` with +`x86_64-elf-gcc` and `x86_64-elf-objcopy` to compile your APE binary. + +### Windows + +If you're developing on Windows then you need to download an x86_64-pc-linux-gnu toolchain beforehand. See the [Compiling on Windows](https://justine.lol/cosmopolitan/windows-compiling.html) tutorial. It's needed because the ELF object format is what makes universal binaries possible. -Cosmopolitan can also be compiled from source on any Linux distro. +## Source Builds + +Cosmopolitan can be compiled from source on any Linux distro. GNU make +needs to be installed beforehand. This is a freestanding hermetic +repository that bootstraps using a vendored static gcc9 executable. +No further dependencies are required. ```sh -wget https://justine.lol/cosmopolitan/cosmopolitan-0.3.tar.gz -tar xf cosmopolitan-0.3.tar.gz # see releases page +wget https://justine.lol/cosmopolitan/cosmopolitan-1.0.tar.gz +tar xf cosmopolitan-1.0.tar.gz # see releases page cd cosmopolitan make -j16 o//examples/hello.com diff --git a/ape/ape.S b/ape/ape.S index e355ae632..b59b31da3 100644 --- a/ape/ape.S +++ b/ape/ape.S @@ -1476,7 +1476,22 @@ kernel: movabs $ape_stack_vaddr,%rsp test %rax,%rax jz 1f movb $METAL,(%rax) -1: xor %eax,%eax +1: push $0 + mov %rsp,%rbp + mov .Lenv0(%rip),%rax + mov %rax,(%rbp) # envp[0][0] + push $0 # argv[0][0] + push $0 # auxv[1][1] + push $0 # auxv[1][0] + push %rbp # auxv[0][1] + push $31 # auxv[0][0] AT_EXECFN + push $0 # envp[1] + push $.Lenv0 # envp[0] + push $0 # argv[1] + push %rbp # argv[0] + push $1 # argc + xor %ebp,%ebp + xor %eax,%eax xor %ecx,%ecx xor %edx,%edx xor %edi,%edi @@ -1485,20 +1500,10 @@ kernel: movabs $ape_stack_vaddr,%rsp xor %r9d,%r9d xor %r10d,%r10d xor %r11d,%r11d - push $0 # auxv[1][1] - push $0 # auxv[1][0] - push $.Larg0 # auxv[0][1] - push $31 # auxv[0][0] AT_EXECFN - push $0 # envp[1] - push $.Lenv0 # envp[0] - push $0 # argv[1] - push $.Larg0 # argv[0] - push $1 # argc jmp _start .endfn kernel .rodata -.Larg0: .asciz "ape.com" .Lenv0: .asciz "METAL=1" .previous diff --git a/build/bootstrap/compile.com b/build/bootstrap/compile.com index 0073592d8..c460b3c60 100755 Binary files a/build/bootstrap/compile.com and b/build/bootstrap/compile.com differ diff --git a/build/findtmp b/build/findtmp index b49c312bf..10ca8e13e 100755 --- a/build/findtmp +++ b/build/findtmp @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh #-*-mode:sh;indent-tabs-mode:nil;tab-width:2;coding:utf-8-*-┐ #───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘ # diff --git a/examples/curl.c b/examples/curl.c index b46b42621..5915815a4 100644 --- a/examples/curl.c +++ b/examples/curl.c @@ -20,7 +20,6 @@ #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/af.h" -#include "libc/sysv/consts/ai.h" #include "libc/sysv/consts/ipproto.h" #include "libc/sysv/consts/shut.h" #include "libc/sysv/consts/sock.h" diff --git a/examples/nc.c b/examples/nc.c index 39b2c0995..f90603ad9 100644 --- a/examples/nc.c +++ b/examples/nc.c @@ -8,11 +8,13 @@ ╚─────────────────────────────────────────────────────────────────*/ #endif #include "libc/calls/calls.h" +#include "libc/dns/dns.h" #include "libc/fmt/conv.h" #include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/runtime/runtime.h" #include "libc/sock/sock.h" +#include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/af.h" #include "libc/sysv/consts/ipproto.h" @@ -21,6 +23,7 @@ #include "libc/sysv/consts/so.h" #include "libc/sysv/consts/sock.h" #include "libc/sysv/consts/sol.h" +#include "third_party/getopt/getopt.h" /** * @fileoverview netcat clone @@ -28,23 +31,62 @@ * Implemented because BusyBox's netcat doesn't detect remote close and * lingers in the CLOSE_WAIT wait possibly due to file descriptor leaks * - * Once upon time we called this command "Telnet" + * Here's an example usage: + * + * make -j8 o//examples/nc.com + * printf 'GET /\r\n\r\n' | o//examples/nc.com justine.lol 80 + * + * Once upon time we called this command "telnet" */ int main(int argc, char *argv[]) { ssize_t rc; size_t i, got; char buf[1500]; - int err, toto, sock; + bool halfclose = true; + const char *host, *port; + int opt, err, toto, sock; + struct addrinfo *ai = NULL; struct linger linger = {true, 1}; - struct sockaddr_in addr = {AF_INET}; struct pollfd fds[2] = {{-1, POLLIN}, {-1, POLLIN}}; + struct addrinfo hint = {AI_NUMERICSERV, AF_INET, SOCK_STREAM, IPPROTO_TCP}; - if (argc != 3) exit(1); - inet_pton(AF_INET, argv[1], &addr.sin_addr); - addr.sin_port = htons(atoi(argv[2])); + while ((opt = getopt(argc, argv, "hH")) != -1) { + switch (opt) { + case 'H': + halfclose = false; + break; + case 'h': + fputs("Usage: ", stdout); + fputs(argv[0], stdout); + fputs(" [-hH] IP PORT\n", stdout); + exit(0); + default: + fprintf(stderr, "bad option %d\n", opt); + exit(1); + } + } + if (argc - optind != 2) { + fputs("missing args\n", stderr); + exit(1); + } + host = argv[optind + 0]; + port = argv[optind + 1]; - if ((sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == -1) { + switch ((rc = getaddrinfo(host, port, &hint, &ai))) { + case EAI_SUCCESS: + break; + case EAI_SYSTEM: + perror("getaddrinfo"); + exit(1); + default: + fputs("EAI_", stderr); + fputs(gai_strerror(rc), stderr); + fputs("\n", stderr); + exit(1); + } + + if ((sock = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol)) == -1) { perror("socket"); exit(1); } @@ -54,7 +96,7 @@ int main(int argc, char *argv[]) { exit(1); } - if (connect(sock, &addr, sizeof(addr)) == -1) { + if (connect(sock, ai->ai_addr, ai->ai_addrlen) == -1) { perror("connect"); exit(1); } @@ -75,7 +117,9 @@ int main(int argc, char *argv[]) { exit(1); } if (!(got = rc)) { - shutdown(sock, SHUT_WR); + if (halfclose) { + shutdown(sock, SHUT_WR); + } fds[0].fd = -1; } for (i = 0; i < got; i += rc) { @@ -108,5 +152,6 @@ int main(int argc, char *argv[]) { exit(1); } + freeaddrinfo(ai); return 0; } diff --git a/libc/bits/popcnt.h b/libc/bits/popcnt.h index c60182245..f406848c0 100644 --- a/libc/bits/popcnt.h +++ b/libc/bits/popcnt.h @@ -7,15 +7,15 @@ COSMOPOLITAN_C_START_ unsigned long popcnt(unsigned long) pureconst; #if defined(__GNUC__) && !defined(__STRICT_ANSI__) -#define popcnt(X) \ - (__builtin_constant_p(X) ? __builtin_popcountll(X) : ({ \ - unsigned long Res, Pop = (X); \ - if (X86_HAVE(POPCNT)) { \ - asm("popcnt\t%1,%0" : "=r"(Res) : "r"(Pop) : "cc"); \ - } else { \ - Res = (popcnt)(Pop); \ - } \ - Res; \ +#define popcnt(X) \ + (__builtin_constant_p(X) ? __builtin_popcountll(X) : ({ \ + unsigned long PoP = (X); \ + if (X86_HAVE(POPCNT)) { \ + asm("popcnt\t%0,%0" : "+r"(PoP) : /* no inputs */ : "cc"); \ + } else { \ + PoP = (popcnt)(PoP); \ + } \ + PoP; \ })) #endif /* GNUC && !ANSI */ diff --git a/libc/calls/calls.h b/libc/calls/calls.h index 360a71049..13dcf4954 100644 --- a/libc/calls/calls.h +++ b/libc/calls/calls.h @@ -137,8 +137,8 @@ int mlockall(int); int munlock(const void *, size_t); int munlockall(void); int nice(int); -int open(const char *, int, ...) nodiscard; -int openanon(char *, unsigned) nodiscard; +int open(const char *, int, ...); +int openanon(char *, unsigned); int openat(int, const char *, int, ...); int pause(void); int personality(uint64_t); @@ -176,7 +176,7 @@ int stat(const char *, struct stat *); int symlink(const char *, const char *); int symlinkat(const char *, int, const char *); int sync_file_range(int, int64_t, int64_t, unsigned); -int sysinfo(struct sysinfo *) paramsnonnull(); +int sysinfo(struct sysinfo *); int touch(const char *, uint32_t); int truncate(const char *, uint64_t); int ttyname_r(int, char *, size_t); diff --git a/libc/calls/fcntl-nt.c b/libc/calls/fcntl-nt.c index 7dce493b9..b7e563678 100644 --- a/libc/calls/fcntl-nt.c +++ b/libc/calls/fcntl-nt.c @@ -18,45 +18,113 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/calls/internal.h" +#include "libc/calls/struct/flock.h" +#include "libc/macros.internal.h" #include "libc/nt/enum/accessmask.h" #include "libc/nt/enum/fileflagandattributes.h" +#include "libc/nt/enum/filelockflags.h" #include "libc/nt/enum/filesharemode.h" +#include "libc/nt/enum/formatmessageflags.h" +#include "libc/nt/errors.h" #include "libc/nt/files.h" +#include "libc/nt/process.h" +#include "libc/nt/runtime.h" +#include "libc/nt/struct/byhandlefileinformation.h" +#include "libc/nt/struct/overlapped.h" +#include "libc/nt/synchronization.h" #include "libc/sysv/consts/f.h" #include "libc/sysv/consts/fd.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" -textwindows int sys_fcntl_nt(int fd, int cmd, unsigned arg) { +static textwindows int sys_fcntl_nt_lock(struct Fd *f, int cmd, uintptr_t arg) { + struct flock *l; + uint32_t flags, err; + struct NtOverlapped ov; + int64_t pos, off, len, size; + struct NtByHandleFileInformation info; + if (!GetFileInformationByHandle(f->handle, &info)) return __winerr(); + if (!SetFilePointerEx(f->handle, 0, &pos, SEEK_CUR)) return __winerr(); + l = (struct flock *)arg; + len = l->l_len; + off = l->l_start; + size = (uint64_t)info.nFileSizeHigh << 32 | info.nFileSizeLow; + switch (l->l_whence) { + case SEEK_SET: + break; + case SEEK_CUR: + off = pos + off; + break; + case SEEK_END: + off = size - off; + break; + default: + return einval(); + } + if (!len) len = size - off; + if (off < 0 || len < 0) return einval(); + offset2overlap(off, &ov); + if (l->l_type == F_RDLCK || l->l_type == F_WRLCK) { + flags = 0; + if (cmd == F_SETLK) flags |= kNtLockfileFailImmediately; + /* TODO: How can we make SQLite locks on Windows to work? */ + /* if (l->l_type == F_WRLCK) flags |= kNtLockfileExclusiveLock; */ + if (LockFileEx(f->handle, flags, 0, len, len >> 32, &ov)) { + return 0; + } else { + err = GetLastError(); + if (err == kNtErrorLockViolation) err = EAGAIN; + errno = err; + return -1; + } + } else if (l->l_type == F_UNLCK) { + if (UnlockFileEx(f->handle, 0, len, len >> 32, &ov)) { + return 0; + } else { + err = GetLastError(); + if (err == kNtErrorNotLocked) { + return 0; + } else { + errno = err; + return -1; + } + } + } else { + return einval(); + } +} + +textwindows int sys_fcntl_nt(int fd, int cmd, uintptr_t arg) { uint32_t flags; if (__isfdkind(fd, kFdFile) || __isfdkind(fd, kFdSocket)) { - switch (cmd) { - case F_GETFL: - return g_fds.p[fd].flags & (O_ACCMODE | O_APPEND | O_ASYNC | O_DIRECT | - O_NOATIME | O_NONBLOCK); - case F_SETFL: - /* - * - O_APPEND doesn't appear to be tunable at cursory glance - * - O_NONBLOCK might require we start doing all i/o in threads - * - O_DSYNC / O_RSYNC / O_SYNC maybe if we fsync() everything - */ - return einval(); - case F_GETFD: - if (g_fds.p[fd].flags & O_CLOEXEC) { - return FD_CLOEXEC; - } else { - return 0; - } - case F_SETFD: - if (arg & FD_CLOEXEC) { - g_fds.p[fd].flags |= O_CLOEXEC; - return FD_CLOEXEC; - } else { - g_fds.p[fd].flags &= ~O_CLOEXEC; - return 0; - } - default: - return einval(); + if (cmd == F_GETFL) { + return g_fds.p[fd].flags & (O_ACCMODE | O_APPEND | O_ASYNC | O_DIRECT | + O_NOATIME | O_NONBLOCK); + } else if (cmd == F_SETFL) { + /* + * - O_APPEND doesn't appear to be tunable at cursory glance + * - O_NONBLOCK might require we start doing all i/o in threads + * - O_DSYNC / O_RSYNC / O_SYNC maybe if we fsync() everything + */ + return einval(); + } else if (cmd == F_GETFD) { + if (g_fds.p[fd].flags & O_CLOEXEC) { + return FD_CLOEXEC; + } else { + return 0; + } + } else if (cmd == F_SETFD) { + if (arg & FD_CLOEXEC) { + g_fds.p[fd].flags |= O_CLOEXEC; + return FD_CLOEXEC; + } else { + g_fds.p[fd].flags &= ~O_CLOEXEC; + return 0; + } + } else if (cmd == F_SETLK || cmd == F_SETLKW) { + return sys_fcntl_nt_lock(g_fds.p + fd, cmd, arg); + } else { + return einval(); } } else { return ebadf(); diff --git a/libc/nexgen32e/lzcntl.S b/libc/calls/fcntl-sysv.c similarity index 57% rename from libc/nexgen32e/lzcntl.S rename to libc/calls/fcntl-sysv.c index 9fe6a7817..0e9a04ff2 100644 --- a/libc/nexgen32e/lzcntl.S +++ b/libc/calls/fcntl-sysv.c @@ -1,7 +1,7 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,30 +16,16 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/macros.internal.h" +#include "libc/calls/internal.h" +#include "libc/calls/struct/flock.h" +#include "libc/sysv/consts/f.h" -// Finds leading bits in 𝑥. -// -// uint32 𝑥 bsf(𝑥) tzcnt(𝑥) ffs(𝑥) bsr(𝑥) lzcnt(𝑥) -// 0x00000000 wut 32 0 wut 32 -// 0x00000001 0 0 1 0 31 -// 0x80000001 0 0 1 31 0 -// 0x80000000 31 31 32 31 0 -// 0x00000010 4 4 5 4 27 -// 0x08000010 4 4 5 27 4 -// 0x08000000 27 27 28 27 4 -// 0xffffffff 0 0 1 31 0 -// -// @param rdi is 64-bit unsigned 𝑥 value -// @return rax number in range [0,64) or 64 if 𝑥 is 0 -// @see also treasure trove of nearly identical functions -lzcntl: .leafprologue - .profilable - mov $31,%eax - mov $-1,%edx - bsr %rdi,%rcx - cmovz %edx,%ecx - sub %ecx,%eax - .endfn lzcntl,globl - .alias lzcntl,lzcntll - .source __FILE__ +int sys_fcntl(int fd, int cmd, uintptr_t arg) { + int rc; + bool islock; + islock = cmd == F_SETLK || cmd == F_SETLKW || cmd == F_GETLK; + if (islock) cosmo2flock(arg); + rc = __sys_fcntl(fd, cmd, arg); + if (islock) flock2cosmo(arg); + return rc; +} diff --git a/libc/calls/fcntl.c b/libc/calls/fcntl.c index 665a2acd1..1a2f506ea 100644 --- a/libc/calls/fcntl.c +++ b/libc/calls/fcntl.c @@ -25,6 +25,15 @@ * * CHECK_NE(-1, fcntl(fd, F_SETFD, FD_CLOEXEC)); * + * This function implements POSIX Advisory Locks, e.g. + * + * CHECK_NE(-1, fcntl(zfd, F_SETLKW, &(struct flock){F_WRLCK})); + * // ... + * CHECK_NE(-1, fcntl(zfd, F_SETLK, &(struct flock){F_UNLCK})); + * + * Please be warned that locks currently do nothing on Windows since + * figuring out how to polyfill them correctly is a work in progress. + * * @param cmd can be F_{GET,SET}{FD,FL}, etc. * @param arg can be FD_CLOEXEC, etc. depending * @return 0 on success, or -1 w/ errno @@ -32,9 +41,9 @@ */ int fcntl(int fd, int cmd, ...) { va_list va; - unsigned arg; + uintptr_t arg; va_start(va, cmd); - arg = va_arg(va, unsigned); + arg = va_arg(va, uintptr_t); va_end(va); if (!IsWindows()) { return sys_fcntl(fd, cmd, arg); diff --git a/libc/calls/fixupnewfd.c b/libc/calls/fixupnewfd.c index 893601ba2..c7a65c920 100644 --- a/libc/calls/fixupnewfd.c +++ b/libc/calls/fixupnewfd.c @@ -28,7 +28,7 @@ int __fixupnewfd(int fd, int flags) { if (fd != -1) { if (flags & O_CLOEXEC) { - sys_fcntl(fd, F_SETFD, FD_CLOEXEC); + __sys_fcntl(fd, F_SETFD, FD_CLOEXEC); } } return fd; diff --git a/libc/calls/flock.c b/libc/calls/flock.c index 1fbdae7ba..c8f7af7f9 100644 --- a/libc/calls/flock.c +++ b/libc/calls/flock.c @@ -23,6 +23,8 @@ /** * Acquires lock on file. * + * Please note multiple file descriptors means multiple locks. + * * @param op can have LOCK_{SH,EX,NB,UN} for shared, exclusive, * non-blocking, and unlocking * @return 0 on success, or -1 w/ errno diff --git a/libc/calls/fstat.c b/libc/calls/fstat.c index 0d5a0290e..553f6b08d 100644 --- a/libc/calls/fstat.c +++ b/libc/calls/fstat.c @@ -20,6 +20,7 @@ #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" @@ -28,6 +29,7 @@ * @asyncsignalsafe */ int fstat(int fd, struct stat *st) { + if (IsAsan() && (!st || !__asan_is_valid(st, sizeof(*st)))) return efault(); if (__isfdkind(fd, kFdZip)) { return weaken(__zipos_fstat)( (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, st); diff --git a/libc/calls/fstatat.c b/libc/calls/fstatat.c index cb6c4a55e..e9187388c 100644 --- a/libc/calls/fstatat.c +++ b/libc/calls/fstatat.c @@ -20,7 +20,9 @@ #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/errno.h" +#include "libc/intrin/asan.internal.h" #include "libc/sysv/consts/at.h" +#include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" /** @@ -36,6 +38,7 @@ */ int fstatat(int dirfd, const char *path, struct stat *st, uint32_t flags) { struct ZiposUri zipname; + if (IsAsan() && (!st || !__asan_is_valid(st, sizeof(*st)))) return efault(); if (weaken(__zipos_stat) && weaken(__zipos_parseuri)(path, &zipname) != -1) { return weaken(__zipos_stat)(&zipname, st); } else if (!IsWindows()) { diff --git a/libc/calls/getcwd-xnu.c b/libc/calls/getcwd-xnu.c index c1ea1ec17..960d56918 100644 --- a/libc/calls/getcwd-xnu.c +++ b/libc/calls/getcwd-xnu.c @@ -34,7 +34,7 @@ char *sys_getcwd_xnu(char *res, size_t size) { if ((fd = sys_openat(AT_FDCWD, ".", O_RDONLY | O_DIRECTORY, 0)) != -1) { if (sys_fstat(fd, &st[0]) != -1) { if (st[0].st_dev && st[0].st_ino) { - if (sys_fcntl(fd, XNU_F_GETPATH, buf) != -1) { + if (__sys_fcntl(fd, XNU_F_GETPATH, (uintptr_t)buf) != -1) { if (sys_fstatat(AT_FDCWD, buf, &st[1], 0) != -1) { if (st[0].st_dev == st[1].st_dev && st[0].st_ino == st[1].st_ino) { if (memccpy(res, buf, '\0', size)) { diff --git a/libc/calls/getppid.c b/libc/calls/getppid.c index e4dbd5663..7ab0b7c45 100644 --- a/libc/calls/getppid.c +++ b/libc/calls/getppid.c @@ -22,7 +22,7 @@ * Returns parent process id. * @asyncsignalsafe */ -int32_t getppid(void) { +int getppid(void) { if (!IsWindows()) { if (!IsNetbsd()) { return sys_getppid(); diff --git a/libc/calls/internal.h b/libc/calls/internal.h index 4c5571864..c1fe14ccf 100644 --- a/libc/calls/internal.h +++ b/libc/calls/internal.h @@ -109,6 +109,7 @@ char *sys_getcwd(char *, u64) hidden; char *sys_getcwd_xnu(char *, u64) hidden; i32 __sys_dup3(i32, i32, i32) hidden; i32 __sys_execve(const char *, char *const[], char *const[]) hidden; +i32 __sys_fcntl(i32, i32, u64) hidden; i32 __sys_fstat(i32, struct stat *) hidden; i32 __sys_fstatat(i32, const char *, struct stat *, i32) hidden; i32 __sys_getrusage(i32, struct rusage *) hidden; @@ -131,7 +132,7 @@ i32 sys_fchmod(i32, u32) hidden; i32 sys_fchmodat(i32, const char *, u32, u32) hidden; i32 sys_fchown(i64, u32, u32) hidden; i32 sys_fchownat(i32, const char *, u32, u32, u32) hidden; -i32 sys_fcntl(i32, i32, ...) hidden; +i32 sys_fcntl(i32, i32, u64) hidden; i32 sys_fdatasync(i32) hidden; i32 sys_flock(i32, i32) hidden; i32 sys_fstat(i32, struct stat *) hidden; @@ -229,6 +230,8 @@ int gethostname_nt(char *, size_t) hidden; size_t __iovec_size(const struct iovec *, size_t) hidden; void __rusage2linux(struct rusage *) hidden; ssize_t WritevUninterruptible(int, struct iovec *, int); +void flock2cosmo(uintptr_t); +void cosmo2flock(uintptr_t); /*───────────────────────────────────────────────────────────────────────────│─╗ │ cosmopolitan § syscalls » windows nt » veneers ─╬─│┼ @@ -245,7 +248,7 @@ int sys_execve_nt(const char *, char *const[], char *const[]) hidden; int sys_faccessat_nt(int, const char *, int, uint32_t) hidden; int sys_fadvise_nt(int, u64, u64, int) hidden; int sys_fchdir_nt(int) hidden; -int sys_fcntl_nt(int, int, unsigned) hidden; +int sys_fcntl_nt(int, int, uintptr_t) hidden; int sys_fdatasync_nt(int) hidden; int sys_flock_nt(int, int) hidden; int sys_fork_nt(void) hidden; diff --git a/libc/calls/metaflock.c b/libc/calls/metaflock.c new file mode 100644 index 000000000..a13b891cc --- /dev/null +++ b/libc/calls/metaflock.c @@ -0,0 +1,164 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/internal.h" +#include "libc/calls/struct/flock.h" + +union metaflock { + struct flock cosmo; + struct flock_linux { + int16_t l_type; + int16_t l_whence; + int64_t l_start; + int64_t l_len; + int32_t l_pid; + } linux; + struct flock_xnu { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + } xnu; + struct flock_freebsd { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + int32_t l_sysid; + } freebsd; + struct flock_openbsd { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + } openbsd; + struct flock_netbsd { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + } netbsd; +}; + +void flock2cosmo(uintptr_t memory) { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + int32_t l_sysid; + union metaflock *u; + u = (union metaflock *)memory; + if (IsLinux()) { + l_start = u->linux.l_start; + l_len = u->linux.l_len; + l_pid = u->linux.l_pid; + l_type = u->linux.l_type; + l_whence = u->linux.l_whence; + l_sysid = 0; + } else if (IsXnu()) { + l_start = u->xnu.l_start; + l_len = u->xnu.l_len; + l_pid = u->xnu.l_pid; + l_type = u->xnu.l_type; + l_whence = u->xnu.l_whence; + l_sysid = 0; + } else if (IsFreebsd()) { + l_start = u->freebsd.l_start; + l_len = u->freebsd.l_len; + l_pid = u->freebsd.l_pid; + l_type = u->freebsd.l_type; + l_whence = u->freebsd.l_whence; + l_sysid = u->freebsd.l_sysid; + } else if (IsOpenbsd()) { + l_start = u->openbsd.l_start; + l_len = u->openbsd.l_len; + l_pid = u->openbsd.l_pid; + l_type = u->openbsd.l_type; + l_whence = u->openbsd.l_whence; + l_sysid = 0; + } else if (IsNetbsd()) { + l_start = u->netbsd.l_start; + l_len = u->netbsd.l_len; + l_pid = u->netbsd.l_pid; + l_type = u->netbsd.l_type; + l_whence = u->netbsd.l_whence; + l_sysid = 0; + } else { + return; + } + u->cosmo.l_start = l_start; + u->cosmo.l_len = l_len; + u->cosmo.l_pid = l_pid; + u->cosmo.l_type = l_type; + u->cosmo.l_whence = l_whence; + u->cosmo.l_sysid = l_sysid; +} + +void cosmo2flock(uintptr_t memory) { + int64_t l_start; + int64_t l_len; + int32_t l_pid; + int16_t l_type; + int16_t l_whence; + int32_t l_sysid; + union metaflock *u; + u = (union metaflock *)memory; + l_start = u->cosmo.l_start; + l_len = u->cosmo.l_len; + l_pid = u->cosmo.l_pid; + l_type = u->cosmo.l_type; + l_whence = u->cosmo.l_whence; + l_sysid = u->cosmo.l_sysid; + if (IsLinux()) { + u->linux.l_start = l_start; + u->linux.l_len = l_len; + u->linux.l_pid = l_pid; + u->linux.l_type = l_type; + u->linux.l_whence = l_whence; + } else if (IsXnu()) { + u->xnu.l_start = l_start; + u->xnu.l_len = l_len; + u->xnu.l_pid = l_pid; + u->xnu.l_type = l_type; + u->xnu.l_whence = l_whence; + } else if (IsFreebsd()) { + u->freebsd.l_start = l_start; + u->freebsd.l_len = l_len; + u->freebsd.l_pid = l_pid; + u->freebsd.l_type = l_type; + u->freebsd.l_whence = l_whence; + u->freebsd.l_sysid = l_sysid; + } else if (IsOpenbsd()) { + u->openbsd.l_start = l_start; + u->openbsd.l_len = l_len; + u->openbsd.l_pid = l_pid; + u->openbsd.l_type = l_type; + u->openbsd.l_whence = l_whence; + } else if (IsNetbsd()) { + u->netbsd.l_start = l_start; + u->netbsd.l_len = l_len; + u->netbsd.l_pid = l_pid; + u->netbsd.l_type = l_type; + u->netbsd.l_whence = l_whence; + } +} diff --git a/libc/calls/openat-sysv.c b/libc/calls/openat-sysv.c index 2bfbfdeab..09dfc678a 100644 --- a/libc/calls/openat-sysv.c +++ b/libc/calls/openat-sysv.c @@ -36,7 +36,7 @@ int sys_openat(int dirfd, const char *file, int flags, unsigned mode) { errno = err; fd = __sys_openat(dirfd, file, flags & ~O_CLOEXEC, mode); if (fd != -1 && (flags & O_CLOEXEC)) { - sys_fcntl(fd, F_SETFD, FD_CLOEXEC); + __sys_fcntl(fd, F_SETFD, FD_CLOEXEC); } } diff --git a/libc/calls/preadv.c b/libc/calls/preadv.c index 2467b9370..3ad2b28f2 100644 --- a/libc/calls/preadv.c +++ b/libc/calls/preadv.c @@ -23,6 +23,7 @@ #include "libc/calls/struct/iovec.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/asan.internal.h" #include "libc/macros.internal.h" #include "libc/sysv/consts/iov.h" #include "libc/sysv/errfuns.h" @@ -45,6 +46,7 @@ ssize_t preadv(int fd, struct iovec *iov, int iovlen, int64_t off) { if (fd < 0) return einval(); if (iovlen < 0) return einval(); + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { return weaken(__zipos_read)( (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, off); diff --git a/libc/calls/pwritev.c b/libc/calls/pwritev.c index 54231e6ec..036ddfdb0 100644 --- a/libc/calls/pwritev.c +++ b/libc/calls/pwritev.c @@ -22,6 +22,7 @@ #include "libc/calls/struct/iovec.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/asan.internal.h" #include "libc/macros.internal.h" #include "libc/sysv/consts/iov.h" #include "libc/sysv/errfuns.h" @@ -49,6 +50,7 @@ ssize_t pwritev(int fd, const struct iovec *iov, int iovlen, int64_t off) { if (fd < 0) return einval(); if (iovlen < 0) return einval(); + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { return weaken(__zipos_write)( (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, off); diff --git a/libc/calls/readv.c b/libc/calls/readv.c index 24212c202..0d2087aa5 100644 --- a/libc/calls/readv.c +++ b/libc/calls/readv.c @@ -20,6 +20,7 @@ #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/calls/struct/iovec.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" @@ -31,18 +32,21 @@ * @asyncsignalsafe */ ssize_t readv(int fd, const struct iovec *iov, int iovlen) { - if (fd < 0) return einval(); - if (iovlen < 0) return einval(); - if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { - return weaken(__zipos_read)( - (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, -1); - } else if (!IsWindows() && !IsMetal()) { - return sys_readv(fd, iov, iovlen); - } else if (fd >= g_fds.n) { - return ebadf(); - } else if (IsMetal()) { - return sys_readv_metal(g_fds.p + fd, iov, iovlen); + if (fd >= 0 && iovlen >= 0) { + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); + if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { + return weaken(__zipos_read)( + (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, -1); + } else if (!IsWindows() && !IsMetal()) { + return sys_readv(fd, iov, iovlen); + } else if (fd >= g_fds.n) { + return ebadf(); + } else if (IsMetal()) { + return sys_readv_metal(g_fds.p + fd, iov, iovlen); + } else { + return sys_readv_nt(g_fds.p + fd, iov, iovlen); + } } else { - return sys_readv_nt(g_fds.p + fd, iov, iovlen); + return einval(); } } diff --git a/libc/calls/struct/flock.h b/libc/calls/struct/flock.h index 52319ee31..2a9650579 100644 --- a/libc/calls/struct/flock.h +++ b/libc/calls/struct/flock.h @@ -2,12 +2,13 @@ #define COSMOPOLITAN_LIBC_CALLS_STRUCT_FLOCK_H_ #if !(__ASSEMBLER__ + __LINKER__ + 0) -struct flock { - short l_type; - short l_whence; - int64_t l_start; - int64_t l_len; - int l_pid; +struct flock { /* cosmopolitan abi */ + int16_t l_type; /* F_RDLCK, F_WRLCK, F_UNLCK */ + int16_t l_whence; /* SEEK_SET, SEEK_CUR, SEEK_END */ + int64_t l_start; /* starting offset */ + int64_t l_len; /* 0 means until end of file */ + int32_t l_pid; /* lock owner */ + int32_t l_sysid; /* remote system id or zero for local */ }; #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/calls/sysinfo.c b/libc/calls/sysinfo.c index 108b84912..f061cf1cc 100644 --- a/libc/calls/sysinfo.c +++ b/libc/calls/sysinfo.c @@ -21,11 +21,13 @@ #include "libc/calls/internal.h" #include "libc/calls/struct/sysinfo.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/nt/accounting.h" #include "libc/nt/runtime.h" #include "libc/nt/struct/memorystatusex.h" #include "libc/nt/systeminfo.h" #include "libc/str/str.h" +#include "libc/sysv/errfuns.h" /** * Returns amount of system ram, cores, etc. @@ -34,6 +36,11 @@ */ int sysinfo(struct sysinfo *info) { int rc; + if (IsAsan()) { + if (info && !__asan_is_valid(info, sizeof(*info))) { + return efault(); + } + } memset(info, 0, sizeof(*info)); if (!IsWindows()) { rc = sys_sysinfo(info); diff --git a/libc/calls/utimes.c b/libc/calls/utimes.c index 7fe812540..1f4f8edcb 100644 --- a/libc/calls/utimes.c +++ b/libc/calls/utimes.c @@ -18,7 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sysv/consts/at.h" +#include "libc/sysv/errfuns.h" #include "libc/time/time.h" /** @@ -30,6 +32,11 @@ * @see stat() */ int utimes(const char *path, const struct timeval tv[2]) { + if (IsAsan()) { + if (tv && !__asan_is_valid(tv, sizeof(*tv) * 2)) { + return efault(); + } + } if (!IsWindows()) { /* * we don't modernize utimes() into utimensat() because the diff --git a/libc/calls/wait4.c b/libc/calls/wait4.c index 564a64848..420ef53eb 100644 --- a/libc/calls/wait4.c +++ b/libc/calls/wait4.c @@ -20,6 +20,8 @@ #include "libc/calls/internal.h" #include "libc/calls/wait4.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" +#include "libc/sysv/errfuns.h" /** * Waits for status to change on process. @@ -35,6 +37,16 @@ */ int wait4(int pid, int *opt_out_wstatus, int options, struct rusage *opt_out_rusage) { + if (IsAsan()) { + if (opt_out_wstatus && + !__asan_is_valid(opt_out_wstatus, sizeof(*opt_out_wstatus))) { + return efault(); + } + if (opt_out_rusage && + !__asan_is_valid(opt_out_rusage, sizeof(*opt_out_rusage))) { + return efault(); + } + } if (!IsWindows()) { return sys_wait4(pid, opt_out_wstatus, options, opt_out_rusage); } else { diff --git a/libc/calls/writev.c b/libc/calls/writev.c index 53821450b..e6d31258c 100644 --- a/libc/calls/writev.c +++ b/libc/calls/writev.c @@ -19,6 +19,7 @@ #include "libc/bits/weaken.h" #include "libc/calls/calls.h" #include "libc/calls/internal.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" @@ -34,18 +35,21 @@ * @return number of bytes actually handed off, or -1 w/ errno */ ssize_t writev(int fd, const struct iovec *iov, int iovlen) { - if (fd < 0) return einval(); - if (iovlen < 0) return einval(); - if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { - return weaken(__zipos_write)( - (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, -1); - } else if (!IsWindows() && !IsMetal()) { - return sys_writev(fd, iov, iovlen); - } else if (fd >= g_fds.n) { - return ebadf(); - } else if (IsMetal()) { - return sys_writev_metal(g_fds.p + fd, iov, iovlen); + if (fd >= 0 && iovlen >= 0) { + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); + if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { + return weaken(__zipos_write)( + (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, -1); + } else if (!IsWindows() && !IsMetal()) { + return sys_writev(fd, iov, iovlen); + } else if (fd >= g_fds.n) { + return ebadf(); + } else if (IsMetal()) { + return sys_writev_metal(g_fds.p + fd, iov, iovlen); + } else { + return sys_writev_nt(g_fds.p + fd, iov, iovlen); + } } else { - return sys_writev_nt(g_fds.p + fd, iov, iovlen); + return einval(); } } diff --git a/libc/dce.h b/libc/dce.h index 73cde783f..6b02b308b 100644 --- a/libc/dce.h +++ b/libc/dce.h @@ -57,6 +57,12 @@ #define IsOptimized() 0 #endif +#ifdef __FSANITIZE_ADDRESS__ +#define IsAsan() 1 +#else +#define IsAsan() 0 +#endif + #if defined(__PIE__) || defined(__PIC__) #define IsPositionIndependent() 1 #else diff --git a/libc/dns/dnsnamecmp.c b/libc/dns/comparednsnames.c similarity index 89% rename from libc/dns/dnsnamecmp.c rename to libc/dns/comparednsnames.c index 81958aca2..6db6fda2b 100644 --- a/libc/dns/dnsnamecmp.c +++ b/libc/dns/comparednsnames.c @@ -16,8 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/bits/safemacros.internal.h" #include "libc/dns/dns.h" +#include "libc/macros.internal.h" #include "libc/str/str.h" forceinline void FindDnsLabel(const char *A, size_t *i, size_t *n) { @@ -36,12 +36,15 @@ forceinline void FindDnsLabel(const char *A, size_t *i, size_t *n) { /** * Compares DNS hostnames in reverse lexicographical asciibetical order. * @return <0, 0, or >0 - * @see test/libc/dns/dnsnamecmp_test.c (the code that matters) + * @see test/libc/dns/comparednsnames_test.c (the code that matters) */ -int dnsnamecmp(const char *A, const char *B) { +int CompareDnsNames(const char *A, const char *B) { + int res; + bool first; + size_t n, m, i, j; if (A == B) return 0; - size_t n = strlen(A); - size_t m = strlen(B); + n = strlen(A); + m = strlen(B); if (!n || !m || ((A[n - 1] == '.') ^ (B[m - 1] == '.'))) { if (n && m && A[n - 1] == '.' && strchr(B, '.')) { --m; @@ -51,9 +54,9 @@ int dnsnamecmp(const char *A, const char *B) { return A[n ? n - 1 : 0] - B[m ? m - 1 : 0]; } } - size_t i = n; - size_t j = m; - bool first = true; + i = n; + j = m; + first = true; for (;;) { FindDnsLabel(A, &i, &n); FindDnsLabel(B, &j, &m); @@ -62,8 +65,7 @@ int dnsnamecmp(const char *A, const char *B) { if (!i && j) return 1; if (!j && i) return -1; } - int res; - if ((res = strncasecmp(&A[i], &B[j], min(n - i + 1, m - j + 1)))) { + if ((res = strncasecmp(&A[i], &B[j], MIN(n - i + 1, m - j + 1)))) { return res; } if (!i || !j) { diff --git a/libc/dns/consts.h b/libc/dns/consts.h index d49f4c0d8..1bf67dcb0 100644 --- a/libc/dns/consts.h +++ b/libc/dns/consts.h @@ -3,7 +3,9 @@ #include "libc/sock/sock.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) -#define DNS_TYPE_A 1 +#define DNS_TYPE_A 0x01 +#define DNS_TYPE_PTR 0x0c + #define DNS_CLASS_IN 1 #define kMinSockaddr4Size \ diff --git a/libc/dns/dns.h b/libc/dns/dns.h index 5205a42a9..31764a4f5 100644 --- a/libc/dns/dns.h +++ b/libc/dns/dns.h @@ -1,5 +1,6 @@ #ifndef COSMOPOLITAN_LIBC_DNS_DNS_H_ #define COSMOPOLITAN_LIBC_DNS_DNS_H_ +#include "libc/calls/weirdtypes.h" #include "libc/dns/resolvconf.h" #include "libc/sock/sock.h" @@ -27,6 +28,15 @@ #define EAI_INTR -104 #define EAI_NOTCANCELED -102 +/* AI_* conforms to NT ABI */ +#define AI_PASSIVE 1 +#define AI_CANONNAME 2 +#define AI_NUMERICHOST 4 +#define AI_NUMERICSERV 8 +#define AI_ALL 0x0100 +#define AI_ADDRCONFIG 0x0400 +#define AI_V4MAPPED 0x0800 + #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -47,11 +57,15 @@ struct addrinfo { int getaddrinfo(const char *, const char *, const struct addrinfo *, struct addrinfo **) paramsnonnull((4)); int freeaddrinfo(struct addrinfo *); +int getnameinfo(const struct sockaddr *, socklen_t, char *, socklen_t, char *, + socklen_t, int); const char *gai_strerror(int); -int dnsnamecmp(const char *, const char *) paramsnonnull(); -int pascalifydnsname(uint8_t *, size_t, const char *) paramsnonnull(); -int resolvedns(const struct ResolvConf *, int, const char *, struct sockaddr *, +int CompareDnsNames(const char *, const char *) paramsnonnull(); +int PascalifyDnsName(uint8_t *, size_t, const char *) paramsnonnull(); +int ResolveDns(const struct ResolvConf *, int, const char *, struct sockaddr *, uint32_t) paramsnonnull(); +int ResolveDnsReverse(const struct ResolvConf *resolvconf, int, const char *, + char *, size_t) paramsnonnull(); struct addrinfo *newaddrinfo(uint16_t); COSMOPOLITAN_C_END_ diff --git a/libc/dns/dnsheader.c b/libc/dns/dnsheader.c index f26392aac..c66727d67 100644 --- a/libc/dns/dnsheader.c +++ b/libc/dns/dnsheader.c @@ -21,43 +21,37 @@ #include "libc/sysv/errfuns.h" /** - * Serializes DNS message header to wire. + * Serializes DNS message h to wire. * * @return number of bytes written (always 12) or -1 w/ errno * @see pascalifydnsname() */ -int serializednsheader(uint8_t *buf, size_t size, - const struct DnsHeader header) { - if (size < 12) return enospc(); - buf[0x0] = header.id >> 010u; - buf[0x1] = header.id >> 000u; - buf[0x2] = header.bf1; - buf[0x3] = header.bf2; - buf[0x4] = header.qdcount >> 010u; - buf[0x5] = header.qdcount >> 000u; - buf[0x6] = header.ancount >> 010u; - buf[0x7] = header.ancount >> 000u; - buf[0x8] = header.nscount >> 010u; - buf[0x9] = header.nscount >> 000u; - buf[0xa] = header.arcount >> 010u; - buf[0xb] = header.arcount >> 000u; - return 12; +void SerializeDnsHeader(uint8_t p[restrict 12], const struct DnsHeader *h) { + p[0x0] = h->id >> 8; + p[0x1] = h->id; + p[0x2] = h->bf1; + p[0x3] = h->bf2; + p[0x4] = h->qdcount >> 8; + p[0x5] = h->qdcount; + p[0x6] = h->ancount >> 8; + p[0x7] = h->ancount; + p[0x8] = h->nscount >> 8; + p[0x9] = h->nscount; + p[0xa] = h->arcount >> 8; + p[0xb] = h->arcount; } /** - * Serializes DNS message header to wire. + * Serializes DNS message h to wire. * * @return number of bytes read (always 12) or -1 w/ errno */ -int deserializednsheader(struct DnsHeader *header, const uint8_t *buf, - size_t size) { - if (size < 12) return ebadmsg(); - header->id = READ16BE(buf + 0); - header->bf1 = buf[2]; - header->bf2 = buf[3]; - header->qdcount = READ16BE(buf + 4); - header->ancount = READ16BE(buf + 6); - header->nscount = READ16BE(buf + 8); - header->arcount = READ16BE(buf + 10); - return 12; +void DeserializeDnsHeader(struct DnsHeader *h, const uint8_t p[restrict 12]) { + h->id = READ16BE(p); + h->bf1 = p[2]; + h->bf2 = p[3]; + h->qdcount = READ16BE(p + 4); + h->ancount = READ16BE(p + 6); + h->nscount = READ16BE(p + 8); + h->arcount = READ16BE(p + 10); } diff --git a/libc/dns/dnsheader.h b/libc/dns/dnsheader.h index 3d23bfb4d..d517ddc04 100644 --- a/libc/dns/dnsheader.h +++ b/libc/dns/dnsheader.h @@ -13,8 +13,8 @@ struct DnsHeader { uint16_t arcount; /* additional record count */ }; -int serializednsheader(uint8_t *, size_t, const struct DnsHeader); -int deserializednsheader(struct DnsHeader *, const uint8_t *, size_t); +void SerializeDnsHeader(uint8_t[restrict 12], const struct DnsHeader *); +void DeserializeDnsHeader(struct DnsHeader *, const uint8_t[restrict 12]); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/dns/dnsquestion.h b/libc/dns/dnsquestion.h index 8463fd6ca..920622ab5 100644 --- a/libc/dns/dnsquestion.h +++ b/libc/dns/dnsquestion.h @@ -9,7 +9,7 @@ struct DnsQuestion { uint16_t qclass; }; -int serializednsquestion(uint8_t *, size_t, struct DnsQuestion); +int SerializeDnsQuestion(uint8_t *, size_t, const struct DnsQuestion *); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/dns/freeaddrinfo.c b/libc/dns/freeaddrinfo.c index 79b0f7330..782b20afe 100644 --- a/libc/dns/freeaddrinfo.c +++ b/libc/dns/freeaddrinfo.c @@ -22,13 +22,13 @@ /** * Frees addresses returned by getaddrinfo(). */ -int freeaddrinfo(struct addrinfo *addrs) { +int freeaddrinfo(struct addrinfo *ai) { struct addrinfo *next; - while (addrs) { + while (ai) { /* we assume ai_addr and ai_canonname are shoehorned */ - next = addrs->ai_next; - free(addrs); - addrs = next; + next = ai->ai_next; + free(ai); + ai = next; } return 0; } diff --git a/libc/dns/freehoststxt.c b/libc/dns/freehoststxt.c index d43ada01c..d586adc10 100644 --- a/libc/dns/freehoststxt.c +++ b/libc/dns/freehoststxt.c @@ -20,9 +20,9 @@ #include "libc/runtime/runtime.h" /** - * Frees HOSTS.TXT data structure populated by parsehoststxt(). + * Frees HOSTS.TXT data structure populated by ParseHostsTxt(). */ -void freehoststxt(struct HostsTxt **ht) { +void FreeHostsTxt(struct HostsTxt **ht) { if (*ht) { free_s(&(*ht)->entries.p); free_s(&(*ht)->strings.p); diff --git a/libc/dns/freeresolvconf.c b/libc/dns/freeresolvconf.c index f663f572f..f11e3e619 100644 --- a/libc/dns/freeresolvconf.c +++ b/libc/dns/freeresolvconf.c @@ -20,9 +20,9 @@ #include "libc/runtime/runtime.h" /** - * Frees resolv.conf data structure populated by parseresolvconf(). + * Frees resolv.conf data structure populated by ParseResolvConf(). */ -void freeresolvconf(struct ResolvConf **rvp) { +void FreeResolvConf(struct ResolvConf **rvp) { if (*rvp) { free_s(&(*rvp)->nameservers.p); free_s(rvp); diff --git a/libc/dns/getaddrinfo.c b/libc/dns/getaddrinfo.c index 3443877ed..8da6160b2 100644 --- a/libc/dns/getaddrinfo.c +++ b/libc/dns/getaddrinfo.c @@ -26,7 +26,6 @@ #include "libc/sock/sock.h" #include "libc/str/str.h" #include "libc/sysv/consts/af.h" -#include "libc/sysv/consts/ai.h" #include "libc/sysv/consts/inaddr.h" #include "libc/sysv/errfuns.h" @@ -68,13 +67,13 @@ int getaddrinfo(const char *name, const char *service, } else if (hints && (hints->ai_flags & AI_NUMERICHOST) == AI_NUMERICHOST) { freeaddrinfo(ai); return EAI_NONAME; - } else if (resolvehoststxt(gethoststxt(), AF_INET, name, ai->ai_addr, + } else if (ResolveHostsTxt(GetHostsTxt(), AF_INET, name, ai->ai_addr, sizeof(ai->ai_addr4), &canon) > 0) { memcpy(ai->ai_canonname, canon, min(strlen(canon), DNS_NAME_MAX) + 1); *res = ai; return 0; } else { - rc = resolvedns(getresolvconf(), AF_INET, name, ai->ai_addr, + rc = ResolveDns(GetResolvConf(), AF_INET, name, ai->ai_addr, sizeof(ai->ai_addr4)); if (rc > 0) { *res = ai; diff --git a/libc/dns/gethoststxt.c b/libc/dns/gethoststxt.c index f2420328d..ab6a5cb04 100644 --- a/libc/dns/gethoststxt.c +++ b/libc/dns/gethoststxt.c @@ -53,7 +53,7 @@ static textwindows noinline char *getnthoststxtpath(char *pathbuf, * * @note yoinking realloc() ensures there's no size limits */ -const struct HostsTxt *gethoststxt(void) { +const struct HostsTxt *GetHostsTxt(void) { FILE *f; const char *path; char pathbuf[PATH_MAX]; @@ -65,16 +65,16 @@ const struct HostsTxt *gethoststxt(void) { init->ht.entries.p = init->entries; init->ht.strings.n = pushpop(ARRAYLEN(init->strings)); init->ht.strings.p = init->strings; - __cxa_atexit(freehoststxt, &g_hoststxt, NULL); + __cxa_atexit(FreeHostsTxt, &g_hoststxt, NULL); path = "/etc/hosts"; if (IsWindows()) { path = firstnonnull(getnthoststxtpath(pathbuf, ARRAYLEN(pathbuf)), path); } - if (!(f = fopen(path, "r")) || parsehoststxt(g_hoststxt, f) == -1) { + if (!(f = fopen(path, "r")) || ParseHostsTxt(g_hoststxt, f) == -1) { /* TODO(jart): Elevate robustness. */ } fclose(f); - sorthoststxt(g_hoststxt); + SortHostsTxt(g_hoststxt); } return g_hoststxt; } diff --git a/libc/dns/getnameinfo.c b/libc/dns/getnameinfo.c new file mode 100644 index 000000000..ef8af10d8 --- /dev/null +++ b/libc/dns/getnameinfo.c @@ -0,0 +1,107 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ This is free and unencumbered software released into the public domain. │ +│ │ +│ Anyone is free to copy, modify, publish, use, compile, sell, or │ +│ distribute this software, either in source code form or as a compiled │ +│ binary, for any purpose, commercial or non-commercial, and by any │ +│ means. │ +│ │ +│ In jurisdictions that recognize copyright laws, the author or authors │ +│ of this software dedicate any and all copyright interest in the │ +│ software to the public domain. We make this dedication for the benefit │ +│ of the public at large and to the detriment of our heirs and │ +│ successors. We intend this dedication to be an overt act of │ +│ relinquishment in perpetuity of all present and future rights to this │ +│ software under copyright law. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │ +│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │ +│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │ +│ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR │ +│ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, │ +│ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR │ +│ OTHER DEALINGS IN THE SOFTWARE. │ +│ │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/bits/safemacros.internal.h" +#include "libc/calls/calls.h" +#include "libc/dns/consts.h" +#include "libc/dns/dns.h" +#include "libc/dns/hoststxt.h" +#include "libc/dns/resolvconf.h" +#include "libc/fmt/conv.h" +#include "libc/fmt/fmt.h" +#include "libc/mem/mem.h" +#include "libc/sock/sock.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/af.h" +#include "libc/sysv/consts/inaddr.h" +#include "libc/sysv/errfuns.h" + +/** + * Resolves name/service for socket address. + * + * @param addr + * @param addrlen + * @param name + * @param namelen + * @param service + * @param servicelen + * @param flags + * + * @return 0 on success or EAI_xxx value + */ +int getnameinfo(const struct sockaddr *addr, socklen_t addrlen, char *name, + socklen_t namelen, char *service, socklen_t servicelen, + int flags) { + char rdomain[1 + sizeof "255.255.255.255.in-addr.arpa"]; + char info[512]; + int rc, port; + uint8_t *ip; + unsigned int valid_flags; + + valid_flags = + (NI_NAMEREQD | NI_NUMERICHOST | NI_NUMERICSERV | NI_NOFQDN | NI_DGRAM); + + if (flags & ~(valid_flags)) return EAI_BADFLAGS; + if (!name && !service) return EAI_NONAME; + if (addr->sa_family != AF_INET || addrlen < sizeof(struct sockaddr_in)) + return EAI_FAMILY; + + ip = (uint8_t *)&(((struct sockaddr_in *)addr)->sin_addr); + sprintf(rdomain, "%d.%d.%d.%d.in-addr.arpa", ip[3], ip[2], ip[1], ip[0]); + info[0] = '\0'; + if (name != NULL && namelen != 0) { + if ((flags & NI_NUMERICHOST) && (flags & NI_NAMEREQD)) return EAI_NONAME; + + if ((flags & NI_NUMERICHOST) && + inet_ntop(AF_INET, ip, info, sizeof(info)) == NULL) + return EAI_SYSTEM; + else if (!info[0] && ResolveHostsReverse(GetHostsTxt(), AF_INET, ip, info, + sizeof(info)) < 0) + return EAI_SYSTEM; + else if (!info[0] && ResolveDnsReverse(GetResolvConf(), AF_INET, rdomain, + info, sizeof(info)) < 0) + return EAI_SYSTEM; + else if (!info[0] && (flags & NI_NAMEREQD)) + return EAI_NONAME; + else if (!info[0] && inet_ntop(AF_INET, ip, info, sizeof(info)) == NULL) + return EAI_SYSTEM; + + if (strlen(info) + 1 > namelen) return EAI_OVERFLOW; + strcpy(name, info); + } + + port = ntohs(((struct sockaddr_in *)addr)->sin_port); + info[0] = '\0'; + if (service != NULL && servicelen != 0) { + itoa(port, info, 10); + /* TODO: reverse lookup on /etc/services to get name of service */ + if (strlen(info) + 1 > servicelen) return EAI_OVERFLOW; + strcpy(service, info); + } + + return 0; +} diff --git a/libc/dns/getntnameservers.c b/libc/dns/getntnameservers.c index d7768600f..7181cb13d 100644 --- a/libc/dns/getntnameservers.c +++ b/libc/dns/getntnameservers.c @@ -38,41 +38,39 @@ * this function will append * @return number of nameservers appended, or -1 w/ errno */ -textwindows int getntnameservers(struct ResolvConf *resolv) { +textwindows int GetNtNameServers(struct ResolvConf *resolv) { int rc; char value8[128]; int64_t hkInterfaces; struct sockaddr_in nameserver; - char16_t value[128], ifaceuuid[64]; - uint32_t i, keycount, valuebytes, ifaceuuidlen; + char16_t value[128], uuid[64]; + uint32_t i, keycount, valuebytes, uuidlen; keycount = 0; hkInterfaces = kNtInvalidHandleValue; if (!RegOpenKeyEx( kNtHkeyLocalMachine, u"SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters\\Interfaces", 0, kNtKeyRead, &hkInterfaces) && - !RegQueryInfoKey(hkInterfaces, NULL, NULL, NULL, &keycount, NULL, NULL, - NULL, NULL, NULL, NULL, NULL)) { + !RegQueryInfoKey(hkInterfaces, 0, 0, 0, &keycount, 0, 0, 0, 0, 0, 0, 0)) { nameserver.sin_family = AF_INET; nameserver.sin_port = htons(DNS_PORT); rc = 0; for (i = 0; i < keycount; ++i) { - ifaceuuidlen = sizeof(ifaceuuid); - if (!RegEnumKeyEx(hkInterfaces, i, ifaceuuid, &ifaceuuidlen, NULL, NULL, - NULL, NULL) && - ((!RegGetValue(hkInterfaces, ifaceuuid, u"DhcpIpAddress", + uuidlen = sizeof(uuid); + if (!RegEnumKeyEx(hkInterfaces, i, uuid, &uuidlen, 0, 0, 0, 0) && + ((!RegGetValue(hkInterfaces, uuid, u"DhcpIpAddress", kNtRrfRtRegSz | kNtRrfRtRegMultiSz, NULL, value, ((valuebytes = sizeof(value)), &valuebytes)) && valuebytes > 2 * sizeof(char16_t)) || - (!RegGetValue(hkInterfaces, ifaceuuid, u"IpAddress", + (!RegGetValue(hkInterfaces, uuid, u"IpAddress", kNtRrfRtRegSz | kNtRrfRtRegMultiSz, NULL, value, ((valuebytes = sizeof(value)), &valuebytes)) && valuebytes > 2 * sizeof(char16_t))) && - ((!RegGetValue(hkInterfaces, ifaceuuid, u"DhcpNameServer", + ((!RegGetValue(hkInterfaces, uuid, u"DhcpNameServer", kNtRrfRtRegSz | kNtRrfRtRegMultiSz, NULL, value, ((valuebytes = sizeof(value)), &valuebytes)) && valuebytes > 2 * sizeof(char16_t)) || - (!RegGetValue(hkInterfaces, ifaceuuid, u"NameServer", + (!RegGetValue(hkInterfaces, uuid, u"NameServer", kNtRrfRtRegSz | kNtRrfRtRegMultiSz, NULL, value, ((valuebytes = sizeof(value)), &valuebytes)) && valuebytes > 2 * sizeof(char16_t)))) { diff --git a/libc/dns/getresolvconf.c b/libc/dns/getresolvconf.c index a9035af92..3681d9d12 100644 --- a/libc/dns/getresolvconf.c +++ b/libc/dns/getresolvconf.c @@ -34,7 +34,7 @@ static struct ResolvConfInitialStaticMemory { /** * Returns singleton with DNS server address. */ -const struct ResolvConf *getresolvconf(void) { +const struct ResolvConf *GetResolvConf(void) { int rc; FILE *f; struct ResolvConfInitialStaticMemory *init; @@ -43,16 +43,16 @@ const struct ResolvConf *getresolvconf(void) { g_resolvconf = &init->rv; pushmov(&init->rv.nameservers.n, ARRAYLEN(init->nameservers)); init->rv.nameservers.p = init->nameservers; - __cxa_atexit(freeresolvconf, &g_resolvconf, NULL); + __cxa_atexit(FreeResolvConf, &g_resolvconf, NULL); if (!IsWindows()) { if ((f = fopen("/etc/resolv.conf", "r"))) { - rc = parseresolvconf(g_resolvconf, f); + rc = ParseResolvConf(g_resolvconf, f); } else { rc = -1; } fclose(f); } else { - rc = getntnameservers(g_resolvconf); + rc = GetNtNameServers(g_resolvconf); } if (rc == -1 && !IsTiny()) { /* TODO(jart): Elevate robustness. */ diff --git a/libc/dns/hoststxt.h b/libc/dns/hoststxt.h index 3972684a5..70cbab571 100644 --- a/libc/dns/hoststxt.h +++ b/libc/dns/hoststxt.h @@ -1,26 +1,23 @@ #ifndef COSMOPOLITAN_LIBC_DNS_HOSTSTXT_H_ #define COSMOPOLITAN_LIBC_DNS_HOSTSTXT_H_ +#include "libc/sock/sock.h" +#include "libc/stdio/stdio.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -struct FILE; -struct sockaddr; - struct HostsTxtEntry { - unsigned char ip[4]; /* inet_ntop(AF_INET, he->ip, buf, size) */ - uint32_t name; /* &ht->strings.p[he->name] */ - uint32_t canon; /* &ht->strings.p[he->canon] */ + uint8_t ip[4]; /* inet_ntop(AF_INET, he->ip, buf, size) */ + uint32_t name; /* &ht->strings.p[he->name] */ + uint32_t canon; /* &ht->strings.p[he->canon] */ }; struct HostsTxtEntries { - size_t i; - size_t n; + size_t i, n; struct HostsTxtEntry *p; }; struct HostsTxtStrings { - size_t i; - size_t n; + size_t i, n; char *p; }; @@ -29,13 +26,15 @@ struct HostsTxt { struct HostsTxtStrings strings; }; -const struct HostsTxt *gethoststxt(void) returnsnonnull; -void freehoststxt(struct HostsTxt **) paramsnonnull(); -int parsehoststxt(struct HostsTxt *, struct FILE *) paramsnonnull(); -void sorthoststxt(struct HostsTxt *) paramsnonnull(); -int resolvehoststxt(const struct HostsTxt *, int, const char *, +const struct HostsTxt *GetHostsTxt(void) returnsnonnull; +void FreeHostsTxt(struct HostsTxt **) paramsnonnull(); +int ParseHostsTxt(struct HostsTxt *, FILE *) paramsnonnull(); +void SortHostsTxt(struct HostsTxt *) paramsnonnull(); +int ResolveHostsTxt(const struct HostsTxt *, int, const char *, struct sockaddr *, uint32_t, const char **) paramsnonnull((1, 3)); +int ResolveHostsReverse(const struct HostsTxt *, int, const uint8_t *, char *, + size_t) paramsnonnull((1, 3)); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/dns/parsehoststxt.c b/libc/dns/parsehoststxt.c index 937ca326a..e5a46d1d4 100644 --- a/libc/dns/parsehoststxt.c +++ b/libc/dns/parsehoststxt.c @@ -43,7 +43,7 @@ * @return 0 on success, or -1 w/ errno * @see hoststxtsort() which is the logical next step */ -int parsehoststxt(struct HostsTxt *ht, FILE *f) { +int ParseHostsTxt(struct HostsTxt *ht, FILE *f) { char *line; size_t linesize; struct HostsTxtEntry entry; diff --git a/libc/dns/parseresolvconf.c b/libc/dns/parseresolvconf.c index 3d51904df..b5159998c 100644 --- a/libc/dns/parseresolvconf.c +++ b/libc/dns/parseresolvconf.c @@ -41,7 +41,7 @@ * @param f is an open stream with file content * @return number of nameservers appended, or -1 w/ errno */ -int parseresolvconf(struct ResolvConf *resolv, struct FILE *f) { +int ParseResolvConf(struct ResolvConf *resolv, struct FILE *f) { /* TODO(jart): options ndots:5 */ int rc; char *line; @@ -49,7 +49,7 @@ int parseresolvconf(struct ResolvConf *resolv, struct FILE *f) { struct sockaddr_in nameserver; char *directive, *value, *tok, *comment; rc = 0; - line = NULL; + line = 0; linesize = 0; nameserver.sin_family = AF_INET; nameserver.sin_port = htons(DNS_PORT); diff --git a/libc/dns/pascalifydnsname.c b/libc/dns/pascalifydnsname.c index 86322e3dd..f771dc3bd 100644 --- a/libc/dns/pascalifydnsname.c +++ b/libc/dns/pascalifydnsname.c @@ -30,7 +30,7 @@ * @param name is a dotted NUL-terminated hostname string * @return bytes written (excluding NUL) or -1 w/ errno */ -int pascalifydnsname(uint8_t *buf, size_t size, const char *name) { +int PascalifyDnsName(uint8_t *buf, size_t size, const char *name) { size_t i, j, k, namelen; if ((namelen = strlen(name)) > DNS_NAME_MAX) return enametoolong(); i = 0; diff --git a/libc/dns/resolvconf.h b/libc/dns/resolvconf.h index 222a7fbc1..2ce3be2a2 100644 --- a/libc/dns/resolvconf.h +++ b/libc/dns/resolvconf.h @@ -14,10 +14,10 @@ struct ResolvConf { struct Nameservers nameservers; }; -const struct ResolvConf *getresolvconf(void) returnsnonnull; -int parseresolvconf(struct ResolvConf *, struct FILE *) paramsnonnull(); -void freeresolvconf(struct ResolvConf **) paramsnonnull(); -int getntnameservers(struct ResolvConf *) paramsnonnull(); +const struct ResolvConf *GetResolvConf(void) returnsnonnull; +int ParseResolvConf(struct ResolvConf *, struct FILE *) paramsnonnull(); +void FreeResolvConf(struct ResolvConf **) paramsnonnull(); +int GetNtNameServers(struct ResolvConf *) paramsnonnull(); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/dns/resolvedns.c b/libc/dns/resolvedns.c index 8870e7616..79d9fef19 100644 --- a/libc/dns/resolvedns.c +++ b/libc/dns/resolvedns.c @@ -38,7 +38,7 @@ /** * Queries Domain Name System for address associated with name. * - * @param resolvconf can be getresolvconf() + * @param resolvconf can be GetResolvConf() * @param af can be AF_INET, AF_UNSPEC * @param name can be a local or fully-qualified hostname * @param addr should point to a struct sockaddr_in; if this function @@ -47,65 +47,62 @@ * @return number of matches found, or -1 w/ errno * @error EAFNOSUPPORT. ENETDOWN, ENAMETOOLONG, EBADMSG */ -int resolvedns(const struct ResolvConf *resolvconf, int af, const char *name, +int ResolveDns(const struct ResolvConf *resolvconf, int af, const char *name, struct sockaddr *addr, uint32_t addrsize) { - size_t msgsize; - int res, fd, rc, rc2; - struct sockaddr_in *addr4; - struct DnsQuestion question; + int rc, fd, n; + struct DnsQuestion q; + struct DnsHeader h, h2; + struct sockaddr_in *a4; + uint8_t *p, *pe, msg[512]; uint16_t rtype, rclass, rdlength; - uint8_t *p, *pe, *outmsg, *inmsg; - struct DnsHeader header, response; if (af != AF_INET && af != AF_UNSPEC) return eafnosupport(); if (!resolvconf->nameservers.i) return 0; - memset(&header, 0, sizeof(header)); - header.id = rand32(); - header.bf1 = 1; /* recursion desired */ - header.qdcount = 1; - question.qname = name; - question.qtype = DNS_TYPE_A; - question.qclass = DNS_CLASS_IN; - res = -1; - if ((outmsg = malloc(kMsgMax)) && (inmsg = malloc(kMsgMax)) && - (rc = serializednsheader(outmsg, kMsgMax, header)) != -1 && - (rc2 = serializednsquestion(outmsg + rc, kMsgMax - rc, question)) != -1) { - msgsize = rc + rc2; - if ((fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) != -1 && - sendto(fd, outmsg, msgsize, 0, (void *)&resolvconf->nameservers.p[0], - sizeof(resolvconf->nameservers.p[0])) == msgsize) { - if ((rc = recv(fd, inmsg, kMsgMax, 0)) != -1 && - (rc2 = deserializednsheader(&response, inmsg, rc)) != -1 && - response.id == header.id) { - res = 0; - if (response.ancount) { - p = inmsg + rc2; - pe = inmsg + rc; - while (p < pe && response.qdcount) { - p += strnlen((char *)p, pe - p) + 1 + 4; - response.qdcount--; + memset(&h, 0, sizeof(h)); + rc = ebadmsg(); + h.id = rand32(); + h.bf1 = 1; /* recursion desired */ + h.qdcount = 1; + q.qname = name; + q.qtype = DNS_TYPE_A; + q.qclass = DNS_CLASS_IN; + memset(msg, 0, sizeof(msg)); + SerializeDnsHeader(msg, &h); + if ((n = SerializeDnsQuestion(msg + 12, 500, &q)) == -1) return -1; + if ((fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1) return -1; + if (sendto(fd, msg, 12 + n, 0, resolvconf->nameservers.p, + sizeof(*resolvconf->nameservers.p)) == 12 + n && + (n = read(fd, msg, 512)) >= 12) { + DeserializeDnsHeader(&h2, msg); + if (h2.id == h.id) { + rc = 0; + if (h2.ancount) { + p = msg + 12; + pe = msg + n; + while (p < pe && h2.qdcount) { + p += strnlen((char *)p, pe - p) + 1 + 4; + h2.qdcount--; + } + if (p + 1 < pe) { + if ((p[0] & 0b11000000) == 0b11000000) { /* name pointer */ + p += 2; + } else { + p += strnlen((char *)p, pe - p) + 1; } - if (p + 1 < pe) { - if ((p[0] & 0b11000000) == 0b11000000) { /* name pointer */ - p += 2; - } else { - p += strnlen((char *)p, pe - p) + 1; - } - if (p + 2 + 2 + 4 + 2 < pe) { - rtype = READ16BE(p), p += 2; - rclass = READ16BE(p), p += 2; - /* ttl */ p += 4; - rdlength = READ16BE(p), p += 2; - if (p + rdlength <= pe && rdlength == 4 && - (rtype == DNS_TYPE_A && rclass == DNS_CLASS_IN)) { - res = 1; - if (addrsize) { - if (addrsize >= kMinSockaddr4Size) { - addr4 = (struct sockaddr_in *)addr; - addr4->sin_family = AF_INET; - memcpy(&addr4->sin_addr.s_addr, p, 4); - } else { - res = einval(); - } + if (p + 2 + 2 + 4 + 2 < pe) { + rtype = READ16BE(p), p += 2; + rclass = READ16BE(p), p += 2; + /* ttl */ p += 4; + rdlength = READ16BE(p), p += 2; + if (p + rdlength <= pe && rdlength == 4 && + (rtype == DNS_TYPE_A && rclass == DNS_CLASS_IN)) { + rc = 1; + if (addrsize) { + if (addrsize >= kMinSockaddr4Size) { + a4 = (struct sockaddr_in *)addr; + a4->sin_family = AF_INET; + memcpy(&a4->sin_addr.s_addr, p, 4); + } else { + rc = einval(); } } } @@ -113,8 +110,7 @@ int resolvedns(const struct ResolvConf *resolvconf, int af, const char *name, } } } - res |= close(fd); } - free(outmsg); - return res; + close(fd); + return rc; } diff --git a/libc/dns/resolvednsreverse.c b/libc/dns/resolvednsreverse.c new file mode 100644 index 000000000..5af5bfe67 --- /dev/null +++ b/libc/dns/resolvednsreverse.c @@ -0,0 +1,132 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ This is free and unencumbered software released into the public domain. │ +│ │ +│ Anyone is free to copy, modify, publish, use, compile, sell, or │ +│ distribute this software, either in source code form or as a compiled │ +│ binary, for any purpose, commercial or non-commercial, and by any │ +│ means. │ +│ │ +│ In jurisdictions that recognize copyright laws, the author or authors │ +│ of this software dedicate any and all copyright interest in the │ +│ software to the public domain. We make this dedication for the benefit │ +│ of the public at large and to the detriment of our heirs and │ +│ successors. We intend this dedication to be an overt act of │ +│ relinquishment in perpetuity of all present and future rights to this │ +│ software under copyright law. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │ +│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │ +│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │ +│ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR │ +│ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, │ +│ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR │ +│ OTHER DEALINGS IN THE SOFTWARE. │ +│ │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/bits/bits.h" +#include "libc/calls/calls.h" +#include "libc/dns/consts.h" +#include "libc/dns/dns.h" +#include "libc/dns/dnsheader.h" +#include "libc/dns/dnsquestion.h" +#include "libc/dns/resolvconf.h" +#include "libc/mem/mem.h" +#include "libc/rand/rand.h" +#include "libc/runtime/runtime.h" +#include "libc/sock/sock.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/af.h" +#include "libc/sysv/consts/ipproto.h" +#include "libc/sysv/consts/sock.h" +#include "libc/sysv/errfuns.h" + +#define kMsgMax 512 + +/** + * Performs reverse DNS lookup with IP address. + * + * @param resolvconf can be GetResolvConf() + * @param af can be AF_INET, AF_UNSPEC + * @param name is a reversed IP address string ending with .in-addr.arpa + * @param buf to store the obtained hostname if any + * @param bufsize is size of buf + * + * @return 0 on success, or -1 w/ errno + * @error EAFNOSUPPORT, ENETDOWN, ENAMETOOLONG, EBADMSG + */ +int ResolveDnsReverse(const struct ResolvConf *resolvconf, int af, + const char *name, char *buf, size_t bufsize) { + int rc, fd, n; + struct DnsQuestion q; + struct DnsHeader h, h2; + uint8_t *p, *pe, msg[512]; + uint16_t rtype, rclass, rdlength; + + if (af != AF_INET && af != AF_UNSPEC) return eafnosupport(); + if (!resolvconf->nameservers.i) return 0; + memset(&h, 0, sizeof(h)); + rc = ebadmsg(); + h.id = rand32(); + h.bf1 = 1; /* recursion desired */ + h.qdcount = 1; + q.qname = name; + q.qtype = DNS_TYPE_PTR; + q.qclass = DNS_CLASS_IN; + memset(msg, 0, sizeof(msg)); + SerializeDnsHeader(msg, &h); + + if ((n = SerializeDnsQuestion(msg + 12, 500, &q)) == -1) return -1; + if ((fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1) return -1; + if (sendto(fd, msg, 12 + n, 0, resolvconf->nameservers.p, + sizeof(*resolvconf->nameservers.p)) == 12 + n && + (n = read(fd, msg, 512)) >= 12) { + DeserializeDnsHeader(&h2, msg); + if (h2.id == h.id) { + rc = 0; + if (h2.ancount) { + p = msg + 12; + pe = msg + n; + while (p < pe && h2.qdcount) { + p += strnlen((char *)p, pe - p) + 1 + 4; + h2.qdcount--; + } + if (p + 1 < pe) { + if ((p[0] & 0b11000000) == 0b11000000) { /* name pointer */ + p += 2; + } else { + p += strnlen((char *)p, pe - p) + 1; + } + if (p + 2 + 2 + 4 + 2 < pe) { + rtype = READ16BE(p), p += 2; + rclass = READ16BE(p), p += 2; + /* ttl */ p += 4; + rdlength = READ16BE(p), p += 2; + + if (p + rdlength <= pe && rtype == DNS_TYPE_PTR && + rclass == DNS_CLASS_IN) { + if (strnlen((char *)p, pe - p) + 1 > bufsize) + rc = -1; + else { + /* domain name starts with a letter */ + for (; !isalnum((char)(*p)) && p < pe; p++) rdlength--; + for (char *tmp = (char *)p; rdlength > 0 && *tmp != '\0'; + tmp++) { + /* each label is alphanumeric or hyphen + * any other character is assumed separator */ + if (!isalnum(*tmp) && *tmp != '-') *tmp = '.'; + rdlength--; + } + strcpy(buf, (char *)p); + } + } else + rc = -1; + } + } + } + } + } + close(fd); + return rc; +} diff --git a/libc/dns/resolvehostsreverse.c b/libc/dns/resolvehostsreverse.c new file mode 100644 index 000000000..78f320294 --- /dev/null +++ b/libc/dns/resolvehostsreverse.c @@ -0,0 +1,45 @@ + +#include "libc/alg/alg.h" +#include "libc/dns/consts.h" +#include "libc/dns/dns.h" +#include "libc/dns/hoststxt.h" +#include "libc/sock/sock.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/af.h" +#include "libc/sysv/errfuns.h" + +/** + * Finds name associated with address in HOSTS.TXT table. + * + * @param ht can be GetHostsTxt() + * @param af can be AF_INET + * @param ip is IP address in binary (sin_addr) + * @param buf is buffer to store the name + * @param bufsize is length of buf + * + * @return 1 if found, 0 if not found, or -1 w/ errno + * @error EAFNOSUPPORT + */ +int ResolveHostsReverse(const struct HostsTxt *ht, int af, const uint8_t *ip, + char *buf, size_t bufsize) { + struct HostsTxtEntry *entry = NULL; + uint32_t v1, v2; + + if (af != AF_INET && af != AF_UNSPEC) return eafnosupport(); + if (!ht->entries.p || !buf || bufsize == 0) return -1; + + v1 = *((uint32_t *)ip); + for (size_t j = 0; j < ht->entries.i; j++) { + v2 = *((uint32_t *)ht->entries.p[j].ip); + if (v1 == v2) { + entry = &(ht->entries.p[j]); + break; + } + } + + if (entry) { + strncpy(buf, &ht->strings.p[entry->name], bufsize); + return 1; + } + return 0; +} diff --git a/libc/dns/resolvehoststxt.c b/libc/dns/resolvehoststxt.c index 3a0d6d77a..f5858b2f3 100644 --- a/libc/dns/resolvehoststxt.c +++ b/libc/dns/resolvehoststxt.c @@ -27,16 +27,16 @@ static int hoststxtgetcmp(const char *node, const struct HostsTxtEntry *entry, const char *strings) { - return dnsnamecmp(node, &strings[entry->name]); + return CompareDnsNames(node, &strings[entry->name]); } /** * Finds address associated with name in HOSTS.TXT table. * - * This function performs binary search, so sorthoststxt() must be + * This function performs binary search, so SortHostsTxt() must be * called on the table beforehand. * - * @param ht can be gethoststxt() + * @param ht can be GetHostsTxt() * @param af can be AF_INET, AF_UNSPEC * @param name can be a local or fully-qualified hostname * @param addr should point to a struct sockaddr_in; if this function @@ -46,7 +46,7 @@ static int hoststxtgetcmp(const char *node, const struct HostsTxtEntry *entry, * @return number of matches found, or -1 w/ errno * @error EAFNOSUPPORT */ -int resolvehoststxt(const struct HostsTxt *ht, int af, const char *name, +int ResolveHostsTxt(const struct HostsTxt *ht, int af, const char *name, struct sockaddr *addr, uint32_t addrsize, const char **canon) { struct sockaddr_in *addr4; diff --git a/libc/dns/serializednsquestion.c b/libc/dns/serializednsquestion.c index 416a566c1..133306514 100644 --- a/libc/dns/serializednsquestion.c +++ b/libc/dns/serializednsquestion.c @@ -26,11 +26,14 @@ * @return number of bytes written * @see pascalifydnsname() */ -int serializednsquestion(uint8_t *buf, size_t size, struct DnsQuestion dq) { +int SerializeDnsQuestion(uint8_t *buf, size_t size, + const struct DnsQuestion *dq) { int wrote; - if ((wrote = pascalifydnsname(buf, size, dq.qname)) == -1) return -1; + if ((wrote = PascalifyDnsName(buf, size, dq->qname)) == -1) return -1; if (wrote + 1 + 4 > size) return enospc(); - buf[wrote + 1] = dq.qtype >> 010, buf[wrote + 2] = dq.qtype >> 000; - buf[wrote + 3] = dq.qclass >> 010, buf[wrote + 4] = dq.qclass >> 000; + buf[wrote + 1] = dq->qtype >> 8; + buf[wrote + 2] = dq->qtype; + buf[wrote + 3] = dq->qclass >> 8; + buf[wrote + 4] = dq->qclass; return wrote + 5; } diff --git a/libc/dns/sorthoststxt.c b/libc/dns/sorthoststxt.c index 957e6d128..e32cc452c 100644 --- a/libc/dns/sorthoststxt.c +++ b/libc/dns/sorthoststxt.c @@ -22,25 +22,26 @@ /** * Compares hostnames in HOSTS.TXT table. - * @see dnsnamecmp(), parsehoststxt() + * @see CompareDnsNames(), ParseHostsTxt() */ static int cmphoststxt(const struct HostsTxtEntry *e1, const struct HostsTxtEntry *e2, const char *strings) { if (e1 == e2) return 0; - return dnsnamecmp(&strings[e1->name], &strings[e2->name]); + return CompareDnsNames(&strings[e1->name], &strings[e2->name]); } /** * Sorts entries in HOSTS.TXT table. * - * This function enables resolvehoststxt() to be called so hard-coded + * This function enables ResolveHostsTxt() to be called so hard-coded * hostname lookups take logarithmic time; you can blackhole all the * spam you want, in your /etc/hosts file. * - * The sorted order, defined by dnsnamecmp(), also makes it possible to - * efficiently search for subdomains, once the initial sort is done. + * The sorted order, defined by CompareDnsNames(), also makes it + * possible to efficiently search for subdomains, once the initial sort + * is done. */ -void sorthoststxt(struct HostsTxt *ht) { +void SortHostsTxt(struct HostsTxt *ht) { if (ht->entries.p) { qsort_r(ht->entries.p, ht->entries.i, sizeof(*ht->entries.p), (void *)cmphoststxt, ht->strings.p); diff --git a/libc/dos.h b/libc/dos.h new file mode 100644 index 000000000..61662d02d --- /dev/null +++ b/libc/dos.h @@ -0,0 +1,9 @@ +#ifndef COSMOPOLITAN_LIBC_DOS_H_ +#define COSMOPOLITAN_LIBC_DOS_H_ + +#define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) \ + (((YEAR)-1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1)) +#define DOS_TIME(HOUR, MINUTE, SECOND) \ + ((HOUR) << 11 | (MINUTE) << 5 | (SECOND) >> 1) + +#endif /* COSMOPOLITAN_LIBC_DOS_H_ */ diff --git a/libc/fmt/fmt.c b/libc/fmt/fmt.c index 4f872a74f..adf3d4edb 100644 --- a/libc/fmt/fmt.c +++ b/libc/fmt/fmt.c @@ -308,7 +308,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { /* evaluate specifier */ qchar = '"'; log2base = 0; - alphabet = "0123456789abcdef"; + alphabet = "0123456789abcdefpx"; switch ((d = *format++)) { case 'p': flags |= FLAGS_HASH; @@ -316,7 +316,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { signbit = 63; goto FormatNumber; case 'X': - alphabet = "0123456789ABCDEF"; + alphabet = "0123456789ABCDEFPX"; /* fallthrough */ case 'x': log2base = 4; @@ -548,8 +548,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { } else { PUT('+'); } - for (c = 2, k = 10; 10 * k <= decpt; c++, k *= 10) { - } + for (c = 2, k = 10; 10 * k <= decpt; c++) k *= 10; for (;;) { i1 = decpt / k; PUT(i1 + '0'); @@ -612,7 +611,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { if (pun.d && prec < 13) { pun.u[1] |= 0x100000; if (prec < 5) { - ui = 1 << ((5 - prec) * 4 - 1); + ui = 1u << ((5 - prec) * 4 - 1); if (pun.u[1] & ui) { if (pun.u[1] & ((ui - 1) | (ui << 1)) || pun.u[0]) { pun.u[1] += ui; @@ -631,7 +630,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { } } else { i1 = (13 - prec) * 4; - ui = 1 << (i1 - 1); + ui = 1u << (i1 - 1); if (pun.u[0] & ui && pun.u[0] & ((ui - 1) | (ui << 1))) { pun.u[0] += ui; if (!(pun.u[0] >> i1)) goto BumpIt; @@ -640,12 +639,13 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { } } else { if ((ui = pun.u[0])) { - for (prec = 6; (ui = (ui << 4) & 0xffffffff); ++prec) { - } + ui = __builtin_ctz(ui); + prec = 6 + ((32 - ROUNDDOWN(ui, 4)) >> 2) - 1; + } else if ((ui = pun.u[1] & 0xfffff)) { + ui = __builtin_ctz(ui); + prec = (20 - ROUNDDOWN(ui, 4)) >> 2; } else { - for (prec = 0, ui = pun.u[1] & 0xfffff; ui; - ++prec, ui = (ui << 4) & 0xfffff) { - } + prec = 0; } } bw = 1; @@ -656,7 +656,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { i1 /= 10; } } - if ((sgn = pun.u[1] & 0x80000000)) { + if (pun.u[1] & 0x80000000) { pun.u[1] &= 0x7fffffff; if (pun.d || sign) sign = '-'; } @@ -685,18 +685,9 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { PUT(alphabet[17]); PUT(c); if (prec > 0 || alt) PUT('.'); - if (prec > 0) { - if ((i1 = prec) > 5) i1 = 5; - prec -= i1; - do { - PUT(alphabet[(pun.u[1] >> 16) & 0xf]); - pun.u[1] <<= 4; - } while (--i1 > 0); - while (prec > 0) { - --prec; - PUT(alphabet[(pun.u[0] >> 28) & 0xf]); - pun.u[0] <<= 4; - } + while (prec-- > 0) { + PUT(alphabet[(pun.q >> 48) & 0xf]); + pun.q <<= 4; } PUT(alphabet[16]); if (bex < 0) { @@ -705,8 +696,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { } else { PUT('+'); } - for (c = 1; 10 * c <= bex; c *= 10) { - } + for (c = 1; 10 * c <= bex;) c *= 10; for (;;) { i1 = bex / c; PUT('0' + i1); @@ -714,7 +704,7 @@ hidden int __fmt(void *fn, void *arg, const char *format, va_list va) { bex -= i1 * c; bex *= 10; } - continue; + break; case '%': PUT('%'); diff --git a/libc/fmt/fmt.mk b/libc/fmt/fmt.mk index e906da317..e73cc9299 100644 --- a/libc/fmt/fmt.mk +++ b/libc/fmt/fmt.mk @@ -68,11 +68,6 @@ o/$(MODE)/libc/fmt/filetimetotimeval.o: \ OVERRIDE_CFLAGS += \ -O3 -o/$(MODE)/libc/fmt/itoa64radix10.greg.o \ -o/$(MODE)/libc/fmt/itoa128radix10.greg.o: \ - OVERRIDE_CFLAGS += \ - -fwrapv - LIBC_FMT_LIBS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x))) LIBC_FMT_SRCS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x)_SRCS)) LIBC_FMT_HDRS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/fmt/itoa128radix10.greg.c b/libc/fmt/itoa128radix10.greg.c index 824c4af20..266d02114 100644 --- a/libc/fmt/itoa128radix10.greg.c +++ b/libc/fmt/itoa128radix10.greg.c @@ -49,5 +49,5 @@ noinline size_t uint128toarray_radix10(uint128_t i, char *a) { size_t int128toarray_radix10(int128_t i, char *a) { if (i >= 0) return uint128toarray_radix10(i, a); *a++ = '-'; - return 1 + uint128toarray_radix10(-i, a); + return 1 + uint128toarray_radix10(-(uint128_t)i, a); } diff --git a/libc/fmt/itoa64.c b/libc/fmt/itoa64.c index 126e21838..c3c0e7017 100644 --- a/libc/fmt/itoa64.c +++ b/libc/fmt/itoa64.c @@ -35,7 +35,7 @@ noinline size_t uint64toarray(uint64_t i, char *a, int r) { size_t int64toarray(int64_t i, char *a, int r) { if (i < 0) { *a++ = '-'; - i = -i; + i = -(uint64_t)i; } return uint64toarray(i, a, r); } diff --git a/libc/fmt/itoa64radix10.greg.c b/libc/fmt/itoa64radix10.greg.c index 380411487..b49742f4a 100644 --- a/libc/fmt/itoa64radix10.greg.c +++ b/libc/fmt/itoa64radix10.greg.c @@ -27,14 +27,10 @@ * @return bytes written w/o nul */ noinline size_t uint64toarray_radix10(uint64_t i, char a[hasatleast 21]) { - size_t j; - j = 0; + size_t j = 0; do { - struct { - uint64_t q, r; - } x = {i / 10, i % 10}; - a[j++] = x.r + '0'; - i = x.q; + a[j++] = i % 10 + '0'; + i = i / 10; } while (i > 0); a[j] = '\0'; reverse(a, j); @@ -49,5 +45,5 @@ noinline size_t uint64toarray_radix10(uint64_t i, char a[hasatleast 21]) { size_t int64toarray_radix10(int64_t i, char a[hasatleast 21]) { if (i >= 0) return uint64toarray_radix10(i, a); *a++ = '-'; - return 1 + uint64toarray_radix10(-i, a); + return 1 + uint64toarray_radix10(-(uint64_t)i, a); } diff --git a/libc/fmt/kerrornames.S b/libc/fmt/kerrornames.S index 7574a5b9a..fa3072923 100644 --- a/libc/fmt/kerrornames.S +++ b/libc/fmt/kerrornames.S @@ -21,7 +21,7 @@ .macro .e e .long \e - kErrorNames .long 1f - kErrorNames - .section .rodata.str1.1 + .rodata.str1.1 1: .string "\e" .previous .endm @@ -114,51 +114,5 @@ kErrorNames: .e ENOTRECOVERABLE .e ENONET .e ERESTART - .e ECHRNG - .e EL2NSYNC - .e EL3HLT - .e EL3RST - .e ELNRNG - .e EUNATCH - .e ENOCSI - .e EL2HLT - .e EBADE - .e EBADR - .e EXFULL - .e ENOANO - .e EBADRQC - .e EBADSLT - .e ENOSTR - .e ENODATA - .e ENOSR - .e ENOPKG - .e ENOLINK - .e EADV - .e ESRMNT - .e ECOMM - .e EMULTIHOP - .e EDOTDOT - .e ENOTUNIQ - .e EBADFD - .e EREMCHG - .e ELIBACC - .e ELIBBAD - .e ELIBSCN - .e ELIBMAX - .e ELIBEXEC - .e ESTRPIPE - .e EUCLEAN - .e ENOTNAM - .e ENAVAIL - .e EISNAM - .e EREMOTEIO - .e ENOMEDIUM - .e EMEDIUMTYPE - .e ENOKEY - .e EKEYEXPIRED - .e EKEYREVOKED - .e EKEYREJECTED - .e ERFKILL - .e EHWPOISON .long 0 .endobj kErrorNames,globl,hidden diff --git a/libc/integral/c.inc b/libc/integral/c.inc index 1eadabba6..9d009345c 100644 --- a/libc/integral/c.inc +++ b/libc/integral/c.inc @@ -584,6 +584,13 @@ typedef uint64_t uintmax_t; #define noasan #endif +#if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 408 || \ + __has_attribute(__no_sanitize_undefined__) +#define noubsan __attribute__((__no_sanitize_undefined__)) +#else +#define noubsan +#endif + #ifndef unreachable #if defined(__GNUC__) && !defined(__STRICT_ANSI__) #define unreachable __builtin_unreachable() diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index 8c3fd939e..5e442eba8 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -21,6 +21,7 @@ #include "libc/bits/likely.h" #include "libc/bits/weaken.h" #include "libc/calls/calls.h" +#include "libc/calls/struct/iovec.h" #include "libc/dce.h" #include "libc/intrin/asan.internal.h" #include "libc/log/log.h" @@ -116,7 +117,7 @@ struct AsanMorgue { static struct AsanMorgue __asan_morgue; -static uint64_t __asan_bsrl(uint64_t x) { +static inline int __asan_bsrl(uint64_t x) { return __builtin_clzll(x) ^ 63; } @@ -137,7 +138,7 @@ static size_t __asan_strlen(const char *s) { static int __asan_strcmp(const char *l, const char *r) { size_t i = 0; while (l[i] == r[i] && r[i]) ++i; - return (l[i] & 0xff) - (r[i] & 0xff); + return (l[i] & 255) - (r[i] & 255); } static char *__asan_stpcpy(char *d, const char *s) { @@ -168,7 +169,7 @@ static void *__asan_memset(void *p, int c, size_t n) { size_t i; uint64_t x; b = p; - x = 0x0101010101010101 * (c & 0xff); + x = 0x0101010101010101ul * (c & 255); switch (n) { case 0: return p; @@ -293,73 +294,105 @@ static void *__asan_memcpy(void *dst, const void *src, size_t n) { return dst; } -static size_t __asan_int2hex(uint64_t x, char b[17], uint8_t k) { - int i; - char *p; - for (p = b; k > 0;) { - *p++ = "0123456789abcdef"[(x >> (k -= 4)) & 15]; - } - *p = '\0'; - return p - b; +static char *__asan_hexcpy(char *p, uint64_t x, uint8_t k) { + while (k) *p++ = "0123456789abcdef"[(x >> (k -= 4)) & 15]; + return p; } -static size_t __asan_uint2str(uint64_t i, char *a) { - size_t j; - j = 0; +static char *__asan_uint2str(char *p, uint64_t i) { + int j = 0; do { - a[j++] = i % 10 + '0'; + p[j++] = i % 10 + '0'; i /= 10; } while (i > 0); - a[j] = '\0'; - reverse(a, j); - return j; + reverse(p, j); + return p + j; } -static size_t __asan_int2str(int64_t i, char *a) { - if (i >= 0) return __asan_uint2str(i, a); - *a++ = '-'; - return 1 + __asan_uint2str(-i, a); +static char *__asan_intcpy(char *p, int64_t i) { + if (i >= 0) return __asan_uint2str(p, i); + *p++ = '-'; + return __asan_uint2str(p, -i); } -void __asan_poison(uintptr_t p, size_t n, int kind) { - int k; - char *s; - if (!n) return; - if (UNLIKELY(p & 7)) { - k = MIN(8 - (p & 7), n); - s = SHADOW(p); - if (*s == 0 || *s > (p & 7)) { - *s = p & 7; - } - n -= k; - p += k; +void __asan_poison(uintptr_t p, size_t n, int t) { + signed char k, *s; + k = p & 7; + s = (signed char *)((p >> 3) + 0x7fff8000); + if (UNLIKELY(k)) { + if (n && (!*s || *s > k) && 8 - k >= n) *s = k; + ++s, n -= MIN(8 - k, n); } - __asan_memset(SHADOW(p), kind, n >> 3); + __asan_memset(s, t, n >> 3); if ((k = n & 7)) { - s = SHADOW(p + n); - if (*s < 0 || (*s > 0 && *s >= k)) { - *s = kind; - } + s += n >> 3; + if (*s < 0 || 0 < *s && *s <= k) *s = t; } } void __asan_unpoison(uintptr_t p, size_t n) { - int k; - char *s; - if (!n) return; - if (UNLIKELY(p & 7)) { - k = MIN(8 - (p & 7), n); - s = SHADOW(p); - *s = 0; - n -= k; - p += k; + signed char k, *s; + k = p & 7; + s = (signed char *)((p >> 3) + 0x7fff8000); + if (UNLIKELY(k)) { + if (n) *s = 0; + ++s, n -= MIN(8 - k, n); } - __asan_memset(SHADOW(p), 0, n >> 3); + __asan_memset(s, 0, n >> 3); if ((k = n & 7)) { - s = SHADOW(p + n); - if (*s && *s < k) { - *s = k; + s += n >> 3; + if (*s && *s < k) *s = k; + } +} + +bool __asan_is_valid(const void *p, size_t n) { + signed char k, *s, *e; + if (n) { + k = (uintptr_t)p & 7; + s = (signed char *)(((uintptr_t)p >> 3) + 0x7fff8000); + if (UNLIKELY(k)) { + if (n && !(!*s || *s >= k + n)) return false; + ++s, n -= MIN(8 - k, n); } + e = s; + k = n & 7; + e += n >> 3; + for (; s + 8 <= e; s += 8) { + if ((uint64_t)(255 & s[0]) << 000 | (uint64_t)(255 & s[1]) << 010 | + (uint64_t)(255 & s[2]) << 020 | (uint64_t)(255 & s[3]) << 030 | + (uint64_t)(255 & s[4]) << 040 | (uint64_t)(255 & s[5]) << 050 | + (uint64_t)(255 & s[6]) << 060 | (uint64_t)(255 & s[7]) << 070) { + return false; + } + } + while (s < e) { + if (*s++) { + return false; + } + } + if (k) { + if (!(!*s || *s >= k)) { + return false; + } + } + } + return true; +} + +bool __asan_is_valid_iov(const struct iovec *iov, int iovlen) { + int i; + size_t size; + if (iovlen >= 0 && + !__builtin_mul_overflow(iovlen, sizeof(struct iovec), &size) && + __asan_is_valid(iov, size)) { + for (i = 0; i < iovlen; ++i) { + if (!__asan_is_valid(iov[i].iov_base, iov[i].iov_len)) { + return false; + } + } + return true; + } else { + return false; } } @@ -376,7 +409,7 @@ static const char *__asan_dscribe_heap_poison(long c) { } } -static const char *__asan_describe_access_poison(char *p) { +static const char *__asan_describe_access_poison(signed char *p) { int c = p[0]; if (1 <= c && c <= 7) c = p[1]; switch (c) { @@ -445,15 +478,10 @@ static ssize_t __asan_write_string(const char *s) { return __asan_write(s, __asan_strlen(s)); } -static wontreturn void __asan_abort(void) { - if (weaken(__die)) weaken(__die)(); - __asan_exit(134); -} - static wontreturn void __asan_die(const char *msg) { __asan_write_string(msg); if (weaken(__die)) weaken(__die)(); - __asan_abort(); + __asan_exit(134); } static char *__asan_report_start(char *p) { @@ -472,9 +500,9 @@ static wontreturn void __asan_report_heap_fault(void *addr, long c) { p = __asan_report_start(buf); p = __asan_stpcpy(p, __asan_dscribe_heap_poison(c)); p = __asan_stpcpy(p, " at 0x"); - p = __asan_mempcpy(p, ibuf, __asan_int2hex((intptr_t)addr, ibuf, 48)); + p = __asan_hexcpy(p, (intptr_t)addr, 48); p = __asan_stpcpy(p, " shadow 0x"); - p = __asan_mempcpy(p, ibuf, __asan_int2hex((intptr_t)SHADOW(addr), ibuf, 48)); + p = __asan_hexcpy(p, (intptr_t)SHADOW(addr), 48); p = __asan_stpcpy(p, "\r\n"); __asan_die(buf); } @@ -485,20 +513,20 @@ static wontreturn void __asan_report_memory_fault(uint8_t *addr, int size, p = __asan_report_start(buf); p = __asan_stpcpy(p, __asan_describe_access_poison(SHADOW(addr))); p = __asan_stpcpy(p, " "); - p = __asan_mempcpy(p, ibuf, __asan_int2str(size, ibuf)); + p = __asan_intcpy(p, size); p = __asan_stpcpy(p, "-byte "); p = __asan_stpcpy(p, kind); p = __asan_stpcpy(p, " at 0x"); - p = __asan_mempcpy(p, ibuf, __asan_int2hex((intptr_t)addr, ibuf, 48)); + p = __asan_hexcpy(p, (uintptr_t)addr, 48); p = __asan_stpcpy(p, " shadow 0x"); - p = __asan_mempcpy(p, ibuf, __asan_int2hex((intptr_t)SHADOW(addr), ibuf, 48)); + p = __asan_hexcpy(p, (uintptr_t)SHADOW(addr), 48); p = __asan_stpcpy(p, "\r\n"); __asan_die(buf); } const void *__asan_morgue_add(void *p) { void *r; - unsigned i, j; + int i, j; for (;;) { i = __asan_morgue.i; j = (i + 1) & (ARRAYLEN(__asan_morgue.p) - 1); @@ -511,8 +539,8 @@ const void *__asan_morgue_add(void *p) { } static void __asan_morgue_flush(void) { + int i; void *p; - unsigned i; for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) { p = __asan_morgue.p[i]; if (cmpxchg(__asan_morgue.p + i, p, NULL)) { @@ -532,15 +560,16 @@ static size_t __asan_heap_size(size_t n) { } static void *__asan_allocate(size_t a, size_t n, int underrun, int overrun) { - char *p; size_t c; + char *p, *f; if ((p = weaken(dlmemalign)(a, __asan_heap_size(n)))) { c = weaken(dlmalloc_usable_size)(p); __asan_unpoison((uintptr_t)p, n); __asan_poison((uintptr_t)p - 16, 16, underrun); /* see dlmalloc design */ __asan_poison((uintptr_t)p + n, c - n, overrun); __asan_memset(p, 0xF9, n); - WRITE64BE(p + c - sizeof(n), n); + f = p + c - 8; + WRITE64BE(f, n); } return p; } @@ -548,7 +577,7 @@ static void *__asan_allocate(size_t a, size_t n, int underrun, int overrun) { static size_t __asan_malloc_usable_size(const void *p) { size_t c, n; if ((c = weaken(dlmalloc_usable_size)(p)) >= 8) { - if ((n = READ64BE((char *)p + c - sizeof(n))) <= c) { + if ((n = READ64BE((char *)p + c - 8)) <= c) { return n; } else { __asan_report_heap_fault(p, n); @@ -561,9 +590,9 @@ static size_t __asan_malloc_usable_size(const void *p) { static void __asan_deallocate(char *p, long kind) { size_t c, n; if ((c = weaken(dlmalloc_usable_size)(p)) >= 8) { - if ((n = READ64BE((char *)p + c - sizeof(n))) <= c) { - WRITE64BE((char *)p + c - sizeof(n), kind); - __asan_poison((uintptr_t)p, n, kind); + if ((n = READ64BE(p + c - 8)) <= c) { + WRITE64BE(p + c - 8, kind); + __asan_poison((uintptr_t)p, c - 8, kind); if (weaken(dlfree)) { weaken(dlfree)(__asan_morgue_add(p)); } @@ -588,43 +617,47 @@ static void *__asan_malloc(size_t size) { return __asan_memalign(16, size); } -static void *__asan_calloc(size_t nelem, size_t elsize) { +static void *__asan_calloc(size_t n, size_t m) { char *p; - size_t n; - if (__builtin_mul_overflow(nelem, elsize, &n)) n = -1; + if (__builtin_mul_overflow(n, m, &n)) n = -1; if ((p = __asan_malloc(n))) __asan_memset(p, 0, n); return p; } static void *__asan_realloc(void *p, size_t n) { - char *p2; + char *q, *f; size_t c, m; if (p) { if (n) { - if ((c = weaken(dlmalloc_usable_size)(p)) < 8) + if ((c = weaken(dlmalloc_usable_size)(p)) >= 8) { + f = (char *)p + c - 8; + if ((m = READ64BE(f)) <= c) { + if (n <= m) { /* shrink */ + __asan_poison((uintptr_t)p + n, m - n, kAsanHeapOverrun); + WRITE64BE(f, n); + q = p; + } else if (n <= c - 8) { /* small growth */ + __asan_unpoison((uintptr_t)p + m, n - m); + WRITE64BE(f, n); + q = p; + } else if ((q = __asan_malloc(n))) { /* exponential growth */ + __asan_memcpy(q, p, m); + __asan_deallocate(p, kAsanRelocated); + } + } else { + __asan_report_heap_fault(p, m); + } + } else { __asan_report_heap_fault(p, 0); - if ((m = READ64BE((char *)p + c - sizeof(n))) > c) - __asan_report_heap_fault(p, m); - if (n <= m) { /* shrink */ - __asan_poison((uintptr_t)p + n, m - n, kAsanHeapOverrun); - WRITE64BE((char *)p + c - sizeof(n), n); - p2 = p; - } else if (n <= c - 8) { /* small growth */ - __asan_unpoison((uintptr_t)p + m, n - m); - WRITE64BE((char *)p + c - sizeof(n), n); - p2 = p; - } else if ((p2 = __asan_malloc(n))) { /* exponential growth */ - __asan_memcpy(p2, p, m); - __asan_deallocate(p, kAsanRelocated); } } else { __asan_free(p); - p2 = NULL; + q = NULL; } } else { - p2 = __asan_malloc(n); + q = __asan_malloc(n); } - return p2; + return q; } static void *__asan_valloc(size_t n) { diff --git a/libc/intrin/asan.internal.h b/libc/intrin/asan.internal.h index 6e6bd3f6d..8f301e39b 100644 --- a/libc/intrin/asan.internal.h +++ b/libc/intrin/asan.internal.h @@ -1,5 +1,6 @@ #ifndef COSMOPOLITAN_LIBC_INTRIN_ASAN_H_ #define COSMOPOLITAN_LIBC_INTRIN_ASAN_H_ +#include "libc/calls/struct/iovec.h" #define kAsanScale 3 #define kAsanMagic 0x7fff8000 @@ -17,10 +18,12 @@ #define kAsanUnscoped -12 #define kAsanUnmapped -13 -#define SHADOW(x) ((char *)(((uintptr_t)(x) >> kAsanScale) + kAsanMagic)) +#define SHADOW(x) ((signed char *)(((uintptr_t)(x) >> kAsanScale) + kAsanMagic)) void __asan_map_shadow(uintptr_t, size_t); void __asan_poison(uintptr_t, size_t, int); void __asan_unpoison(uintptr_t, size_t); +bool __asan_is_valid(const void *, size_t); +bool __asan_is_valid_iov(const struct iovec *, int); #endif /* COSMOPOLITAN_LIBC_INTRIN_ASAN_H_ */ diff --git a/libc/intrin/intrin.mk b/libc/intrin/intrin.mk index d71f495e7..5faebf2f7 100644 --- a/libc/intrin/intrin.mk +++ b/libc/intrin/intrin.mk @@ -41,13 +41,11 @@ $(LIBC_INTRIN_A).pkg: \ $(LIBC_INTRIN_A_OBJS) \ $(foreach x,$(LIBC_INTRIN_A_DIRECTDEPS),$($(x)_A).pkg) -$(LIBC_INTRIN_A_OBJS): \ - OVERRIDE_CFLAGS += \ - $(NO_MAGIC) \ - -O3 - -o/$(MODE)/libc/intrin/asan.o: \ +o/$(MODE)/libc/intrin/asan.o \ +o/$(MODE)/libc/intrin/ubsan.o: \ OVERRIDE_CFLAGS += \ + -fno-sanitize=all \ + -fno-stack-protector \ -mgeneral-regs-only \ -O2 diff --git a/libc/intrin/pabsd.c b/libc/intrin/pabsd.c index d9ab15607..b01f3b12b 100644 --- a/libc/intrin/pabsd.c +++ b/libc/intrin/pabsd.c @@ -28,7 +28,7 @@ void(pabsd)(uint32_t a[4], const int32_t b[4]) { unsigned i; uint32_t r[4]; for (i = 0; i < 4; ++i) { - r[i] = ABS(b[i]); + r[i] = b[i] >= 0 ? b[i] : -(uint32_t)b[i]; } memcpy(a, r, 16); } diff --git a/libc/intrin/paddd.c b/libc/intrin/paddd.c index 019628279..e894c8adf 100644 --- a/libc/intrin/paddd.c +++ b/libc/intrin/paddd.c @@ -27,9 +27,11 @@ * @param 𝑐 [r/o] supplies second input vector * @mayalias */ -void(paddd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { +void(paddd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) { unsigned i; - int32_t r[4]; - for (i = 0; i < 4; ++i) r[i] = b[i] + c[i]; + uint32_t r[4]; + for (i = 0; i < 4; ++i) { + r[i] = b[i] + c[i]; + } memcpy(a, r, 16); } diff --git a/libc/intrin/paddd.h b/libc/intrin/paddd.h index 110d5807c..29e936740 100644 --- a/libc/intrin/paddd.h +++ b/libc/intrin/paddd.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void paddd(int32_t[4], const int32_t[4], const int32_t[4]); +void paddd(uint32_t[4], const uint32_t[4], const uint32_t[4]); #define paddd(A, B, C) \ INTRIN_SSEVEX_X_X_X_(paddd, SSE2, "paddd", INTRIN_COMMUTATIVE, A, B, C) diff --git a/libc/intrin/paddq.c b/libc/intrin/paddq.c index 072166497..56168c91d 100644 --- a/libc/intrin/paddq.c +++ b/libc/intrin/paddq.c @@ -27,9 +27,9 @@ * @param 𝑐 [r/o] supplies second input vector * @mayalias */ -void(paddq)(int64_t a[2], const int64_t b[2], const int64_t c[2]) { +void(paddq)(uint64_t a[2], const uint64_t b[2], const uint64_t c[2]) { unsigned i; - int64_t r[2]; + uint64_t r[2]; for (i = 0; i < 2; ++i) r[i] = b[i] + c[i]; memcpy(a, r, 16); } diff --git a/libc/intrin/paddq.h b/libc/intrin/paddq.h index 906f460b9..b7dc5635c 100644 --- a/libc/intrin/paddq.h +++ b/libc/intrin/paddq.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void paddq(int64_t[2], const int64_t[2], const int64_t[2]); +void paddq(uint64_t[2], const uint64_t[2], const uint64_t[2]); #define paddq(A, B, C) \ INTRIN_SSEVEX_X_X_X_(paddq, SSE2, "paddq", INTRIN_COMMUTATIVE, A, B, C) diff --git a/libc/intrin/phaddd.c b/libc/intrin/phaddd.c index 465c01d87..8ead1b4bb 100644 --- a/libc/intrin/phaddd.c +++ b/libc/intrin/phaddd.c @@ -28,7 +28,7 @@ * @note goes fast w/ ssse3 (intel c. 2004, amd c. 2011) * @mayalias */ -void(phaddd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { +void(phaddd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) { int32_t t[4]; t[0] = b[0] + b[1]; t[1] = b[2] + b[3]; diff --git a/libc/intrin/phaddd.h b/libc/intrin/phaddd.h index 96e68bd8a..7751dd1c6 100644 --- a/libc/intrin/phaddd.h +++ b/libc/intrin/phaddd.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void phaddd(int32_t[4], const int32_t[4], const int32_t[4]); +void phaddd(uint32_t[4], const uint32_t[4], const uint32_t[4]); #define phaddd(A, B, C) \ INTRIN_SSEVEX_X_X_X_(phaddd, SSSE3, "phaddd", INTRIN_NONCOMMUTATIVE, A, B, C) diff --git a/libc/intrin/phsubd.c b/libc/intrin/phsubd.c index 5285c3772..c8efc0430 100644 --- a/libc/intrin/phsubd.c +++ b/libc/intrin/phsubd.c @@ -28,8 +28,8 @@ * @note goes fast w/ ssse3 * @mayalias */ -void(phsubd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { - int32_t t[4]; +void(phsubd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) { + uint32_t t[4]; t[0] = b[0] - b[1]; t[1] = b[2] - b[3]; t[2] = c[0] - c[1]; diff --git a/libc/intrin/phsubd.h b/libc/intrin/phsubd.h index 641272fb9..a35a2b266 100644 --- a/libc/intrin/phsubd.h +++ b/libc/intrin/phsubd.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void phsubd(int32_t[4], const int32_t[4], const int32_t[4]); +void phsubd(uint32_t[4], const uint32_t[4], const uint32_t[4]); #define phsubd(A, B, C) \ INTRIN_SSEVEX_X_X_X_(phsubd, SSSE3, "phsubd", INTRIN_NONCOMMUTATIVE, A, B, C) diff --git a/libc/intrin/pmulhuw.c b/libc/intrin/pmulhuw.c index 2c162fa3b..0614cdedd 100644 --- a/libc/intrin/pmulhuw.c +++ b/libc/intrin/pmulhuw.c @@ -29,9 +29,13 @@ */ void(pmulhuw)(uint16_t a[8], const uint16_t b[8], const uint16_t c[8]) { unsigned i; + uint32_t x; uint16_t r[8]; for (i = 0; i < 8; ++i) { - r[i] = ((b[i] * c[i]) & 0xffff0000) >> 16; + x = b[i]; + x *= c[i]; + x >>= 16; + r[i] = x; } memcpy(a, r, 16); } diff --git a/libc/intrin/pmulld.c b/libc/intrin/pmulld.c index 9b7fe9e5f..b8c29dee9 100644 --- a/libc/intrin/pmulld.c +++ b/libc/intrin/pmulld.c @@ -20,7 +20,7 @@ #include "libc/str/str.h" /** - * Multiplies 32-bit signed integers. + * Multiplies 32-bit integers. * * @param 𝑎 [w/o] receives result * @param 𝑏 [r/o] supplies first input vector @@ -28,9 +28,9 @@ * @see pmuludq() * @mayalias */ -void(pmulld)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { +void(pmulld)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) { unsigned i; - int32_t r[4]; + uint32_t r[4]; for (i = 0; i < 4; ++i) { r[i] = b[i] * c[i]; } diff --git a/libc/intrin/pmulld.h b/libc/intrin/pmulld.h index 8365f5b1c..74e9563ae 100644 --- a/libc/intrin/pmulld.h +++ b/libc/intrin/pmulld.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void pmulld(int32_t[4], const int32_t[4], const int32_t[4]); +void pmulld(uint32_t[4], const uint32_t[4], const uint32_t[4]); #define pmulld(A, B, C) \ INTRIN_SSEVEX_X_X_X_(pmulld, SSE4_1, "pmulld", INTRIN_COMMUTATIVE, A, B, C) diff --git a/libc/intrin/psignd.c b/libc/intrin/psignd.c index 0ff4fc19e..2ae6e6c35 100644 --- a/libc/intrin/psignd.c +++ b/libc/intrin/psignd.c @@ -29,7 +29,7 @@ void(psignd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { if (!c[i]) { a[i] = 0; } else if (c[i] < 0) { - a[i] = -b[i]; + a[i] = -(uint32_t)b[i]; } else { a[i] = b[i]; } diff --git a/libc/intrin/pslld.c b/libc/intrin/pslld.c index a8c570ab6..18f342dc2 100644 --- a/libc/intrin/pslld.c +++ b/libc/intrin/pslld.c @@ -26,8 +26,8 @@ * @mayalias */ void(pslld)(uint32_t a[4], const uint32_t b[4], unsigned char c) { + unsigned i; if (c <= 31) { - unsigned i; for (i = 0; i < 4; ++i) { a[i] = b[i] << c; } diff --git a/libc/intrin/pslldq.c b/libc/intrin/pslldq.c index 8d6e5129e..74115e46d 100644 --- a/libc/intrin/pslldq.c +++ b/libc/intrin/pslldq.c @@ -20,7 +20,7 @@ #include "libc/str/str.h" /** - * Shifts vector right by n bytes w/ zero-fill. + * Shifts vector left by n bytes w/ zero-fill. * * @param a is input vector * @param b receives output @@ -28,7 +28,9 @@ */ void(pslldq)(uint8_t b[16], const uint8_t a[16], unsigned long n) { unsigned i; + uint8_t t[16]; if (n > 16) n = 16; - memmove(b + n, a, 16 - n); - memset(b, 0, n); + for (i = 0; i < n; ++i) t[i] = 0; + for (i = 0; i < 16 - n; ++i) t[n + i] = a[i]; + memcpy(b, t, 16); } diff --git a/libc/intrin/psllwv.c b/libc/intrin/psllwv.c index abb7f7b88..08da0d7ad 100644 --- a/libc/intrin/psllwv.c +++ b/libc/intrin/psllwv.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/psllw.h" +#include "libc/str/str.h" /** * Multiplies shorts by two power. @@ -25,13 +26,11 @@ */ void(psllwv)(uint16_t a[8], const uint16_t b[8], const uint64_t c[2]) { unsigned i; - if (c[0] > 15) { - for (i = 0; i < 8; ++i) { - a[i] = 0; - } - } else { + if (c[0] < 16) { for (i = 0; i < 8; ++i) { a[i] = b[i] << c[0]; } + } else { + memset(a, 0, 16); } } diff --git a/libc/intrin/psrad.c b/libc/intrin/psrad.c index c6fda7ed6..694998048 100644 --- a/libc/intrin/psrad.c +++ b/libc/intrin/psrad.c @@ -26,14 +26,9 @@ * @mayalias */ void(psrad)(int32_t a[4], const int32_t b[4], unsigned char k) { - unsigned i, x, m; + unsigned i; if (k > 31) k = 31; for (i = 0; i < 4; ++i) { - m = 0; - x = b[i]; - if (x & 0x80000000) m = ~(0xffffffffu >> k); - x >>= k; - x |= m; - a[i] = x; + a[i] = b[i] >> k; } } diff --git a/libc/intrin/psradv.c b/libc/intrin/psradv.c index 36e49eb94..8506a4ffa 100644 --- a/libc/intrin/psradv.c +++ b/libc/intrin/psradv.c @@ -25,15 +25,10 @@ * @mayalias */ void(psradv)(int32_t a[4], const int32_t b[4], const uint64_t c[2]) { + unsigned i; unsigned char k; - unsigned i, x, m; k = c[0] > 31 ? 31 : c[0]; for (i = 0; i < 4; ++i) { - m = 0; - x = b[i]; - if (x & 0x80000000u) m = ~(0xffffffffu >> k); - x >>= k; - x |= m; - a[i] = x & 0xffffffffu; + a[i] = b[i] >> k; } } diff --git a/libc/intrin/psraw.c b/libc/intrin/psraw.c index 0ab46b0da..d263cf094 100644 --- a/libc/intrin/psraw.c +++ b/libc/intrin/psraw.c @@ -26,14 +26,9 @@ * @mayalias */ void(psraw)(int16_t a[8], const int16_t b[8], unsigned char k) { - unsigned i, x, m; + unsigned i; if (k > 15) k = 15; for (i = 0; i < 8; ++i) { - m = 0; - x = b[i]; - if (x & 0x8000) m = ~(0xffff >> k); - x >>= k; - x |= m; - a[i] = x; + a[i] = b[i] >> k; } } diff --git a/libc/intrin/psrawv.c b/libc/intrin/psrawv.c index b1109bf83..0024bffb4 100644 --- a/libc/intrin/psrawv.c +++ b/libc/intrin/psrawv.c @@ -25,15 +25,10 @@ * @mayalias */ void(psrawv)(int16_t a[8], const int16_t b[8], const uint64_t c[2]) { + unsigned i; unsigned char k; - unsigned i, x, m; k = c[0] > 15 ? 15 : c[0]; for (i = 0; i < 8; ++i) { - m = 0; - x = b[i]; - if (x & 0x8000) m = ~(0xffffu >> k); - x >>= k; - x |= m; - a[i] = x & 0xffffu; + a[i] = b[i] >> k; } } diff --git a/libc/intrin/psubd.c b/libc/intrin/psubd.c index 0df04acfe..f93d9dac5 100644 --- a/libc/intrin/psubd.c +++ b/libc/intrin/psubd.c @@ -27,9 +27,9 @@ * @param 𝑐 [r/o] supplies second input vector * @mayalias */ -void(psubd)(int32_t a[4], const int32_t b[4], const int32_t c[4]) { +void(psubd)(uint32_t a[4], const uint32_t b[4], const uint32_t c[4]) { unsigned i; - int32_t r[4]; + uint32_t r[4]; for (i = 0; i < 4; ++i) { r[i] = b[i] - c[i]; } diff --git a/libc/intrin/psubd.h b/libc/intrin/psubd.h index c8f396da4..4aa30b9ce 100644 --- a/libc/intrin/psubd.h +++ b/libc/intrin/psubd.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void psubd(int32_t[4], const int32_t[4], const int32_t[4]); +void psubd(uint32_t[4], const uint32_t[4], const uint32_t[4]); #define psubd(A, B, C) \ INTRIN_SSEVEX_X_X_X_(psubd, SSE2, "psubd", INTRIN_NONCOMMUTATIVE, A, B, C) diff --git a/libc/intrin/psubq.c b/libc/intrin/psubq.c index 363afc35c..495f6d5ca 100644 --- a/libc/intrin/psubq.c +++ b/libc/intrin/psubq.c @@ -27,9 +27,9 @@ * @param 𝑐 [r/o] supplies second input vector * @mayalias */ -void(psubq)(int64_t a[2], const int64_t b[2], const int64_t c[2]) { +void(psubq)(uint64_t a[2], const uint64_t b[2], const uint64_t c[2]) { unsigned i; - int64_t r[2]; + uint64_t r[2]; for (i = 0; i < 2; ++i) { r[i] = b[i] - c[i]; } diff --git a/libc/intrin/psubq.h b/libc/intrin/psubq.h index d08bb6f7c..9b4a5d3b9 100644 --- a/libc/intrin/psubq.h +++ b/libc/intrin/psubq.h @@ -4,7 +4,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void psubq(int64_t[2], const int64_t[2], const int64_t[2]); +void psubq(uint64_t[2], const uint64_t[2], const uint64_t[2]); #define psubq(A, B, C) \ INTRIN_SSEVEX_X_X_X_(psubq, SSE2, "psubq", INTRIN_NONCOMMUTATIVE, A, B, C) diff --git a/libc/intrin/somanyasan.S b/libc/intrin/somanyasan.S index 5bd1076be..818314184 100644 --- a/libc/intrin/somanyasan.S +++ b/libc/intrin/somanyasan.S @@ -19,6 +19,16 @@ #include "libc/macros.internal.h" .source __FILE__ + .macro .acall fn:req + xor %eax,%eax + mov $1,%r10b + cmpxchg %r10b,__asan_noreentry(%rip) + jnz 2f + call \fn + decb __asan_noreentry(%rip) +2: nop + .endm + .rodata.cst4 __asan_option_detect_stack_use_after_return: .long 0 @@ -32,181 +42,362 @@ __asan_noreentry: .previous __asan_report_load1: - push $1 - jmp OnReportLoad + push %rbp + mov %rsp,%rbp + .profilable + mov $1,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load1,globl + __asan_report_load2: - push $2 - jmp OnReportLoad + push %rbp + mov %rsp,%rbp + .profilable + mov $2,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load2,globl + __asan_report_load4: - push $4 - jmp OnReportLoad + push %rbp + mov %rsp,%rbp + .profilable + mov $4,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load4,globl + __asan_report_load8: - push $8 - jmp OnReportLoad + push %rbp + mov %rsp,%rbp + .profilable + mov $8,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load8,globl + __asan_report_load16: - push $16 - jmp OnReportLoad + push %rbp + mov %rsp,%rbp + .profilable + mov $16,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load16,globl + __asan_report_load32: - push $32 -// 𝑠𝑙𝑖𝑑𝑒 + push %rbp + mov %rsp,%rbp + .profilable + mov $32,%esi + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load32,globl -OnReportLoad: - pop %rsi -// 𝑠𝑙𝑖𝑑𝑒 - .endfn OnReportLoad + __asan_report_load_n: - lea __asan_report_load(%rip),%r11 - jmp __asan_report_noreentry + push %rbp + mov %rsp,%rbp + .profilable + .acall __asan_report_load + pop %rbp + ret .endfn __asan_report_load_n,globl __asan_report_store1: - push $1 - jmp ReportStore - .endfn __asan_report_store1,globl -__asan_report_store2: - push $2 - jmp ReportStore - .endfn __asan_report_store2,globl -__asan_report_store4: - push $4 - jmp ReportStore - .endfn __asan_report_store4,globl -__asan_report_store8: - push $8 - jmp ReportStore - .endfn __asan_report_store8,globl -__asan_report_store16: - push $16 - jmp ReportStore - .endfn __asan_report_store16,globl -__asan_report_store32: - push $32 -// 𝑠𝑙𝑖𝑑𝑒 - .endfn __asan_report_store32,globl -ReportStore: - pop %rsi -// 𝑠𝑙𝑖𝑑𝑒 - .endfn ReportStore -__asan_report_store_n: - lea __asan_report_store(%rip),%r11 -// 𝑠𝑙𝑖𝑑𝑒 - .endfn __asan_report_store_n,globl - -__asan_report_noreentry: push %rbp mov %rsp,%rbp - xor %eax,%eax - mov $1,%r10b - cmpxchg %r10b,__asan_noreentry(%rip) - jnz 2f - call *%r11 - decb __asan_noreentry(%rip) -2: pop %rbp + .profilable + mov $1,%esi + .acall __asan_report_store + pop %rbp ret - .endfn __asan_report_noreentry + .endfn __asan_report_store1,globl + +__asan_report_store2: + push %rbp + mov %rsp,%rbp + .profilable + mov $2,%esi + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store2,globl + +__asan_report_store4: + push %rbp + mov %rsp,%rbp + .profilable + mov $4,%esi + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store4,globl + +__asan_report_store8: + push %rbp + mov %rsp,%rbp + .profilable + mov $8,%esi + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store8,globl + +__asan_report_store16: + push %rbp + mov %rsp,%rbp + .profilable + mov $16,%esi + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store16,globl + +__asan_report_store32: + push %rbp + mov %rsp,%rbp + .profilable + mov $32,%esi + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store32,globl + +__asan_report_store_n: + push %rbp + mov %rsp,%rbp + .profilable + .acall __asan_report_store + pop %rbp + ret + .endfn __asan_report_store_n,globl __asan_stack_free_0: - push $0 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $0,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_0,globl + __asan_stack_free_1: - push $1 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $1,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_1,globl + __asan_stack_free_2: - push $2 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $2,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_2,globl + __asan_stack_free_3: - push $3 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $3,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_3,globl + __asan_stack_free_4: - push $4 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $4,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_4,globl + __asan_stack_free_5: - push $5 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $5,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_5,globl + __asan_stack_free_6: - push $6 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $6,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_6,globl + __asan_stack_free_7: - push $7 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $7,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_7,globl + __asan_stack_free_8: - push $8 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $8,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_8,globl + __asan_stack_free_9: - push $9 - jmp OnStackFree + push %rbp + mov %rsp,%rbp + .profilable + mov $9,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_9,globl + __asan_stack_free_10: - push $10 -// 𝑠𝑙𝑖𝑑𝑒 + push %rbp + mov %rsp,%rbp + .profilable + mov $10,%edx + call __asan_stack_free + pop %rbp + ret .endfn __asan_stack_free_10,globl -OnStackFree: - pop %rdx - jmp __asan_stack_free - .endfn OnStackFree __asan_stack_malloc_0: - push $0 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $0,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_0,globl + __asan_stack_malloc_1: - push $1 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $1,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_1,globl + __asan_stack_malloc_2: - push $2 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $2,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_2,globl + __asan_stack_malloc_3: - push $3 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $3,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_3,globl + __asan_stack_malloc_4: - push $4 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $4,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_4,globl + __asan_stack_malloc_5: - push $5 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $5,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_5,globl + __asan_stack_malloc_6: - push $6 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $6,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_6,globl + __asan_stack_malloc_7: - push $7 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $7,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_7,globl + __asan_stack_malloc_8: - push $8 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $8,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_8,globl + __asan_stack_malloc_9: - push $9 - jmp OnStackMalloc + push %rbp + mov %rsp,%rbp + .profilable + mov $9,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_9,globl + __asan_stack_malloc_10: - push $10 -// 𝑠𝑙𝑖𝑑𝑒 + push %rbp + mov %rsp,%rbp + .profilable + mov $10,%esi + call __asan_stack_malloc + pop %rbp + ret .endfn __asan_stack_malloc_10,globl -OnStackMalloc: - pop %rsi - jmp __asan_stack_malloc - .endfn OnStackMalloc __asan_version_mismatch_check_v8: ret @@ -242,26 +433,31 @@ __asan_load1: mov %rsp,%rbp ud2 .endfn __asan_load1,globl + __asan_load2: push %rbp mov %rsp,%rbp ud2 .endfn __asan_load2,globl + __asan_load4: push %rbp mov %rsp,%rbp ud2 .endfn __asan_load4,globl + __asan_load8: push %rbp mov %rsp,%rbp ud2 .endfn __asan_load8,globl + __asan_load16: push %rbp mov %rsp,%rbp ud2 .endfn __asan_load16,globl + __asan_load32: push %rbp mov %rsp,%rbp @@ -273,26 +469,31 @@ __asan_store1: mov %rsp,%rbp ud2 .endfn __asan_store1,globl + __asan_store2: push %rbp mov %rsp,%rbp ud2 .endfn __asan_store2,globl + __asan_store4: push %rbp mov %rsp,%rbp ud2 .endfn __asan_store4,globl + __asan_store8: push %rbp mov %rsp,%rbp ud2 .endfn __asan_store8,globl + __asan_store16: push %rbp mov %rsp,%rbp ud2 .endfn __asan_store16,globl + __asan_store32: push %rbp mov %rsp,%rbp diff --git a/libc/intrin/ubsan.c b/libc/intrin/ubsan.c new file mode 100644 index 000000000..150133fca --- /dev/null +++ b/libc/intrin/ubsan.c @@ -0,0 +1,593 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/alg/reverse.internal.h" +#include "libc/bits/pushpop.h" +#include "libc/bits/weaken.h" +#include "libc/calls/calls.h" +#include "libc/fmt/fmt.h" +#include "libc/log/internal.h" +#include "libc/log/log.h" +#include "libc/nt/runtime.h" +#include "libc/runtime/internal.h" +#include "libc/runtime/runtime.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/fileno.h" +#include "libc/sysv/consts/nr.h" + +#define kUbsanKindInt 0 +#define kUbsanKindFloat 1 +#define kUbsanKindUnknown 0xffff + +struct UbsanSourceLocation { + const char *file; + uint32_t line; + uint32_t column; +}; + +struct UbsanTypeDescriptor { + uint16_t kind; /* int,float,... */ + uint16_t info; /* if int bit 0 if signed, remaining bits are log2(sizeof*8) */ + char name[]; +}; + +struct UbsanTypeMismatchInfo { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; + uintptr_t alignment; + uint8_t type_check_kind; +}; + +struct UbsanTypeMismatchInfoClang { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; + unsigned char log_alignment; /* https://reviews.llvm.org/D28244 */ + uint8_t type_check_kind; +}; + +struct UbsanOutOfBoundsInfo { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *array_type; + struct UbsanTypeDescriptor *index_type; +}; + +struct UbsanUnreachableData { + struct UbsanSourceLocation location; +}; + +struct UbsanVlaBoundData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; +}; + +struct UbsanNonnullArgData { + struct UbsanSourceLocation location; + struct UbsanSourceLocation attr_location; +}; + +struct UbsanCfiBadIcallData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; +}; + +struct UbsanNonnullReturnData { + struct UbsanSourceLocation location; + struct UbsanSourceLocation attr_location; +}; + +struct UbsanFunctionTypeMismatchData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; +}; + +struct UbsanInvalidValueData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; +}; + +struct UbsanOverflowData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *type; +}; + +struct UbsanFloatCastOverflowData { +#if __GNUC__ + 0 >= 6 + struct UbsanSourceLocation location; +#endif + struct UbsanTypeDescriptor *from_type; + struct UbsanTypeDescriptor *to_type; +}; + +struct UbsanOutOfBoundsData { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *arraytype; + struct UbsanTypeDescriptor *index_type; +}; + +struct UbsanShiftOutOfBoundsInfo { + struct UbsanSourceLocation location; + struct UbsanTypeDescriptor *lhs_type; + struct UbsanTypeDescriptor *rhs_type; +}; + +static const char kUbsanTypeCheckKinds[] = "\ +load of\0\ +store to\0\ +reference binding to\0\ +member access within\0\ +member call on\0\ +constructor call on\0\ +downcast of\0\ +downcast of\0\ +upcast of\0\ +cast to virtual base of\0\ +\0"; + +static int __ubsan_bits(struct UbsanTypeDescriptor *t) { + return 1 << (t->info >> 1); +} + +static bool __ubsan_signed(struct UbsanTypeDescriptor *t) { + return t->info & 1; +} + +static bool __ubsan_negative(struct UbsanTypeDescriptor *t, uintptr_t x) { + return __ubsan_signed(t) && (intptr_t)x < 0; +} + +static size_t __ubsan_strlen(const char *s) { + size_t n = 0; + while (*s++) ++n; + return n; +} + +static char *__ubsan_stpcpy(char *d, const char *s) { + size_t i; + for (i = 0;; ++i) { + if (!(d[i] = s[i])) { + return d + i; + } + } +} + +static char *__ubsan_poscpy(char *p, uintptr_t i) { + int j = 0; + do { + p[j++] = i % 10 + '0'; + i /= 10; + } while (i > 0); + reverse(p, j); + return p + j; +} + +static char *__ubsan_intcpy(char *p, intptr_t i) { + if (i >= 0) return __ubsan_poscpy(p, i); + *p++ = '-'; + return __ubsan_poscpy(p, -i); +} + +static char *__ubsan_hexcpy(char *p, uintptr_t x, int k) { + while (k) *p++ = "0123456789abcdef"[(x >> (k -= 4)) & 15]; + return p; +} + +static char *__ubsan_itpcpy(char *p, struct UbsanTypeDescriptor *t, + uintptr_t x) { + if (__ubsan_signed(t)) { + return __ubsan_intcpy(p, x); + } else { + return __ubsan_poscpy(p, x); + } +} + +static const char *__ubsan_dubnul(const char *s, unsigned i) { + size_t n; + while (i--) { + if ((n = __ubsan_strlen(s))) { + s += n + 1; + } else { + return NULL; + } + } + return s; +} + +static uintptr_t __ubsan_extend(struct UbsanTypeDescriptor *t, uintptr_t x) { + int w; + w = __ubsan_bits(t); + if (w < sizeof(x) * CHAR_BIT) { + x <<= sizeof(x) * CHAR_BIT - w; + if (__ubsan_signed(t)) { + x = (intptr_t)x >> w; + } else { + x >>= w; + } + } + return x; +} + +static privileged noinline wontreturn void __ubsan_exit(int rc) { + if (!IsWindows()) { + asm volatile("syscall" + : /* no outputs */ + : "a"(__NR_exit_group), "D"(rc) + : "memory"); + unreachable; + } else { + ExitProcess(rc); + } +} + +static privileged noinline ssize_t __ubsan_write(const void *data, + size_t size) { + ssize_t rc; + uint32_t wrote; + if (!IsWindows()) { + asm volatile("syscall" + : "=a"(rc) + : "0"(__NR_write), "D"(2), "S"(data), "d"(size) + : "rcx", "r11", "memory"); + return rc; + } else { + if (WriteFile(GetStdHandle(kNtStdErrorHandle), data, size, &wrote, 0)) { + return wrote; + } else { + return -1; + } + } +} + +static ssize_t __ubsan_write_string(const char *s) { + return __ubsan_write(s, __ubsan_strlen(s)); +} + +void __ubsan_abort(const struct UbsanSourceLocation *loc, + const char *description) { + char buf[1024], *p = buf; + p = __ubsan_stpcpy(p, "error: "); + p = __ubsan_stpcpy(p, loc->file), *p++ = ':'; + p = __ubsan_intcpy(p, loc->line); + p = __ubsan_stpcpy(p, ": "); + p = __ubsan_stpcpy(p, description); + p = __ubsan_stpcpy(p, "\r\n"); + __ubsan_write_string(buf); + if (weaken(__die)) weaken(__die)(); + __ubsan_exit(134); +} + +static const char *__ubsan_describe_shift( + struct UbsanShiftOutOfBoundsInfo *info, uintptr_t lhs, uintptr_t rhs) { + if (__ubsan_negative(info->rhs_type, rhs)) { + return "shift exponent is negative"; + } else if (rhs >= __ubsan_bits(info->lhs_type)) { + return "shift exponent too large for type"; + } else if (__ubsan_negative(info->lhs_type, lhs)) { + return "left shift of negative value"; + } else if (__ubsan_signed(info->lhs_type)) { + return "signed left shift changed sign bit or overflowed"; + } else { + return "wut shift out of bounds"; + } +} + +void __ubsan_handle_shift_out_of_bounds(struct UbsanShiftOutOfBoundsInfo *info, + uintptr_t lhs, uintptr_t rhs) { + char buf[512], *p = buf; + lhs = __ubsan_extend(info->lhs_type, lhs); + rhs = __ubsan_extend(info->rhs_type, rhs); + p = __ubsan_stpcpy(p, __ubsan_describe_shift(info, lhs, rhs)), *p++ = ' '; + p = __ubsan_itpcpy(p, info->lhs_type, lhs), *p++ = ' '; + p = __ubsan_stpcpy(p, info->lhs_type->name), *p++ = ' '; + p = __ubsan_itpcpy(p, info->rhs_type, rhs), *p++ = ' '; + p = __ubsan_stpcpy(p, info->rhs_type->name); + __ubsan_abort(&info->location, buf); +} + +void __ubsan_handle_shift_out_of_bounds_abort( + struct UbsanShiftOutOfBoundsInfo *info, uintptr_t lhs, uintptr_t rhs) { + __ubsan_handle_shift_out_of_bounds(info, lhs, rhs); +} + +void __ubsan_handle_out_of_bounds(struct UbsanOutOfBoundsInfo *info, + uintptr_t index) { + char buf[512], *p = buf; + p = __ubsan_stpcpy(p, info->index_type->name); + p = __ubsan_stpcpy(p, " index "); + p = __ubsan_itpcpy(p, info->index_type, index); + p = __ubsan_stpcpy(p, " into "); + p = __ubsan_stpcpy(p, info->array_type->name); + p = __ubsan_stpcpy(p, " out of bounds"); + __ubsan_abort(&info->location, buf); +} + +void __ubsan_handle_out_of_bounds_abort(struct UbsanOutOfBoundsInfo *info, + uintptr_t index) { + __ubsan_handle_out_of_bounds(info, index); +} + +void __ubsan_handle_type_mismatch(struct UbsanTypeMismatchInfo *info, + uintptr_t pointer) { + const char *kind; + char buf[512], *p = buf; + if (!pointer) __ubsan_abort(&info->location, "null pointer access"); + kind = __ubsan_dubnul(kUbsanTypeCheckKinds, info->type_check_kind); + if (info->alignment && (pointer & (info->alignment - 1))) { + p = __ubsan_stpcpy(p, "unaligned "); + p = __ubsan_stpcpy(p, kind), *p++ = ' '; + p = __ubsan_stpcpy(p, info->type->name), *p++ = ' ', *p++ = '@'; + p = __ubsan_itpcpy(p, info->type, pointer); + p = __ubsan_stpcpy(p, " align "); + p = __ubsan_intcpy(p, info->alignment); + } else { + p = __ubsan_stpcpy(p, "insufficient size\r\n\t"); + p = __ubsan_stpcpy(p, kind); + p = __ubsan_stpcpy(p, " address 0x"); + p = __ubsan_hexcpy(p, pointer, sizeof(pointer) * CHAR_BIT); + p = __ubsan_stpcpy(p, " with insufficient space for object of type "); + p = __ubsan_stpcpy(p, info->type->name); + } + __ubsan_abort(&info->location, buf); +} + +void __ubsan_handle_type_mismatch_abort(struct UbsanTypeMismatchInfo *info, + uintptr_t pointer) { + __ubsan_handle_type_mismatch(info, pointer); +} + +void __ubsan_handle_type_mismatch_v1( + struct UbsanTypeMismatchInfoClang *type_mismatch, uintptr_t pointer) { + struct UbsanTypeMismatchInfo mm; + mm.location = type_mismatch->location; + mm.type = type_mismatch->type; + mm.alignment = 1u << type_mismatch->log_alignment; + mm.type_check_kind = type_mismatch->type_check_kind; + __ubsan_handle_type_mismatch(&mm, pointer); +} + +void __ubsan_handle_type_mismatch_v1_abort( + struct UbsanTypeMismatchInfoClang *type_mismatch, uintptr_t pointer) { + __ubsan_handle_type_mismatch_v1(type_mismatch, pointer); +} + +void __ubsan_handle_float_cast_overflow(void *data_raw, void *from_raw) { + struct UbsanFloatCastOverflowData *data = + (struct UbsanFloatCastOverflowData *)data_raw; +#if __GNUC__ + 0 >= 6 + __ubsan_abort(&data->location, "float cast overflow"); +#else + const struct UbsanSourceLocation kUnknownLocation = { + "", + pushpop(0), + pushpop(0), + }; + __ubsan_abort(((void)data, &kUnknownLocation), "float cast overflow"); +#endif +} + +void __ubsan_handle_float_cast_overflow_abort(void *data_raw, void *from_raw) { + __ubsan_handle_float_cast_overflow(data_raw, from_raw); +} + +void __ubsan_handle_add_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "add overflow"); +} + +void __ubsan_handle_add_overflow_abort(const struct UbsanSourceLocation *loc) { + __ubsan_handle_add_overflow(loc); +} + +void __ubsan_handle_alignment_assumption( + const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "alignment assumption"); +} + +void __ubsan_handle_alignment_assumption_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_alignment_assumption(loc); +} + +void __ubsan_handle_builtin_unreachable(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "builtin unreachable"); +} + +void __ubsan_handle_builtin_unreachable_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_builtin_unreachable(loc); +} + +void __ubsan_handle_cfi_bad_type(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "cfi bad type"); +} + +void __ubsan_handle_cfi_bad_type_abort(const struct UbsanSourceLocation *loc) { + __ubsan_handle_cfi_bad_type(loc); +} + +void __ubsan_handle_cfi_check_fail(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "cfi check fail"); +} + +void __ubsan_handle_cfi_check_fail_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_cfi_check_fail(loc); +} + +void __ubsan_handle_divrem_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "divrem overflow"); +} + +void __ubsan_handle_divrem_overflow_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_divrem_overflow(loc); +} + +void __ubsan_handle_dynamic_type_cache_miss( + const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "dynamic type cache miss"); +} + +void __ubsan_handle_dynamic_type_cache_miss_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_dynamic_type_cache_miss(loc); +} + +void __ubsan_handle_function_type_mismatch( + const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "function type mismatch"); +} + +void __ubsan_handle_function_type_mismatch_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_function_type_mismatch(loc); +} + +void __ubsan_handle_implicit_conversion(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "implicit conversion"); +} + +void __ubsan_handle_implicit_conversion_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_implicit_conversion(loc); +} + +void __ubsan_handle_invalid_builtin(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "invalid builtin"); +} + +void __ubsan_handle_invalid_builtin_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_invalid_builtin(loc); +} + +void __ubsan_handle_load_invalid_value(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "load invalid value (uninitialized? bool∌[01]?)"); +} + +void __ubsan_handle_load_invalid_value_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_load_invalid_value(loc); +} + +void __ubsan_handle_missing_return(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "missing return"); +} + +void __ubsan_handle_missing_return_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_missing_return(loc); +} + +void __ubsan_handle_mul_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "multiply overflow"); +} + +void __ubsan_handle_mul_overflow_abort(const struct UbsanSourceLocation *loc) { + __ubsan_handle_mul_overflow(loc); +} + +void __ubsan_handle_negate_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "negate overflow"); +} + +void __ubsan_handle_negate_overflow_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_negate_overflow(loc); +} + +void __ubsan_handle_nonnull_arg(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "nonnull argument"); +} + +void __ubsan_handle_nonnull_arg_abort(const struct UbsanSourceLocation *loc) { + __ubsan_handle_nonnull_arg(loc); +} + +void __ubsan_handle_nonnull_return_v1(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "non-null return (v1)"); +} + +void __ubsan_handle_nonnull_return_v1_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_nonnull_return_v1(loc); +} + +void __ubsan_handle_nullability_arg(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "nullability arg"); +} + +void __ubsan_handle_nullability_arg_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_nullability_arg(loc); +} + +void __ubsan_handle_nullability_return_v1( + const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "nullability return (v1)"); +} + +void __ubsan_handle_nullability_return_v1_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_nullability_return_v1(loc); +} + +void __ubsan_handle_pointer_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "pointer overflow"); +} + +void __ubsan_handle_pointer_overflow_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_pointer_overflow(loc); +} + +void __ubsan_handle_sub_overflow(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "subtract overflow"); +} + +void __ubsan_handle_sub_overflow_abort(const struct UbsanSourceLocation *loc) { + __ubsan_handle_sub_overflow(loc); +} + +void __ubsan_handle_vla_bound_not_positive( + const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "vla bound not positive"); +} + +void __ubsan_handle_vla_bound_not_positive_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_vla_bound_not_positive(loc); +} + +void __ubsan_handle_nonnull_return(const struct UbsanSourceLocation *loc) { + __ubsan_abort(loc, "nonnull return"); +} + +void __ubsan_handle_nonnull_return_abort( + const struct UbsanSourceLocation *loc) { + __ubsan_handle_nonnull_return(loc); +} + +void __ubsan_default_options(void) { +} + +void __ubsan_on_report(void) { +} + +void *__ubsan_get_current_report_data(void) { + return 0; +} diff --git a/libc/isystem/iso646.h b/libc/isystem/iso646.h new file mode 100644 index 000000000..23b124b48 --- /dev/null +++ b/libc/isystem/iso646.h @@ -0,0 +1,16 @@ +#ifndef COSMOPOLITAN_LIBC_ISYSTEM_ISO646_H_ +#define COSMOPOLITAN_LIBC_ISYSTEM_ISO646_H_ + +#define and && +#define and_eq &= +#define bitand & +#define bitor | +#define compl ~ +#define not ! +#define not_eq != +#define or || +#define or_eq |= +#define xor ^ +#define xor_eq ^= + +#endif diff --git a/libc/log/die.c b/libc/log/die.c index 066c27682..998598564 100644 --- a/libc/log/die.c +++ b/libc/log/die.c @@ -29,10 +29,8 @@ relegated wontreturn void __die(void) { static bool once; if (cmpxchg(&once, false, true)) { - if (!IsTiny()) { - if (IsDebuggerPresent(false)) DebugBreak(); - ShowBacktrace(2, NULL); - } + if (IsDebuggerPresent(false)) DebugBreak(); + ShowBacktrace(2, NULL); } exit(77); } diff --git a/libc/log/log.h b/libc/log/log.h index 170153b4c..ba54a42eb 100644 --- a/libc/log/log.h +++ b/libc/log/log.h @@ -1,5 +1,6 @@ #ifndef COSMOPOLITAN_LIBC_LOG_LOG_H_ #define COSMOPOLITAN_LIBC_LOG_LOG_H_ +#include "libc/bits/likely.h" #include "libc/calls/struct/sigset.h" #include "libc/calls/struct/winsize.h" #include "libc/stdio/stdio.h" diff --git a/libc/log/oncrash.c b/libc/log/oncrash.c index 3c4bbbcd2..803c918ba 100644 --- a/libc/log/oncrash.c +++ b/libc/log/oncrash.c @@ -133,7 +133,7 @@ relegated static void ShowGeneralRegisters(int fd, ucontext_t *ctx) { } else { memset(&st, 0, sizeof(st)); } - dprintf(fd, " %s(%zu) %Lf", "ST", k, st); + dprintf(fd, " %s(%zu) %Lg", "ST", k, st); ++k; write(fd, "\r\n", 2); } diff --git a/libc/log/perror.c b/libc/log/perror.c index e8ea6e21e..bb5a9c25f 100644 --- a/libc/log/perror.c +++ b/libc/log/perror.c @@ -17,12 +17,8 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/fmt/fmt.h" -#include "libc/log/internal.h" #include "libc/log/log.h" -#include "libc/runtime/runtime.h" #include "libc/stdio/stdio.h" -#include "libc/str/str.h" /** * Writes error messages to standard error. diff --git a/libc/log/showcrashreports.c b/libc/log/showcrashreports.c index d7a0b726d..c07acccf7 100644 --- a/libc/log/showcrashreports.c +++ b/libc/log/showcrashreports.c @@ -23,7 +23,6 @@ #include "libc/dce.h" #include "libc/log/internal.h" #include "libc/log/log.h" -#include "libc/log/ubsan.internal.h" #include "libc/macros.internal.h" #include "libc/nt/signals.h" #include "libc/str/str.h" @@ -31,7 +30,6 @@ #include "libc/sysv/consts/sig.h" STATIC_YOINK("__die"); -STATIC_YOINK("__ubsan_abort"); extern const unsigned char __oncrash_thunks[7][11]; diff --git a/libc/log/somanyubsan.S b/libc/log/somanyubsan.S deleted file mode 100644 index 9e481ef09..000000000 --- a/libc/log/somanyubsan.S +++ /dev/null @@ -1,265 +0,0 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/macros.internal.h" -.text.unlikely -.source __FILE__ - -__ubsan_default_options: - ret - .endfn __ubsan_default_options - -__ubsan_on_report: - ret - .endfn __ubsan_on_report,globl - -__ubsan_get_current_report_data: - xor %eax,%eax - ret - .endfn __ubsan_get_current_report_data,globl - -__ubsan_handle_type_mismatch_abort: - jmp __ubsan_handle_type_mismatch - .endfn __ubsan_handle_type_mismatch_abort,globl - -__ubsan_handle_float_cast_overflow_abort: - jmp __ubsan_handle_float_cast_overflow - .endfn __ubsan_handle_float_cast_overflow_abort,globl - -__ubsan_handle_type_mismatch_v1: -__ubsan_handle_type_mismatch_v1_abort: - jmp ___ubsan_handle_type_mismatch_v1 - .endfn __ubsan_handle_type_mismatch_v1,globl - .endfn __ubsan_handle_type_mismatch_v1_abort,globl - -__ubsan_handle_add_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_add_overflow_abort,globl -__ubsan_handle_add_overflow: - loadstr "add_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_add_overflow,globl - -__ubsan_handle_alignment_assumption_abort: - nop -// fallthrough - .endfn __ubsan_handle_alignment_assumption_abort,globl -__ubsan_handle_alignment_assumption: - loadstr "alignment_assumption",si - jmp __ubsan_hop - .endfn __ubsan_handle_alignment_assumption,globl - -__ubsan_handle_builtin_unreachable_abort: - nop -// fallthrough - .endfn __ubsan_handle_builtin_unreachable_abort,globl -__ubsan_handle_builtin_unreachable: - loadstr "builtin_unreachable",si - jmp __ubsan_hop - .endfn __ubsan_handle_builtin_unreachable,globl - -__ubsan_handle_cfi_bad_type_abort: - nop -// fallthrough - .endfn __ubsan_handle_cfi_bad_type_abort,globl -__ubsan_handle_cfi_bad_type: - loadstr "cfi_bad_type",si - jmp __ubsan_hop - .endfn __ubsan_handle_cfi_bad_type,globl - -__ubsan_handle_cfi_check_fail_abort: - nop -// fallthrough - .endfn __ubsan_handle_cfi_check_fail_abort,globl -__ubsan_handle_cfi_check_fail: - loadstr "cfi_check_fail",si - jmp __ubsan_hop - .endfn __ubsan_handle_cfi_check_fail,globl - -__ubsan_handle_divrem_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_divrem_overflow_abort,globl -__ubsan_handle_divrem_overflow: - loadstr "divrem_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_divrem_overflow,globl - -__ubsan_handle_dynamic_type_cache_miss_abort: - nop -// fallthrough - .endfn __ubsan_handle_dynamic_type_cache_miss_abort,globl -__ubsan_handle_dynamic_type_cache_miss: - loadstr "dynamic_type_cache_miss",si - jmp __ubsan_hop - .endfn __ubsan_handle_dynamic_type_cache_miss,globl - -__ubsan_handle_function_type_mismatch_abort: - nop -// fallthrough - .endfn __ubsan_handle_function_type_mismatch_abort,globl -__ubsan_handle_function_type_mismatch: - loadstr "function_type_mismatch",si - jmp __ubsan_hop - .endfn __ubsan_handle_function_type_mismatch,globl - -__ubsan_handle_implicit_conversion_abort: - nop -// fallthrough - .endfn __ubsan_handle_implicit_conversion_abort,globl -__ubsan_handle_implicit_conversion: - loadstr "implicit_conversion",si - jmp __ubsan_hop - .endfn __ubsan_handle_implicit_conversion,globl - -__ubsan_handle_invalid_builtin_abort: - nop -// fallthrough - .endfn __ubsan_handle_invalid_builtin_abort,globl -__ubsan_handle_invalid_builtin: - loadstr "invalid_builtin",si - jmp __ubsan_hop - .endfn __ubsan_handle_invalid_builtin,globl - -__ubsan_handle_load_invalid_value_abort: - nop -// fallthrough - .endfn __ubsan_handle_load_invalid_value_abort,globl -__ubsan_handle_load_invalid_value: - loadstr "load_invalid_value (try checking for uninitialized variables)",si - jmp __ubsan_hop - .endfn __ubsan_handle_load_invalid_value,globl - -__ubsan_handle_missing_return_abort: - nop -// fallthrough - .endfn __ubsan_handle_missing_return_abort,globl -__ubsan_handle_missing_return: - loadstr "missing_return",si - jmp __ubsan_hop - .endfn __ubsan_handle_missing_return,globl - -__ubsan_handle_mul_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_mul_overflow_abort,globl -__ubsan_handle_mul_overflow: - loadstr "mul_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_mul_overflow,globl - -__ubsan_handle_negate_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_negate_overflow_abort,globl -__ubsan_handle_negate_overflow: - loadstr "negate_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_negate_overflow,globl - -__ubsan_handle_nonnull_arg_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_arg_abort,globl -__ubsan_handle_nonnull_arg: - loadstr "nonnull_arg",si - jmp __ubsan_hop - .endfn __ubsan_handle_nonnull_arg,globl - -__ubsan_handle_nonnull_return_v1_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return_v1_abort,globl -__ubsan_handle_nonnull_return_v1: - loadstr "nonnull_return_v1",si - jmp __ubsan_hop - .endfn __ubsan_handle_nonnull_return_v1,globl - -__ubsan_hop: - jmp __ubsan_abort - .endfn __ubsan_hop - -__ubsan_handle_nullability_arg_abort: - nop -// fallthrough - .endfn __ubsan_handle_nullability_arg_abort,globl -__ubsan_handle_nullability_arg: - loadstr "nullability_arg",si - jmp __ubsan_hop - .endfn __ubsan_handle_nullability_arg,globl - -__ubsan_handle_nullability_return_v1_abort: - nop -// fallthrough - .endfn __ubsan_handle_nullability_return_v1_abort,globl -__ubsan_handle_nullability_return_v1: - loadstr "nullability_return_v1",si - jmp __ubsan_hop - .endfn __ubsan_handle_nullability_return_v1,globl - -__ubsan_handle_pointer_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_pointer_overflow_abort,globl -__ubsan_handle_pointer_overflow: - loadstr "pointer_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_pointer_overflow,globl - -__ubsan_handle_shift_out_of_bounds_abort: - nop -// fallthrough - .endfn __ubsan_handle_shift_out_of_bounds_abort,globl -__ubsan_handle_shift_out_of_bounds: - loadstr "shift_out_of_bounds",si - jmp __ubsan_hop - .endfn __ubsan_handle_shift_out_of_bounds,globl - -__ubsan_handle_sub_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_sub_overflow_abort,globl -__ubsan_handle_sub_overflow: - loadstr "sub_overflow",si - jmp __ubsan_hop - .endfn __ubsan_handle_sub_overflow,globl - -__ubsan_handle_vla_bound_not_positive_abort: - nop -// fallthrough - .endfn __ubsan_handle_vla_bound_not_positive_abort,globl -__ubsan_handle_vla_bound_not_positive: - loadstr "vla_bound_not_positive",si - jmp __ubsan_hop - .endfn __ubsan_handle_vla_bound_not_positive,globl - -__ubsan_handle_nonnull_return_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return_abort,globl -__ubsan_handle_nonnull_return: - loadstr "nonnull_return",si - jmp __ubsan_hop - .endfn __ubsan_handle_nonnull_return,globl - -__ubsan_handle_out_of_bounds_abort: - jmp __ubsan_handle_out_of_bounds - .endfn __ubsan_handle_out_of_bounds_abort,globl - -.previous diff --git a/libc/log/ubsan.c b/libc/log/ubsan.c deleted file mode 100644 index b47e7ea90..000000000 --- a/libc/log/ubsan.c +++ /dev/null @@ -1,120 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/bits/pushpop.h" -#include "libc/calls/calls.h" -#include "libc/fmt/fmt.h" -#include "libc/log/internal.h" -#include "libc/log/log.h" -#include "libc/log/ubsan.internal.h" -#include "libc/runtime/internal.h" -#include "libc/runtime/runtime.h" -#include "libc/stdio/stdio.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/fileno.h" - -static char __ubsan_buf[256]; - -static const char kUbsanTypeCheckKinds[] = "\ -load of\0\ -store to\0\ -reference binding to\0\ -member access within\0\ -member call on\0\ -constructor call on\0\ -downcast of\0\ -downcast of\0\ -upcast of\0\ -cast to virtual base of\0\ -\0"; - -void __ubsan_abort(const struct UbsanSourceLocation *loc, - const char *description) { - static bool once; - if (!once) { - once = true; - } else { - abort(); - } - if (IsDebuggerPresent(false)) DebugBreak(); - __start_fatal(loc->file, loc->line); - fprintf(stderr, "%s\r\n", description); - __die(); - unreachable; -} - -void __ubsan_handle_out_of_bounds(struct UbsanOutOfBoundsInfo *info, - uintptr_t index) { - snprintf(__ubsan_buf, sizeof(__ubsan_buf), - "%s index %,lu into %s out of bounds", info->index_type->name, index, - info->array_type->name); - __ubsan_abort(&info->location, __ubsan_buf); - unreachable; -} - -void __ubsan_handle_type_mismatch(struct UbsanTypeMismatchInfo *type_mismatch, - uintptr_t pointer) { - struct UbsanSourceLocation *loc = &type_mismatch->location; - const char *description; - const char *kind = IndexDoubleNulString(kUbsanTypeCheckKinds, - type_mismatch->type_check_kind); - if (pointer == 0) { - description = "null pointer access"; - } else if (type_mismatch->alignment != 0 && - (pointer & (type_mismatch->alignment - 1))) { - description = __ubsan_buf; - snprintf(__ubsan_buf, sizeof(__ubsan_buf), "%s %s %s @%p %s %d", - "unaligned", kind, type_mismatch->type->name, pointer, "align", - type_mismatch->alignment); - } else { - description = __ubsan_buf; - snprintf(__ubsan_buf, sizeof(__ubsan_buf), "%s\r\n\t%s %s %p %s %s", - "insufficient size", kind, "address", pointer, - "with insufficient space for object of type", - type_mismatch->type->name); - } - __ubsan_abort(loc, description); - unreachable; -} - -void ___ubsan_handle_type_mismatch_v1( - struct UbsanTypeMismatchInfoClang *type_mismatch, uintptr_t pointer) { - struct UbsanTypeMismatchInfo mm; - mm.location = type_mismatch->location; - mm.type = type_mismatch->type; - mm.alignment = 1u << type_mismatch->log_alignment; - mm.type_check_kind = type_mismatch->type_check_kind; - __ubsan_handle_type_mismatch(&mm, pointer); - unreachable; -} - -void __ubsan_handle_float_cast_overflow(void *data_raw, void *from_raw) { - struct UbsanFloatCastOverflowData *data = - (struct UbsanFloatCastOverflowData *)data_raw; -#if __GNUC__ + 0 >= 6 - __ubsan_abort(&data->location, "float cast overflow"); -#else - const struct UbsanSourceLocation kUnknownLocation = { - "", - pushpop(0), - pushpop(0), - }; - __ubsan_abort(((void)data, &kUnknownLocation), "float cast overflow"); -#endif - unreachable; -} diff --git a/libc/log/ubsan.internal.h b/libc/log/ubsan.internal.h deleted file mode 100644 index f3cc34990..000000000 --- a/libc/log/ubsan.internal.h +++ /dev/null @@ -1,110 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_UBSAN_H_ -#define COSMOPOLITAN_LIBC_UBSAN_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) - -/*───────────────────────────────────────────────────────────────────────────│─╗ -│ cosmopolitan § runtime » behavior enforcement ─╬─│┼ -╚────────────────────────────────────────────────────────────────────────────│*/ - -struct UbsanSourceLocation { - const char *file; - uint32_t line; - uint32_t column; -}; - -struct UbsanTypeDescriptor { - uint16_t kind; - uint16_t info; - char name[]; -}; - -struct UbsanTypeMismatchInfo { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; - uintptr_t alignment; - uint8_t type_check_kind; -}; - -struct UbsanTypeMismatchInfoClang { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; - unsigned char log_alignment; /* https://reviews.llvm.org/D28244 */ - uint8_t type_check_kind; -}; - -struct UbsanOutOfBoundsInfo { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *array_type; - struct UbsanTypeDescriptor *index_type; -}; - -struct UbsanUnreachableData { - struct UbsanSourceLocation location; -}; - -struct UbsanVlaBoundData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; -}; - -struct UbsanNonnullArgData { - struct UbsanSourceLocation location; - struct UbsanSourceLocation attr_location; -}; - -struct UbsanCfiBadIcallData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; -}; - -struct UbsanNonnullReturnData { - struct UbsanSourceLocation location; - struct UbsanSourceLocation attr_location; -}; - -struct UbsanFunctionTypeMismatchData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; -}; - -struct UbsanInvalidValueData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; -}; - -struct UbsanOverflowData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *type; -}; - -struct UbsanFloatCastOverflowData { -#if __GNUC__ + 0 >= 6 - struct UbsanSourceLocation location; -#endif - struct UbsanTypeDescriptor *from_type; - struct UbsanTypeDescriptor *to_type; -}; - -struct UbsanOutOfBoundsData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *arraytype; - struct UbsanTypeDescriptor *index_type; -}; - -struct UbsanShiftOutOfBoundsData { - struct UbsanSourceLocation location; - struct UbsanTypeDescriptor *lhs_type; - struct UbsanTypeDescriptor *rhs_type; -}; - -void __ubsan_abort(const struct UbsanSourceLocation *, - const char *) relegated hidden wontreturn; -void __ubsan_handle_type_mismatch(struct UbsanTypeMismatchInfo *, - uintptr_t) relegated hidden wontreturn; -void ___ubsan_handle_type_mismatch_v1(struct UbsanTypeMismatchInfoClang *, - uintptr_t) relegated hidden wontreturn; -void __ubsan_handle_float_cast_overflow(void *, - void *) relegated hidden wontreturn; - -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_UBSAN_H_ */ diff --git a/libc/macros.internal.h b/libc/macros.internal.h index 5c1939f08..5bbff6e92 100644 --- a/libc/macros.internal.h +++ b/libc/macros.internal.h @@ -26,8 +26,6 @@ #define TYPE_BIT(type) (sizeof(type) * CHAR_BIT) #define TYPE_SIGNED(type) (((type)-1) < 0) #define TYPE_INTEGRAL(type) (((type)0.5) != 0.5) -#define INT_STRLEN_MAXIMUM(type) \ - ((TYPE_BIT(type) - TYPE_SIGNED(type)) * 302 / 1000 + 1 + TYPE_SIGNED(type)) #define ARRAYLEN(A) \ ((sizeof(A) / sizeof(*(A))) / ((unsigned)!(sizeof(A) % sizeof(*(A))))) diff --git a/libc/math.h b/libc/math.h index 1c7a0cc7d..552a0038b 100644 --- a/libc/math.h +++ b/libc/math.h @@ -287,6 +287,9 @@ void sincos(double, double *, double *); void sincosf(float, float *, float *); void sincosl(long double, long double *, long double *); +float fsumf(const float *, size_t); +double fsum(const double *, size_t); + COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ #endif /* COSMOPOLITAN_LIBC_MATH_H_ */ diff --git a/libc/nexgen32e/bsf.h b/libc/nexgen32e/bsf.h index beaf27322..3f6a0cd31 100644 --- a/libc/nexgen32e/bsf.h +++ b/libc/nexgen32e/bsf.h @@ -22,10 +22,20 @@ int bsfl(long); int bsfll(long long); int bsfmax(uintmax_t); -#ifdef __GNUC__ -#define bsf(u) __builtin_ctz(u) -#define bsfl(u) __builtin_ctzl(u) -#define bsfll(u) __builtin_ctzll(u) +#if defined(__GNUC__) && !defined(__STRICT_ANSI__) +#define bsf(u) \ + ({ \ + unsigned BiTs; \ + asm("bsf\t%0,%0" : "=r"(BiTs) : "0"((unsigned)(u)) : "cc"); \ + BiTs; \ + }) +#define bsfl(u) \ + ({ \ + unsigned long BiTs; \ + asm("bsf\t%0,%0" : "=r"(BiTs) : "0"((unsigned long)(u)) : "cc"); \ + (unsigned)BiTs; \ + }) +#define bsfll(u) bsfl(u) #endif COSMOPOLITAN_C_END_ diff --git a/libc/nexgen32e/bsr.h b/libc/nexgen32e/bsr.h index 74492e03c..d1e65a10d 100644 --- a/libc/nexgen32e/bsr.h +++ b/libc/nexgen32e/bsr.h @@ -22,10 +22,20 @@ int bsrl(long); int bsrll(long long); int bsrmax(uintmax_t); -#ifdef __GNUC__ -#define bsr(u) ((sizeof(int) * 8 - 1) ^ __builtin_clz(u)) -#define bsrl(u) ((sizeof(long) * 8 - 1) ^ __builtin_clzl(u)) -#define bsrll(u) ((sizeof(long long) * 8 - 1) ^ __builtin_clzll(u)) +#if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__) +#define bsr(u) \ + ({ \ + unsigned BiTs; \ + asm("bsr\t%0,%0" : "=r"(BiTs) : "0"((unsigned)(u)) : "cc"); \ + BiTs; \ + }) +#define bsrl(u) \ + ({ \ + unsigned long BiTs; \ + asm("bsr\t%0,%0" : "=r"(BiTs) : "0"((unsigned long)(u)) : "cc"); \ + (unsigned)BiTs; \ + }) +#define bsrll(u) bsrl(u) #endif COSMOPOLITAN_C_END_ diff --git a/libc/nexgen32e/lzcnt.h b/libc/nexgen32e/lzcnt.h deleted file mode 100644 index b30f1e8dc..000000000 --- a/libc/nexgen32e/lzcnt.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_NEXGEN32E_LZCNT_H_ -#define COSMOPOLITAN_LIBC_NEXGEN32E_LZCNT_H_ -#include "libc/nexgen32e/x86feature.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -unsigned lzcnt(unsigned) libcesque pureconst; -unsigned long lzcntl(unsigned long) libcesque pureconst; - -#define lzcnt(X) \ - ({ \ - typeof(X) Res; \ - if (X86_HAVE(ABM)) { \ - /* amd piledriver+ (a.k.a. bdver1) c. 2011 */ \ - /* intel haswell+ c. 2013 */ \ - asm("lzcnt\t%1,%0" : "=r,r"(Res) : "r,m"(X) : "cc"); \ - } else { \ - Res = (lzcnt)(X); \ - } \ - Res; \ - }) - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_NEXGEN32E_LZCNT_H_ */ diff --git a/libc/nexgen32e/tzcnt.h b/libc/nexgen32e/tzcnt.h deleted file mode 100644 index 9b243ac87..000000000 --- a/libc/nexgen32e/tzcnt.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_NEXGEN32E_TZCNT_H_ -#define COSMOPOLITAN_LIBC_NEXGEN32E_TZCNT_H_ -#include "libc/nexgen32e/x86feature.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -unsigned tzcnt(unsigned) libcesque pureconst; -unsigned long tzcntl(unsigned long) libcesque pureconst; - -#define tzcnt(X) \ - ({ \ - typeof(X) Res; \ - if (X86_HAVE(BMI)) { \ - /* amd piledriver+ (a.k.a. bdver1) c. 2011 */ \ - /* intel haswell+ c. 2013 */ \ - asm("tzcnt\t%1,%0" : "=r,r"(Res) : "r,m"(X) : "cc"); \ - } else { \ - Res = (tzcnt)(X); \ - } \ - Res; \ - }) - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_NEXGEN32E_TZCNT_H_ */ diff --git a/libc/nt/enum/lockfile.h b/libc/nt/enum/lockfile.h new file mode 100644 index 000000000..dae4f2149 --- /dev/null +++ b/libc/nt/enum/lockfile.h @@ -0,0 +1,7 @@ +#ifndef COSMOPOLITAN_LIBC_NT_ENUM_LOCKFILE_H_ +#define COSMOPOLITAN_LIBC_NT_ENUM_LOCKFILE_H_ + +#define kNtLockfileFailImmediately 1 +#define kNtLockfileExclusiveLock 2 + +#endif /* COSMOPOLITAN_LIBC_NT_ENUM_LOCKFILE_H_ */ diff --git a/libc/runtime/ftrace.c b/libc/runtime/ftrace.c index 0fa7538fe..3c5d5c68f 100644 --- a/libc/runtime/ftrace.c +++ b/libc/runtime/ftrace.c @@ -23,9 +23,13 @@ #include "libc/calls/internal.h" #include "libc/calls/struct/sigset.h" #include "libc/dce.h" +#include "libc/fmt/itoa.h" #include "libc/intrin/repmovsb.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/rdtsc.h" +#include "libc/nexgen32e/rdtscp.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/x86feature.h" #include "libc/nt/files.h" #include "libc/nt/runtime.h" #include "libc/nt/thunk/msabi.h" @@ -49,6 +53,7 @@ void ftrace_hook(void); static int noreentry; +static uint64_t laststamp; static char g_buf[512]; static const char *g_lastsymbol; static struct SymbolTable *g_symbols; @@ -70,11 +75,14 @@ static noasan int GetNestingLevel(struct StackFrame *frame) { * according to the System Five NexGen32e ABI. */ privileged noasan void ftrace(void) { - size_t i, j, nesting; + char *p; + uint64_t stamp; const char *symbol; struct StackFrame *frame; + size_t nesting, symbolsize; if (!cmpxchg(&noreentry, 0, 1)) return; if (g_symbols) { + stamp = rdtsc(); frame = __builtin_frame_address(0); frame = frame->next; symbol = @@ -84,31 +92,30 @@ privileged noasan void ftrace(void) { g_symbols->count, frame->addr - g_symbols->addr_base)] .name_rva]; - if (symbol != g_lastsymbol && - (nesting = GetNestingLevel(frame)) * 2 < ARRAYLEN(g_buf) - 4) { - i = 2; - j = 0; - while (nesting--) { - asm volatile("" : : : "memory"); - g_buf[i++] = ' '; - g_buf[i++] = ' '; + if (symbol != g_lastsymbol) { + symbolsize = strlen(symbol); + nesting = GetNestingLevel(frame); + if (2 + nesting * 2 + symbolsize + 1 + 21 + 2 <= ARRAYLEN(g_buf)) { + p = g_buf; + *p++ = '+'; + *p++ = ' '; + memset(p, ' ', nesting * 2); + p += nesting * 2; + p = mempcpy(p, symbol, symbolsize); + *p++ = ' '; + p += uint64toarray_radix10((stamp - laststamp) / 3.3, p); + *p++ = '\r'; + *p++ = '\n'; + write(2, g_buf, p - g_buf); } - while (i < ARRAYLEN(g_buf) - 2 && symbol[j]) { - asm volatile("" : : : "memory"); - g_buf[i++] = symbol[j++]; - } - g_buf[i++] = '\r'; - g_buf[i++] = '\n'; - write(2, g_buf, i); } g_lastsymbol = symbol; + laststamp = X86_HAVE(RDTSCP) ? rdtscp(0) : rdtsc(); } noreentry = 0; } textstartup void ftrace_install(void) { - g_buf[0] = '+'; - g_buf[1] = ' '; if ((g_symbols = OpenSymbolTable(FindDebugBinary()))) { __hook(ftrace_hook, g_symbols); } else { diff --git a/libc/runtime/getdosargv.c b/libc/runtime/getdosargv.c index 699f1d1af..d21113715 100644 --- a/libc/runtime/getdosargv.c +++ b/libc/runtime/getdosargv.c @@ -89,14 +89,16 @@ textwindows noasan int GetDosArgv(const char16_t *cmdline, char *buf, argc = 0; st.wc = DecodeDosArgv(&st.s); while (st.wc) { - while (st.wc && isspace(st.wc)) st.wc = DecodeDosArgv(&st.s); + while (st.wc && (st.wc == ' ' || st.wc == '\t')) { + st.wc = DecodeDosArgv(&st.s); + } if (!st.wc) break; if (++argc < max) { argv[argc - 1] = st.p < st.pe ? st.p : NULL; } inquote = false; while (st.wc) { - if (!inquote && isspace(st.wc)) break; + if (!inquote && (st.wc == ' ' || st.wc == '\t')) break; if (st.wc == '"' || st.wc == '\\') { slashes = 0; quotes = 0; diff --git a/libc/runtime/msync.c b/libc/runtime/msync.c index fde0b2b9f..adcd9a0e9 100644 --- a/libc/runtime/msync.c +++ b/libc/runtime/msync.c @@ -29,6 +29,7 @@ * Without this, there's no guarantee memory is written back to disk. In * practice, what that means is just Windows NT. * + * @param addr needs to be 4096-byte page aligned * @param flags needs MS_ASYNC or MS_SYNC and can have MS_INVALIDATE * @return 0 on success or -1 w/ errno */ diff --git a/libc/sock/accept4.c b/libc/sock/accept4.c index e92d3331e..1d24bb048 100644 --- a/libc/sock/accept4.c +++ b/libc/sock/accept4.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/sysv/errfuns.h" @@ -36,6 +37,7 @@ int accept4(int fd, void *out_addr, uint32_t *inout_addrsize, int flags) { if (!out_addr) return efault(); if (!inout_addrsize) return efault(); + if (IsAsan() && !__asan_is_valid(out_addr, *inout_addrsize)) return efault(); if (!IsWindows()) { return sys_accept4(fd, out_addr, inout_addrsize, flags); } else if (__isfdkind(fd, kFdSocket)) { diff --git a/libc/sock/bind.c b/libc/sock/bind.c index d9b28a4a3..28cb9c689 100644 --- a/libc/sock/bind.c +++ b/libc/sock/bind.c @@ -19,6 +19,7 @@ #include "libc/assert.h" #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/str/str.h" @@ -36,12 +37,14 @@ */ int bind(int fd, const void *addr, uint32_t addrsize) { if (!addr) return efault(); + if (IsAsan() && !__asan_is_valid(addr, addrsize)) return efault(); if (addrsize == sizeof(struct sockaddr_in)) { if (!IsWindows()) { if (!IsBsd()) { return sys_bind(fd, addr, addrsize); } else { - char addr2[sizeof(struct sockaddr_un_bsd)]; /* sockaddr_un_bsd is the largest */ + char addr2[sizeof( + struct sockaddr_un_bsd)]; /* sockaddr_un_bsd is the largest */ assert(addrsize <= sizeof(addr2)); memcpy(&addr2, addr, addrsize); sockaddr2bsd(&addr2[0]); diff --git a/libc/sock/connect.c b/libc/sock/connect.c index 9cbb2515e..10d7beaec 100644 --- a/libc/sock/connect.c +++ b/libc/sock/connect.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/sysv/errfuns.h" @@ -34,6 +35,7 @@ */ int connect(int fd, const void *addr, uint32_t addrsize) { if (!addr) return efault(); + if (IsAsan() && !__asan_is_valid(addr, addrsize)) return efault(); if (!IsWindows()) { return sys_connect(fd, addr, addrsize); } else if (__isfdkind(fd, kFdSocket)) { diff --git a/libc/sock/getpeername.c b/libc/sock/getpeername.c index 0da57eb99..444a3e60c 100644 --- a/libc/sock/getpeername.c +++ b/libc/sock/getpeername.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/sysv/errfuns.h" @@ -28,6 +29,7 @@ * @see getsockname() */ int getpeername(int fd, void *out_addr, uint32_t *out_addrsize) { + if (IsAsan() && !__asan_is_valid(out_addr, *out_addrsize)) return efault(); if (!IsWindows()) { return sys_getpeername(fd, out_addr, out_addrsize); } else if (__isfdkind(fd, kFdSocket)) { diff --git a/libc/sock/getsockname.c b/libc/sock/getsockname.c index 70409656b..49c7e9dbe 100644 --- a/libc/sock/getsockname.c +++ b/libc/sock/getsockname.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/sysv/errfuns.h" @@ -28,6 +29,7 @@ * @see getpeername() */ int getsockname(int fd, void *out_addr, uint32_t *out_addrsize) { + if (IsAsan() && !__asan_is_valid(out_addr, *out_addrsize)) return efault(); if (!IsWindows()) { return sys_getsockname(fd, out_addr, out_addrsize); } else if (__isfdkind(fd, kFdSocket)) { diff --git a/libc/sock/poll.c b/libc/sock/poll.c index ab8c5a07f..b19201144 100644 --- a/libc/sock/poll.c +++ b/libc/sock/poll.c @@ -19,8 +19,10 @@ #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" +#include "libc/sysv/errfuns.h" /** * Waits for something to happen on multiple file descriptors at once. @@ -34,9 +36,11 @@ * @return fds[𝑖].revents flags can have: * (fds[𝑖].events & POLL{IN,OUT,PRI,HUP,ERR,NVAL}) * @asyncsignalsafe - * @see ppoll() */ int poll(struct pollfd *fds, uint64_t nfds, int32_t timeout_ms) { + if (IsAsan() && !__asan_is_valid(fds, nfds * sizeof(struct pollfd))) { + return efault(); + } if (!IsWindows()) { return sys_poll(fds, nfds, timeout_ms); } else { diff --git a/libc/sock/recvfrom.c b/libc/sock/recvfrom.c index f05d9a904..65f2ddc87 100644 --- a/libc/sock/recvfrom.c +++ b/libc/sock/recvfrom.c @@ -19,6 +19,7 @@ #include "libc/calls/internal.h" #include "libc/calls/struct/iovec.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/nt/winsock.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" @@ -43,6 +44,12 @@ ssize_t recvfrom(int fd, void *buf, size_t size, uint32_t flags, void *opt_out_srcaddr, uint32_t *opt_inout_srcaddrsize) { ssize_t got; + if (IsAsan() && + (!__asan_is_valid(buf, size) || + (opt_out_srcaddr && + !__asan_is_valid(opt_out_srcaddr, *opt_inout_srcaddrsize)))) { + return efault(); + } if (!IsWindows()) { got = sys_recvfrom(fd, buf, size, flags, opt_out_srcaddr, opt_inout_srcaddrsize); diff --git a/libc/sock/sendto.c b/libc/sock/sendto.c index cf55bc415..c09333aa7 100644 --- a/libc/sock/sendto.c +++ b/libc/sock/sendto.c @@ -20,6 +20,7 @@ #include "libc/calls/internal.h" #include "libc/calls/struct/iovec.h" #include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/sock/internal.h" #include "libc/sock/sock.h" #include "libc/str/str.h" @@ -47,6 +48,10 @@ */ ssize_t sendto(int fd, const void *buf, size_t size, uint32_t flags, const void *opt_addr, uint32_t addrsize) { + if (IsAsan() && (!__asan_is_valid(buf, size) || + (opt_addr && !__asan_is_valid(opt_addr, addrsize)))) { + return efault(); + } if (!IsWindows()) { if (!IsBsd() || !opt_addr) { return sys_sendto(fd, buf, size, flags, opt_addr, addrsize); diff --git a/libc/sock/sock.h b/libc/sock/sock.h index fbc873fc1..ffb41c941 100644 --- a/libc/sock/sock.h +++ b/libc/sock/sock.h @@ -9,7 +9,13 @@ COSMOPOLITAN_C_START_ #define INET_ADDRSTRLEN 22 -#define NI_DGRAM 0x10 +#define NI_NUMERICHOST 0x01 +#define NI_NUMERICSERV 0x02 +#define NI_NOFQDN 0x04 +#define NI_NAMEREQD 0x08 +#define NI_DGRAM 0x10 + +#define NI_MAXHOST 0xff #define NI_MAXSERV 0x20 #define htons(u16) bswap_16(u16) diff --git a/libc/stdio/spawnf.c b/libc/stdio/spawnf.c index 3e893b150..acced624d 100644 --- a/libc/stdio/spawnf.c +++ b/libc/stdio/spawnf.c @@ -59,7 +59,7 @@ static int add_to_file_actions(posix_spawn_file_actions_t *file_actions, int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *file_actions, int fildes) { char temp[100]; - sprintf(temp, "close(%d)", fildes); + (sprintf)(temp, "close(%d)", fildes); return add_to_file_actions(file_actions, temp); } @@ -69,7 +69,7 @@ int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *file_actions, int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *file_actions, int fildes, int newfildes) { char temp[100]; - sprintf(temp, "dup2(%d,%d)", fildes, newfildes); + (sprintf)(temp, "dup2(%d,%d)", fildes, newfildes); return add_to_file_actions(file_actions, temp); } @@ -80,6 +80,6 @@ int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *file_actions, int fildes, const char *path, int oflag, unsigned mode) { char temp[100]; - sprintf(temp, "open(%d,%s*%o,%o)", fildes, path, oflag, mode); + (sprintf)(temp, "open(%d,%s*%o,%o)", fildes, path, oflag, mode); return add_to_file_actions(file_actions, temp); } diff --git a/libc/stdio/tmpfile.c b/libc/stdio/tmpfile.c index b2197e463..1bee40c56 100644 --- a/libc/stdio/tmpfile.c +++ b/libc/stdio/tmpfile.c @@ -35,7 +35,7 @@ FILE *tmpfile(void) { char *tmp, *sep, tpl[PATH_MAX]; tmp = firstnonnull(getenv("TMPDIR"), kTmpPath); sep = !isempty(tmp) && !endswith(tmp, "/") ? "/" : ""; - if (snprintf(tpl, PATH_MAX, "%s%stmp.XXXXXX", tmp, sep) < PATH_MAX) { + if ((snprintf)(tpl, PATH_MAX, "%s%stmp.XXXXXX", tmp, sep) < PATH_MAX) { if ((fd = mkostemps(tpl, 0, 0)) != -1) { return fdopen(fd, "w+"); } diff --git a/libc/str/getzipcdir.c b/libc/str/getzipcdir.c index 18d1d956e..1ba8d7815 100644 --- a/libc/str/getzipcdir.c +++ b/libc/str/getzipcdir.c @@ -21,9 +21,9 @@ /** * Locates End Of Central Directory record in ZIP file. * - * The ZIP spec says this header can be anywhere in the last 64kb. - * We search it backwards for the ZIP-64 "PK♠♠" magic number. If that's - * not found, then we search again for the original "PK♣♠" magnum. The + * The ZIP spec says this header can be anywhere in the last 64kb. We + * search it backwards for the ZIP-64 "PK♠•" magic number. If that's not + * found, then we search again for the original "PK♣♠" magnum. The * caller needs to check the first four bytes of the returned value to * determine whether to use ZIP_CDIR_xxx() or ZIP_CDIR64_xxx() macros. * @@ -31,23 +31,25 @@ * @param n is byte size of file * @return pointer to EOCD64 or EOCD, or NULL if not found */ -uint8_t *GetZipCdir(const uint8_t *p, size_t n) { +void *GetZipCdir(const uint8_t *p, size_t n) { size_t i, j; - if (n >= kZipCdirHdrMinSize) { - i = n - kZipCdirHdrMinSize; - do { - if (READ32LE(p + i) == kZipCdir64HdrMagic && IsZipCdir64(p, n, i)) { - return (/*unconst*/ uint8_t *)(p + i); - } else if (READ32LE(p + i) == kZipCdirHdrMagic && IsZipCdir32(p, n, i)) { - j = i; - do { - if (READ32LE(p + j) == kZipCdir64HdrMagic && IsZipCdir64(p, n, j)) { - return (/*unconst*/ uint8_t *)(p + j); - } - } while (j-- && i - j < 64 * 1024); - return (/*unconst*/ uint8_t *)(p + i); - } - } while (i--); - } + i = n - 4; + do { + if (READ32LE(p + i) == kZipCdir64LocatorMagic && + i + kZipCdir64LocatorSize <= n && + IsZipCdir64(p, n, ZIP_LOCATE64_OFFSET(p + i))) { + return (void *)(p + ZIP_LOCATE64_OFFSET(p + i)); + } else if (READ32LE(p + i) == kZipCdirHdrMagic && IsZipCdir32(p, n, i)) { + j = i; + do { + if (READ32LE(p + j) == kZipCdir64LocatorMagic && + j + kZipCdir64LocatorSize <= n && + IsZipCdir64(p, n, ZIP_LOCATE64_OFFSET(p + j))) { + return (void *)(p + ZIP_LOCATE64_OFFSET(p + j)); + } + } while (j-- && i - j < 64 * 1024); + return (void *)(p + i); + } + } while (i--); return NULL; } diff --git a/libc/str/getzipcdircomment.c b/libc/str/getzipcdircomment.c index cefd8ab82..b738b4e05 100644 --- a/libc/str/getzipcdircomment.c +++ b/libc/str/getzipcdircomment.c @@ -22,7 +22,7 @@ * Returns comment of zip central directory. */ void *GetZipCdirComment(const uint8_t *eocd) { - if (READ32LE(eocd) == kZipCdir64HdrMagic) { + if (READ32LE(eocd) == kZipCdir64HdrMagic && ZIP_CDIR64_COMMENTSIZE(eocd)) { return ZIP_CDIR64_COMMENT(eocd); } else { return ZIP_CDIR_COMMENT(eocd); diff --git a/libc/str/getzipcdircommentsize.c b/libc/str/getzipcdircommentsize.c index 6741fadac..78165b245 100644 --- a/libc/str/getzipcdircommentsize.c +++ b/libc/str/getzipcdircommentsize.c @@ -22,7 +22,7 @@ * Returns comment of zip central directory. */ uint64_t GetZipCdirCommentSize(const uint8_t *eocd) { - if (READ32LE(eocd) == kZipCdir64HdrMagic) { + if (READ32LE(eocd) == kZipCdir64HdrMagic && ZIP_CDIR64_COMMENTSIZE(eocd)) { return ZIP_CDIR64_COMMENTSIZE(eocd); } else { return ZIP_CDIR_COMMENTSIZE(eocd); diff --git a/libc/str/getzipcdirsize.c b/libc/str/getzipcdirsize.c new file mode 100644 index 000000000..67985c214 --- /dev/null +++ b/libc/str/getzipcdirsize.c @@ -0,0 +1,30 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/zip.h" + +/** + * Returns size of zip central directory. + */ +uint64_t GetZipCdirSize(const uint8_t *eocd) { + if (READ32LE(eocd) == kZipCdir64HdrMagic) { + return ZIP_CDIR64_SIZE(eocd); + } else { + return ZIP_CDIR_SIZE(eocd); + } +} diff --git a/libc/str/isspace.c b/libc/str/isspace.c index d9bcfb832..2c18305c7 100644 --- a/libc/str/isspace.c +++ b/libc/str/isspace.c @@ -19,7 +19,8 @@ #include "libc/str/str.h" /** - * Returns true if c is space, \t, \r, \n, \f, or \v. + * Returns nonzero if c is space, \t, \r, \n, \f, or \v. + * @see isblank() */ int isspace(int c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || diff --git a/libc/str/iswalnum.c b/libc/str/iswalnum.c index e23c2acc8..58e54da55 100644 --- a/libc/str/iswalnum.c +++ b/libc/str/iswalnum.c @@ -18,6 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswalnum(wint_t wc) { - return isalnum(wc); +/** + * Returns nonzero if c is lower, alpha, or digit. + */ +int iswalnum(wint_t c) { + return iswdigit(c) || iswalpha(c); } diff --git a/libc/str/iswalpha.c b/libc/str/iswalpha.c index 5a209d341..fa246056c 100644 --- a/libc/str/iswalpha.c +++ b/libc/str/iswalpha.c @@ -18,6 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswalpha(wint_t wc) { - return isalpha(wc); +/** + * Returns nonzero if c is alphabetical. + */ +int iswalpha(wint_t c) { + return iswupper(c) || iswlower(c); } diff --git a/libc/str/iswblank.c b/libc/str/iswblank.c index f9340404e..d3af4a1ba 100644 --- a/libc/str/iswblank.c +++ b/libc/str/iswblank.c @@ -18,6 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswblank(wint_t wc) { - return isblank(wc); +/** + * Returns nonzero if c is space or tab. + */ +int iswblank(wint_t c) { + return c == ' ' || c == '\t'; } diff --git a/libc/str/iswcntrl.c b/libc/str/iswcntrl.c index a8167c2fe..5ce225bcb 100644 --- a/libc/str/iswcntrl.c +++ b/libc/str/iswcntrl.c @@ -19,8 +19,8 @@ #include "libc/str/str.h" /** - * Returns nonzero if wc is C0 or C1 control code. + * Returns nonzero if c is C0 or C1 control code. */ -int iswcntrl(wint_t wc) { - return (0x00 <= wc && wc <= 0x1F) || (0x7F <= wc && wc <= 0x9F); +int iswcntrl(wint_t c) { + return (0x00 <= c && c <= 0x1F) || (0x7F <= c && c <= 0x9F); } diff --git a/libc/str/iswctype.c b/libc/str/iswctype.c index 531f40228..a5e0fa401 100644 --- a/libc/str/iswctype.c +++ b/libc/str/iswctype.c @@ -19,71 +19,32 @@ #include "libc/macros.internal.h" #include "libc/str/str.h" -#define ALNUM 1 -#define ALPHA 2 -#define BLANK 3 -#define CNTRL 4 -#define DIGIT 5 -#define GRAPH 6 -#define LOWER 7 -#define PRINT 8 -#define PUNCT 9 -#define SPACE 10 -#define UPPER 11 -#define XDIGIT 12 +typedef int (*isw_f)(wint_t); -static const struct { - char name[7]; - char type; -} kWcTypes[] = { - {"alnum", ALNUM}, {"alpha", ALPHA}, {"blank", BLANK}, {"cntrl", CNTRL}, - {"digit", DIGIT}, {"graph", GRAPH}, {"lower", LOWER}, {"print", PRINT}, - {"punct", PUNCT}, {"space", SPACE}, {"upper", UPPER}, {"xdigit", XDIGIT}, +static const isw_f kWcTypeFuncs[] = { + iswalnum, // + iswalpha, // + iswblank, // + iswcntrl, // + iswdigit, // + iswgraph, // + iswlower, // + iswprint, // + iswpunct, // + iswspace, // + iswupper, // + iswxdigit, // }; -static int CompareStrings(const char *l, const char *r) { - size_t i = 0; - while (l[i] == r[i] && r[i]) ++i; - return (l[i] & 0xff) - (r[i] & 0xff); -} - -wctype_t wctype(const char *name) { - unsigned i; - for (i = 0; i < ARRAYLEN(kWcTypes); ++i) { - if (CompareStrings(name, kWcTypes[i].name) == 0) { - return kWcTypes[i].type; - } - } - return 0; -} - -int iswctype(wint_t wc, wctype_t type) { - switch (type) { - case ALNUM: - return iswalnum(wc); - case ALPHA: - return iswalpha(wc); - case BLANK: - return iswblank(wc); - case CNTRL: - return iswcntrl(wc); - case DIGIT: - return iswdigit(wc); - case GRAPH: - return iswgraph(wc); - case LOWER: - return iswlower(wc); - case PRINT: - return iswprint(wc); - case PUNCT: - return iswpunct(wc); - case SPACE: - return iswspace(wc); - case UPPER: - return iswupper(wc); - case XDIGIT: - return iswxdigit(wc); - default: - return 0; +/** + * Returns nonzero if c has property. + * + * @param t is number returned by wctype + */ +int iswctype(wint_t c, wctype_t t) { + if (1 <= t && t <= ARRAYLEN(kWcTypeFuncs)) { + return kWcTypeFuncs[t - 1](c); + } else { + return 0; } } diff --git a/libc/str/iswdigit.c b/libc/str/iswdigit.c index 9f65a2cfc..9a008c422 100644 --- a/libc/str/iswdigit.c +++ b/libc/str/iswdigit.c @@ -18,6 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswdigit(wint_t wc) { - return isdigit(wc); +/** + * Returns nonzero if c is decimal digit. + */ +int iswdigit(wint_t c) { + return '0' <= c && c <= '9'; } diff --git a/libc/str/iswgraph.c b/libc/str/iswgraph.c index 51fd29bf2..b4c740b02 100644 --- a/libc/str/iswgraph.c +++ b/libc/str/iswgraph.c @@ -18,6 +18,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswgraph(wint_t wc) { - return isgraph(wc); +/** + * Returns nonzero if c is printable and not a space. + */ +int iswgraph(wint_t c) { + return iswprint(c) && !iswspace(c); } diff --git a/libc/str/iswlower.c b/libc/str/iswlower.c index 56e5f1ea5..4e7769aa9 100644 --- a/libc/str/iswlower.c +++ b/libc/str/iswlower.c @@ -18,6 +18,2139 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswlower(wint_t wc) { - return islower(wc); +/** + * Returns nonzero if c is lowercase letter. + */ +int iswlower(wint_t c) { + if (isascii(c)) return islower(c); + switch (c) { + case u'µ': // MICRO SIGN (0x00b5) + case u'ß': // LATIN SMALL SHARP S (0x00df) + case u'à': // LATIN SMALL A W/ GRAVE (0x00e0) + case u'á': // LATIN SMALL A W/ ACUTE (0x00e1) + case u'â': // LATIN SMALL A W/ CIRCUMFLEX (0x00e2) + case u'ã': // LATIN SMALL A W/ TILDE (0x00e3) + case u'ä': // LATIN SMALL A W/ DIAERESIS (0x00e4) + case u'å': // LATIN SMALL A W/ RING ABOVE (0x00e5) + case u'æ': // LATIN SMALL AE (0x00e6) + case u'ç': // LATIN SMALL C W/ CEDILLA (0x00e7) + case u'è': // LATIN SMALL E W/ GRAVE (0x00e8) + case u'é': // LATIN SMALL E W/ ACUTE (0x00e9) + case u'ê': // LATIN SMALL E W/ CIRCUMFLEX (0x00ea) + case u'ë': // LATIN SMALL E W/ DIAERESIS (0x00eb) + case u'ì': // LATIN SMALL I W/ GRAVE (0x00ec) + case u'í': // LATIN SMALL I W/ ACUTE (0x00ed) + case u'î': // LATIN SMALL I W/ CIRCUMFLEX (0x00ee) + case u'ï': // LATIN SMALL I W/ DIAERESIS (0x00ef) + case u'ð': // LATIN SMALL ETH (0x00f0) + case u'ñ': // LATIN SMALL N W/ TILDE (0x00f1) + case u'ò': // LATIN SMALL O W/ GRAVE (0x00f2) + case u'ó': // LATIN SMALL O W/ ACUTE (0x00f3) + case u'ô': // LATIN SMALL O W/ CIRCUMFLEX (0x00f4) + case u'õ': // LATIN SMALL O W/ TILDE (0x00f5) + case u'ö': // LATIN SMALL O W/ DIAERESIS (0x00f6) + case u'ø': // LATIN SMALL O W/ STROKE (0x00f8) + case u'ù': // LATIN SMALL U W/ GRAVE (0x00f9) + case u'ú': // LATIN SMALL U W/ ACUTE (0x00fa) + case u'û': // LATIN SMALL U W/ CIRCUMFLEX (0x00fb) + case u'ü': // LATIN SMALL U W/ DIAERESIS (0x00fc) + case u'ý': // LATIN SMALL Y W/ ACUTE (0x00fd) + case u'þ': // LATIN SMALL THORN (0x00fe) + case u'ÿ': // LATIN SMALL Y W/ DIAERESIS (0x00ff) + case u'ā': // LATIN SMALL A W/ MACRON (0x0101) + case u'ă': // LATIN SMALL A W/ BREVE (0x0103) + case u'ą': // LATIN SMALL A W/ OGONEK (0x0105) + case u'ć': // LATIN SMALL C W/ ACUTE (0x0107) + case u'ĉ': // LATIN SMALL C W/ CIRCUMFLEX (0x0109) + case u'ċ': // LATIN SMALL C W/ DOT ABOVE (0x010b) + case u'č': // LATIN SMALL C W/ CARON (0x010d) + case u'ď': // LATIN SMALL D W/ CARON (0x010f) + case u'đ': // LATIN SMALL D W/ STROKE (0x0111) + case u'ē': // LATIN SMALL E W/ MACRON (0x0113) + case u'ĕ': // LATIN SMALL E W/ BREVE (0x0115) + case u'ė': // LATIN SMALL E W/ DOT ABOVE (0x0117) + case u'ę': // LATIN SMALL E W/ OGONEK (0x0119) + case u'ě': // LATIN SMALL E W/ CARON (0x011b) + case u'ĝ': // LATIN SMALL G W/ CIRCUMFLEX (0x011d) + case u'ğ': // LATIN SMALL G W/ BREVE (0x011f) + case u'ġ': // LATIN SMALL G W/ DOT ABOVE (0x0121) + case u'ģ': // LATIN SMALL G W/ CEDILLA (0x0123) + case u'ĥ': // LATIN SMALL H W/ CIRCUMFLEX (0x0125) + case u'ħ': // LATIN SMALL H W/ STROKE (0x0127) + case u'ĩ': // LATIN SMALL I W/ TILDE (0x0129) + case u'ī': // LATIN SMALL I W/ MACRON (0x012b) + case u'ĭ': // LATIN SMALL I W/ BREVE (0x012d) + case u'į': // LATIN SMALL I W/ OGONEK (0x012f) + case u'ı': // LATIN SMALL DOTLESS I (0x0131) + case u'ij': // LATIN SMALL LIGATURE IJ (0x0133) + case u'ĵ': // LATIN SMALL J W/ CIRCUMFLEX (0x0135) + case u'ķ': // LATIN SMALL K W/ CEDILLA (0x0137) + case u'ĸ': // LATIN SMALL KRA (0x0138) + case u'ĺ': // LATIN SMALL L W/ ACUTE (0x013a) + case u'ļ': // LATIN SMALL L W/ CEDILLA (0x013c) + case u'ľ': // LATIN SMALL L W/ CARON (0x013e) + case u'ŀ': // LATIN SMALL L W/ MIDDLE DOT (0x0140) + case u'ł': // LATIN SMALL L W/ STROKE (0x0142) + case u'ń': // LATIN SMALL N W/ ACUTE (0x0144) + case u'ņ': // LATIN SMALL N W/ CEDILLA (0x0146) + case u'ň': // LATIN SMALL N W/ CARON (0x0148) + case u'ʼn': // LATIN SMALL N PRECEDED BY APOSTROPHE (0x0149) + case u'ŋ': // LATIN SMALL ENG (0x014b) + case u'ō': // LATIN SMALL O W/ MACRON (0x014d) + case u'ŏ': // LATIN SMALL O W/ BREVE (0x014f) + case u'ő': // LATIN SMALL O W/ DOUBLE ACUTE (0x0151) + case u'œ': // LATIN SMALL LIGATURE OE (0x0153) + case u'ŕ': // LATIN SMALL R W/ ACUTE (0x0155) + case u'ŗ': // LATIN SMALL R W/ CEDILLA (0x0157) + case u'ř': // LATIN SMALL R W/ CARON (0x0159) + case u'ś': // LATIN SMALL S W/ ACUTE (0x015b) + case u'ŝ': // LATIN SMALL S W/ CIRCUMFLEX (0x015d) + case u'ş': // LATIN SMALL S W/ CEDILLA (0x015f) + case u'š': // LATIN SMALL S W/ CARON (0x0161) + case u'ţ': // LATIN SMALL T W/ CEDILLA (0x0163) + case u'ť': // LATIN SMALL T W/ CARON (0x0165) + case u'ŧ': // LATIN SMALL T W/ STROKE (0x0167) + case u'ũ': // LATIN SMALL U W/ TILDE (0x0169) + case u'ū': // LATIN SMALL U W/ MACRON (0x016b) + case u'ŭ': // LATIN SMALL U W/ BREVE (0x016d) + case u'ů': // LATIN SMALL U W/ RING ABOVE (0x016f) + case u'ű': // LATIN SMALL U W/ DOUBLE ACUTE (0x0171) + case u'ų': // LATIN SMALL U W/ OGONEK (0x0173) + case u'ŵ': // LATIN SMALL W W/ CIRCUMFLEX (0x0175) + case u'ŷ': // LATIN SMALL Y W/ CIRCUMFLEX (0x0177) + case u'ź': // LATIN SMALL Z W/ ACUTE (0x017a) + case u'ż': // LATIN SMALL Z W/ DOT ABOVE (0x017c) + case u'ž': // LATIN SMALL Z W/ CARON (0x017e) + case u'ſ': // LATIN SMALL LONG S (0x017f) + case u'ƀ': // LATIN SMALL B W/ STROKE (0x0180) + case u'ƃ': // LATIN SMALL B W/ TOPBAR (0x0183) + case u'ƅ': // LATIN SMALL TONE SIX (0x0185) + case u'ƈ': // LATIN SMALL C W/ HOOK (0x0188) + case u'ƌ': // LATIN SMALL D W/ TOPBAR (0x018c) + case u'ƍ': // LATIN SMALL TURNED DELTA (0x018d) + case u'ƒ': // LATIN SMALL F W/ HOOK (0x0192) + case u'ƕ': // LATIN SMALL HV (0x0195) + case u'ƙ': // LATIN SMALL K W/ HOOK (0x0199) + case u'ƚ': // LATIN SMALL L W/ BAR (0x019a) + case u'ƛ': // LATIN SMALL LAMBDA W/ STROKE (0x019b) + case u'ƞ': // LATIN SMALL N W/ LONG RIGHT LEG (0x019e) + case u'ơ': // LATIN SMALL O W/ HORN (0x01a1) + case u'ƣ': // LATIN SMALL OI (0x01a3) + case u'ƥ': // LATIN SMALL P W/ HOOK (0x01a5) + case u'ƨ': // LATIN SMALL TONE TWO (0x01a8) + case u'ƪ': // LATIN LETTER REVERSED ESH LOOP (0x01aa) + case u'ƫ': // LATIN SMALL T W/ PALATAL HOOK (0x01ab) + case u'ƭ': // LATIN SMALL T W/ HOOK (0x01ad) + case u'ư': // LATIN SMALL U W/ HORN (0x01b0) + case u'ƴ': // LATIN SMALL Y W/ HOOK (0x01b4) + case u'ƶ': // LATIN SMALL Z W/ STROKE (0x01b6) + case u'ƹ': // LATIN SMALL EZH REVERSED (0x01b9) + case u'ƺ': // LATIN SMALL EZH W/ TAIL (0x01ba) + case u'ƽ': // LATIN SMALL TONE FIVE (0x01bd) + case u'ƾ': // LATIN LETTER INVERTED GLOTTAL STOP W/ STROKE (0x01be) + case u'ƿ': // LATIN LETTER WYNN (0x01bf) + case u'dž': // LATIN SMALL DZ W/ CARON (0x01c6) + case u'lj': // LATIN SMALL LJ (0x01c9) + case u'nj': // LATIN SMALL NJ (0x01cc) + case u'ǎ': // LATIN SMALL A W/ CARON (0x01ce) + case u'ǐ': // LATIN SMALL I W/ CARON (0x01d0) + case u'ǒ': // LATIN SMALL O W/ CARON (0x01d2) + case u'ǔ': // LATIN SMALL U W/ CARON (0x01d4) + case u'ǖ': // LATIN SMALL U W/ DIAERESIS AND MACRON (0x01d6) + case u'ǘ': // LATIN SMALL U W/ DIAERESIS AND ACUTE (0x01d8) + case u'ǚ': // LATIN SMALL U W/ DIAERESIS AND CARON (0x01da) + case u'ǜ': // LATIN SMALL U W/ DIAERESIS AND GRAVE (0x01dc) + case u'ǝ': // LATIN SMALL TURNED E (0x01dd) + case u'ǟ': // LATIN SMALL A W/ DIAERESIS AND MACRON (0x01df) + case u'ǡ': // LATIN SMALL A W/ DOT ABOVE AND MACRON (0x01e1) + case u'ǣ': // LATIN SMALL AE W/ MACRON (0x01e3) + case u'ǥ': // LATIN SMALL G W/ STROKE (0x01e5) + case u'ǧ': // LATIN SMALL G W/ CARON (0x01e7) + case u'ǩ': // LATIN SMALL K W/ CARON (0x01e9) + case u'ǫ': // LATIN SMALL O W/ OGONEK (0x01eb) + case u'ǭ': // LATIN SMALL O W/ OGONEK AND MACRON (0x01ed) + case u'ǯ': // LATIN SMALL EZH W/ CARON (0x01ef) + case u'ǰ': // LATIN SMALL J W/ CARON (0x01f0) + case u'dz': // LATIN SMALL DZ (0x01f3) + case u'ǵ': // LATIN SMALL G W/ ACUTE (0x01f5) + case u'ǹ': // LATIN SMALL N W/ GRAVE (0x01f9) + case u'ǻ': // LATIN SMALL A W/ RING ABOVE AND ACUTE (0x01fb) + case u'ǽ': // LATIN SMALL AE W/ ACUTE (0x01fd) + case u'ǿ': // LATIN SMALL O W/ STROKE AND ACUTE (0x01ff) + case u'ȁ': // LATIN SMALL A W/ DOUBLE GRAVE (0x0201) + case u'ȃ': // LATIN SMALL A W/ INVERTED BREVE (0x0203) + case u'ȅ': // LATIN SMALL E W/ DOUBLE GRAVE (0x0205) + case u'ȇ': // LATIN SMALL E W/ INVERTED BREVE (0x0207) + case u'ȉ': // LATIN SMALL I W/ DOUBLE GRAVE (0x0209) + case u'ȋ': // LATIN SMALL I W/ INVERTED BREVE (0x020b) + case u'ȍ': // LATIN SMALL O W/ DOUBLE GRAVE (0x020d) + case u'ȏ': // LATIN SMALL O W/ INVERTED BREVE (0x020f) + case u'ȑ': // LATIN SMALL R W/ DOUBLE GRAVE (0x0211) + case u'ȓ': // LATIN SMALL R W/ INVERTED BREVE (0x0213) + case u'ȕ': // LATIN SMALL U W/ DOUBLE GRAVE (0x0215) + case u'ȗ': // LATIN SMALL U W/ INVERTED BREVE (0x0217) + case u'ș': // LATIN SMALL S W/ COMMA BELOW (0x0219) + case u'ț': // LATIN SMALL T W/ COMMA BELOW (0x021b) + case u'ȝ': // LATIN SMALL YOGH (0x021d) + case u'ȟ': // LATIN SMALL H W/ CARON (0x021f) + case u'ȡ': // LATIN SMALL D W/ CURL (0x0221) + case u'ȣ': // LATIN SMALL OU (0x0223) + case u'ȥ': // LATIN SMALL Z W/ HOOK (0x0225) + case u'ȧ': // LATIN SMALL A W/ DOT ABOVE (0x0227) + case u'ȩ': // LATIN SMALL E W/ CEDILLA (0x0229) + case u'ȫ': // LATIN SMALL O W/ DIAERESIS AND MACRON (0x022b) + case u'ȭ': // LATIN SMALL O W/ TILDE AND MACRON (0x022d) + case u'ȯ': // LATIN SMALL O W/ DOT ABOVE (0x022f) + case u'ȱ': // LATIN SMALL O W/ DOT ABOVE AND MACRON (0x0231) + case u'ȳ': // LATIN SMALL Y W/ MACRON (0x0233) + case u'ȴ': // LATIN SMALL L W/ CURL (0x0234) + case u'ȵ': // LATIN SMALL N W/ CURL (0x0235) + case u'ȶ': // LATIN SMALL T W/ CURL (0x0236) + case u'ȷ': // LATIN SMALL DOTLESS J (0x0237) + case u'ȸ': // LATIN SMALL DB DIGRAPH (0x0238) + case u'ȹ': // LATIN SMALL QP DIGRAPH (0x0239) + case u'ȼ': // LATIN SMALL C W/ STROKE (0x023c) + case u'ȿ': // LATIN SMALL S W/ SWASH TAIL (0x023f) + case u'ɀ': // LATIN SMALL Z W/ SWASH TAIL (0x0240) + case u'ɂ': // LATIN SMALL GLOTTAL STOP (0x0242) + case u'ɇ': // LATIN SMALL E W/ STROKE (0x0247) + case u'ɉ': // LATIN SMALL J W/ STROKE (0x0249) + case u'ɋ': // LATIN SMALL Q W/ HOOK TAIL (0x024b) + case u'ɍ': // LATIN SMALL R W/ STROKE (0x024d) + case u'ɏ': // LATIN SMALL Y W/ STROKE (0x024f) + case u'ɐ': // LATIN SMALL TURNED A (0x0250) + case u'ɑ': // LATIN SMALL ALPHA (0x0251) + case u'ɒ': // LATIN SMALL TURNED ALPHA (0x0252) + case u'ɓ': // LATIN SMALL B W/ HOOK (0x0253) + case u'ɔ': // LATIN SMALL OPEN O (0x0254) + case u'ɕ': // LATIN SMALL C W/ CURL (0x0255) + case u'ɖ': // LATIN SMALL D W/ TAIL (0x0256) + case u'ɗ': // LATIN SMALL D W/ HOOK (0x0257) + case u'ɘ': // LATIN SMALL REVERSED E (0x0258) + case u'ə': // LATIN SMALL SCHWA (0x0259) + case u'ɚ': // LATIN SMALL SCHWA W/ HOOK (0x025a) + case u'ɛ': // LATIN SMALL OPEN E (0x025b) + case u'ɜ': // LATIN SMALL REVERSED OPEN E (0x025c) + case u'ɝ': // LATIN SMALL REVERSED OPEN E W/ HOOK (0x025d) + case u'ɞ': // LATIN SMALL CLOSED REVERSED OPEN E (0x025e) + case u'ɟ': // LATIN SMALL DOTLESS J W/ STROKE (0x025f) + case u'ɠ': // LATIN SMALL G W/ HOOK (0x0260) + case u'ɡ': // LATIN SMALL SCRIPT G (0x0261) + case u'ɢ': // LATIN LETTER SMALL CAPITAL G (0x0262) + case u'ɣ': // LATIN SMALL GAMMA (0x0263) + case u'ɤ': // LATIN SMALL RAMS HORN (0x0264) + case u'ɥ': // LATIN SMALL TURNED H (0x0265) + case u'ɦ': // LATIN SMALL H W/ HOOK (0x0266) + case u'ɧ': // LATIN SMALL HENG W/ HOOK (0x0267) + case u'ɨ': // LATIN SMALL I W/ STROKE (0x0268) + case u'ɩ': // LATIN SMALL IOTA (0x0269) + case u'ɪ': // LATIN LETTER SMALL CAPITAL I (0x026a) + case u'ɫ': // LATIN SMALL L W/ MIDDLE TILDE (0x026b) + case u'ɬ': // LATIN SMALL L W/ BELT (0x026c) + case u'ɭ': // LATIN SMALL L W/ RETROFLEX HOOK (0x026d) + case u'ɮ': // LATIN SMALL LEZH (0x026e) + case u'ɯ': // LATIN SMALL TURNED M (0x026f) + case u'ɰ': // LATIN SMALL TURNED M W/ LONG LEG (0x0270) + case u'ɱ': // LATIN SMALL M W/ HOOK (0x0271) + case u'ɲ': // LATIN SMALL N W/ LEFT HOOK (0x0272) + case u'ɳ': // LATIN SMALL N W/ RETROFLEX HOOK (0x0273) + case u'ɴ': // LATIN LETTER SMALL CAPITAL N (0x0274) + case u'ɵ': // LATIN SMALL BARRED O (0x0275) + case u'ɶ': // LATIN LETTER SMALL CAPITAL OE (0x0276) + case u'ɷ': // LATIN SMALL CLOSED OMEGA (0x0277) + case u'ɸ': // LATIN SMALL PHI (0x0278) + case u'ɹ': // LATIN SMALL TURNED R (0x0279) + case u'ɺ': // LATIN SMALL TURNED R W/ LONG LEG (0x027a) + case u'ɻ': // LATIN SMALL TURNED R W/ HOOK (0x027b) + case u'ɼ': // LATIN SMALL R W/ LONG LEG (0x027c) + case u'ɽ': // LATIN SMALL R W/ TAIL (0x027d) + case u'ɾ': // LATIN SMALL R W/ FISHHOOK (0x027e) + case u'ɿ': // LATIN SMALL REVERSED R W/ FISHHOOK (0x027f) + case u'ʀ': // LATIN LETTER SMALL CAPITAL R (0x0280) + case u'ʁ': // LATIN LETTER SMALL CAPITAL INVERTED R (0x0281) + case u'ʂ': // LATIN SMALL S W/ HOOK (0x0282) + case u'ʃ': // LATIN SMALL ESH (0x0283) + case u'ʄ': // LATIN SMALL DOTLESS J W/ STROKE AND HOOK (0x0284) + case u'ʅ': // LATIN SMALL SQUAT REVERSED ESH (0x0285) + case u'ʆ': // LATIN SMALL ESH W/ CURL (0x0286) + case u'ʇ': // LATIN SMALL TURNED T (0x0287) + case u'ʈ': // LATIN SMALL T W/ RETROFLEX HOOK (0x0288) + case u'ʉ': // LATIN SMALL U BAR (0x0289) + case u'ʊ': // LATIN SMALL UPSILON (0x028a) + case u'ʋ': // LATIN SMALL V W/ HOOK (0x028b) + case u'ʌ': // LATIN SMALL TURNED V (0x028c) + case u'ʍ': // LATIN SMALL TURNED W (0x028d) + case u'ʎ': // LATIN SMALL TURNED Y (0x028e) + case u'ʏ': // LATIN LETTER SMALL CAPITAL Y (0x028f) + case u'ʐ': // LATIN SMALL Z W/ RETROFLEX HOOK (0x0290) + case u'ʑ': // LATIN SMALL Z W/ CURL (0x0291) + case u'ʒ': // LATIN SMALL EZH (0x0292) + case u'ʓ': // LATIN SMALL EZH W/ CURL (0x0293) + case u'ʕ': // LATIN LETTER PHARYNGEAL VOICED FRICATIVE (0x0295) + case u'ʖ': // LATIN LETTER INVERTED GLOTTAL STOP (0x0296) + case u'ʗ': // LATIN LETTER STRETCHED C (0x0297) + case u'ʘ': // LATIN LETTER BILABIAL CLICK (0x0298) + case u'ʙ': // LATIN LETTER SMALL CAPITAL B (0x0299) + case u'ʚ': // LATIN SMALL CLOSED OPEN E (0x029a) + case u'ʛ': // LATIN LETTER SMALL CAPITAL G W/ HOOK (0x029b) + case u'ʜ': // LATIN LETTER SMALL CAPITAL H (0x029c) + case u'ʝ': // LATIN SMALL J W/ CROSSED-TAIL (0x029d) + case u'ʞ': // LATIN SMALL TURNED K (0x029e) + case u'ʟ': // LATIN LETTER SMALL CAPITAL L (0x029f) + case u'ʠ': // LATIN SMALL Q W/ HOOK (0x02a0) + case u'ʡ': // LATIN LETTER GLOTTAL STOP W/ STROKE (0x02a1) + case u'ʢ': // LATIN LETTER REVERSED GLOTTAL STOP W/ STROKE (0x02a2) + case u'ʣ': // LATIN SMALL DZ DIGRAPH (0x02a3) + case u'ʤ': // LATIN SMALL DEZH DIGRAPH (0x02a4) + case u'ʥ': // LATIN SMALL DZ DIGRAPH W/ CURL (0x02a5) + case u'ʦ': // LATIN SMALL TS DIGRAPH (0x02a6) + case u'ʧ': // LATIN SMALL TESH DIGRAPH (0x02a7) + case u'ʨ': // LATIN SMALL TC DIGRAPH W/ CURL (0x02a8) + case u'ʩ': // LATIN SMALL FENG DIGRAPH (0x02a9) + case u'ʪ': // LATIN SMALL LS DIGRAPH (0x02aa) + case u'ʫ': // LATIN SMALL LZ DIGRAPH (0x02ab) + case u'ʬ': // LATIN LETTER BILABIAL PERCUSSIVE (0x02ac) + case u'ʭ': // LATIN LETTER BIDENTAL PERCUSSIVE (0x02ad) + case u'ʮ': // LATIN SMALL TURNED H W/ FISHHOOK (0x02ae) + case u'ʯ': // LATIN SMALL TURNED H W/ FISHHOOK AND TAIL (0x02af) + case u'ͱ': // GREEK SMALL HETA (0x0371) + case u'ͳ': // GREEK SMALL ARCHAIC SAMPI (0x0373) + case u'ͷ': // GREEK SMALL PAMPHYLIAN DIGAMMA (0x0377) + case u'ͻ': // GREEK SMALL REVERSED LUNATE SIGMA SYMBOL (0x037b) + case u'ͼ': // GREEK SMALL DOTTED LUNATE SIGMA SYMBOL (0x037c) + case u'ͽ': // GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL (0x037d) + case u'ΐ': // GREEK SMALL IOTA W/ DIALYTIKA AND TONOS (0x0390) + case u'ά': // GREEK SMALL ALPHA W/ TONOS (0x03ac) + case u'έ': // GREEK SMALL EPSILON W/ TONOS (0x03ad) + case u'ή': // GREEK SMALL ETA W/ TONOS (0x03ae) + case u'ί': // GREEK SMALL IOTA W/ TONOS (0x03af) + case u'ΰ': // GREEK SMALL UPSILON W/ DIALYTIKA AND TONOS (0x03b0) + case u'α': // GREEK SMALL ALPHA (0x03b1) + case u'β': // GREEK SMALL BETA (0x03b2) + case u'γ': // GREEK SMALL GAMMA (0x03b3) + case u'δ': // GREEK SMALL DELTA (0x03b4) + case u'ε': // GREEK SMALL EPSILON (0x03b5) + case u'ζ': // GREEK SMALL ZETA (0x03b6) + case u'η': // GREEK SMALL ETA (0x03b7) + case u'θ': // GREEK SMALL THETA (0x03b8) + case u'ι': // GREEK SMALL IOTA (0x03b9) + case u'κ': // GREEK SMALL KAPPA (0x03ba) + case u'λ': // GREEK SMALL LAMDA (0x03bb) + case u'μ': // GREEK SMALL MU (0x03bc) + case u'ν': // GREEK SMALL NU (0x03bd) + case u'ξ': // GREEK SMALL XI (0x03be) + case u'ο': // GREEK SMALL OMICRON (0x03bf) + case u'π': // GREEK SMALL PI (0x03c0) + case u'ρ': // GREEK SMALL RHO (0x03c1) + case u'ς': // GREEK SMALL FINAL SIGMA (0x03c2) + case u'σ': // GREEK SMALL SIGMA (0x03c3) + case u'τ': // GREEK SMALL TAU (0x03c4) + case u'υ': // GREEK SMALL UPSILON (0x03c5) + case u'φ': // GREEK SMALL PHI (0x03c6) + case u'χ': // GREEK SMALL CHI (0x03c7) + case u'ψ': // GREEK SMALL PSI (0x03c8) + case u'ω': // GREEK SMALL OMEGA (0x03c9) + case u'ϊ': // GREEK SMALL IOTA W/ DIALYTIKA (0x03ca) + case u'ϋ': // GREEK SMALL UPSILON W/ DIALYTIKA (0x03cb) + case u'ό': // GREEK SMALL OMICRON W/ TONOS (0x03cc) + case u'ύ': // GREEK SMALL UPSILON W/ TONOS (0x03cd) + case u'ώ': // GREEK SMALL OMEGA W/ TONOS (0x03ce) + case u'ϐ': // GREEK BETA SYMBOL (0x03d0) + case u'ϑ': // GREEK THETA SYMBOL (0x03d1) + case u'ϕ': // GREEK PHI SYMBOL (0x03d5) + case u'ϖ': // GREEK PI SYMBOL (0x03d6) + case u'ϗ': // GREEK KAI SYMBOL (0x03d7) + case u'ϙ': // GREEK SMALL ARCHAIC KOPPA (0x03d9) + case u'ϛ': // GREEK SMALL STIGMA (0x03db) + case u'ϝ': // GREEK SMALL DIGAMMA (0x03dd) + case u'ϟ': // GREEK SMALL KOPPA (0x03df) + case u'ϡ': // GREEK SMALL SAMPI (0x03e1) + case u'ϣ': // COPTIC SMALL SHEI (0x03e3) + case u'ϥ': // COPTIC SMALL FEI (0x03e5) + case u'ϧ': // COPTIC SMALL KHEI (0x03e7) + case u'ϩ': // COPTIC SMALL HORI (0x03e9) + case u'ϫ': // COPTIC SMALL GANGIA (0x03eb) + case u'ϭ': // COPTIC SMALL SHIMA (0x03ed) + case u'ϯ': // COPTIC SMALL DEI (0x03ef) + case u'ϰ': // GREEK KAPPA SYMBOL (0x03f0) + case u'ϱ': // GREEK RHO SYMBOL (0x03f1) + case u'ϲ': // GREEK LUNATE SIGMA SYMBOL (0x03f2) + case u'ϳ': // GREEK LETTER YOT (0x03f3) + case u'ϵ': // GREEK LUNATE EPSILON SYMBOL (0x03f5) + case u'ϸ': // GREEK SMALL SHO (0x03f8) + case u'ϻ': // GREEK SMALL SAN (0x03fb) + case u'ϼ': // GREEK RHO W/ STROKE SYMBOL (0x03fc) + case u'а': // CYRILLIC SMALL A (0x0430) + case u'б': // CYRILLIC SMALL BE (0x0431) + case u'в': // CYRILLIC SMALL VE (0x0432) + case u'г': // CYRILLIC SMALL GHE (0x0433) + case u'д': // CYRILLIC SMALL DE (0x0434) + case u'е': // CYRILLIC SMALL IE (0x0435) + case u'ж': // CYRILLIC SMALL ZHE (0x0436) + case u'з': // CYRILLIC SMALL ZE (0x0437) + case u'и': // CYRILLIC SMALL I (0x0438) + case u'й': // CYRILLIC SMALL SHORT I (0x0439) + case u'к': // CYRILLIC SMALL KA (0x043a) + case u'л': // CYRILLIC SMALL EL (0x043b) + case u'м': // CYRILLIC SMALL EM (0x043c) + case u'н': // CYRILLIC SMALL EN (0x043d) + case u'о': // CYRILLIC SMALL O (0x043e) + case u'п': // CYRILLIC SMALL PE (0x043f) + case u'р': // CYRILLIC SMALL ER (0x0440) + case u'с': // CYRILLIC SMALL ES (0x0441) + case u'т': // CYRILLIC SMALL TE (0x0442) + case u'у': // CYRILLIC SMALL U (0x0443) + case u'ф': // CYRILLIC SMALL EF (0x0444) + case u'х': // CYRILLIC SMALL HA (0x0445) + case u'ц': // CYRILLIC SMALL TSE (0x0446) + case u'ч': // CYRILLIC SMALL CHE (0x0447) + case u'ш': // CYRILLIC SMALL SHA (0x0448) + case u'щ': // CYRILLIC SMALL SHCHA (0x0449) + case u'ъ': // CYRILLIC SMALL HARD SIGN (0x044a) + case u'ы': // CYRILLIC SMALL YERU (0x044b) + case u'ь': // CYRILLIC SMALL SOFT SIGN (0x044c) + case u'э': // CYRILLIC SMALL E (0x044d) + case u'ю': // CYRILLIC SMALL YU (0x044e) + case u'я': // CYRILLIC SMALL YA (0x044f) + case u'ѐ': // CYRILLIC SMALL IE W/ GRAVE (0x0450) + case u'ё': // CYRILLIC SMALL IO (0x0451) + case u'ђ': // CYRILLIC SMALL DJE (0x0452) + case u'ѓ': // CYRILLIC SMALL GJE (0x0453) + case u'є': // CYRILLIC SMALL UKRAINIAN IE (0x0454) + case u'ѕ': // CYRILLIC SMALL DZE (0x0455) + case u'і': // CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I (0x0456) + case u'ї': // CYRILLIC SMALL YI (0x0457) + case u'ј': // CYRILLIC SMALL JE (0x0458) + case u'љ': // CYRILLIC SMALL LJE (0x0459) + case u'њ': // CYRILLIC SMALL NJE (0x045a) + case u'ћ': // CYRILLIC SMALL TSHE (0x045b) + case u'ќ': // CYRILLIC SMALL KJE (0x045c) + case u'ѝ': // CYRILLIC SMALL I W/ GRAVE (0x045d) + case u'ў': // CYRILLIC SMALL SHORT U (0x045e) + case u'џ': // CYRILLIC SMALL DZHE (0x045f) + case u'ѡ': // CYRILLIC SMALL OMEGA (0x0461) + case u'ѣ': // CYRILLIC SMALL YAT (0x0463) + case u'ѥ': // CYRILLIC SMALL IOTIFIED E (0x0465) + case u'ѧ': // CYRILLIC SMALL LITTLE YUS (0x0467) + case u'ѩ': // CYRILLIC SMALL IOTIFIED LITTLE YUS (0x0469) + case u'ѫ': // CYRILLIC SMALL BIG YUS (0x046b) + case u'ѭ': // CYRILLIC SMALL IOTIFIED BIG YUS (0x046d) + case u'ѯ': // CYRILLIC SMALL KSI (0x046f) + case u'ѱ': // CYRILLIC SMALL PSI (0x0471) + case u'ѳ': // CYRILLIC SMALL FITA (0x0473) + case u'ѵ': // CYRILLIC SMALL IZHITSA (0x0475) + case u'ѷ': // CYRILLIC SMALL IZHITSA W/ DOUBLE GRAVE ACCENT (0x0477) + case u'ѹ': // CYRILLIC SMALL UK (0x0479) + case u'ѻ': // CYRILLIC SMALL ROUND OMEGA (0x047b) + case u'ѽ': // CYRILLIC SMALL OMEGA W/ TITLO (0x047d) + case u'ѿ': // CYRILLIC SMALL OT (0x047f) + case u'ҁ': // CYRILLIC SMALL KOPPA (0x0481) + case u'ҋ': // CYRILLIC SMALL SHORT I W/ TAIL (0x048b) + case u'ҍ': // CYRILLIC SMALL SEMISOFT SIGN (0x048d) + case u'ҏ': // CYRILLIC SMALL ER W/ TICK (0x048f) + case u'ґ': // CYRILLIC SMALL GHE W/ UPTURN (0x0491) + case u'ғ': // CYRILLIC SMALL GHE W/ STROKE (0x0493) + case u'ҕ': // CYRILLIC SMALL GHE W/ MIDDLE HOOK (0x0495) + case u'җ': // CYRILLIC SMALL ZHE W/ DESCENDER (0x0497) + case u'ҙ': // CYRILLIC SMALL ZE W/ DESCENDER (0x0499) + case u'қ': // CYRILLIC SMALL KA W/ DESCENDER (0x049b) + case u'ҝ': // CYRILLIC SMALL KA W/ VERTICAL STROKE (0x049d) + case u'ҟ': // CYRILLIC SMALL KA W/ STROKE (0x049f) + case u'ҡ': // CYRILLIC SMALL BASHKIR KA (0x04a1) + case u'ң': // CYRILLIC SMALL EN W/ DESCENDER (0x04a3) + case u'ҥ': // CYRILLIC SMALL LIGATURE EN GHE (0x04a5) + case u'ҧ': // CYRILLIC SMALL PE W/ MIDDLE HOOK (0x04a7) + case u'ҩ': // CYRILLIC SMALL ABKHASIAN HA (0x04a9) + case u'ҫ': // CYRILLIC SMALL ES W/ DESCENDER (0x04ab) + case u'ҭ': // CYRILLIC SMALL TE W/ DESCENDER (0x04ad) + case u'ү': // CYRILLIC SMALL STRAIGHT U (0x04af) + case u'ұ': // CYRILLIC SMALL STRAIGHT U W/ STROKE (0x04b1) + case u'ҳ': // CYRILLIC SMALL HA W/ DESCENDER (0x04b3) + case u'ҵ': // CYRILLIC SMALL LIGATURE TE TSE (0x04b5) + case u'ҷ': // CYRILLIC SMALL CHE W/ DESCENDER (0x04b7) + case u'ҹ': // CYRILLIC SMALL CHE W/ VERTICAL STROKE (0x04b9) + case u'һ': // CYRILLIC SMALL SHHA (0x04bb) + case u'ҽ': // CYRILLIC SMALL ABKHASIAN CHE (0x04bd) + case u'ҿ': // CYRILLIC SMALL ABKHASIAN CHE W/ DESCENDER (0x04bf) + case u'ӂ': // CYRILLIC SMALL ZHE W/ BREVE (0x04c2) + case u'ӄ': // CYRILLIC SMALL KA W/ HOOK (0x04c4) + case u'ӆ': // CYRILLIC SMALL EL W/ TAIL (0x04c6) + case u'ӈ': // CYRILLIC SMALL EN W/ HOOK (0x04c8) + case u'ӊ': // CYRILLIC SMALL EN W/ TAIL (0x04ca) + case u'ӌ': // CYRILLIC SMALL KHAKASSIAN CHE (0x04cc) + case u'ӎ': // CYRILLIC SMALL EM W/ TAIL (0x04ce) + case u'ӏ': // CYRILLIC SMALL PALOCHKA (0x04cf) + case u'ӑ': // CYRILLIC SMALL A W/ BREVE (0x04d1) + case u'ӓ': // CYRILLIC SMALL A W/ DIAERESIS (0x04d3) + case u'ӕ': // CYRILLIC SMALL LIGATURE A IE (0x04d5) + case u'ӗ': // CYRILLIC SMALL IE W/ BREVE (0x04d7) + case u'ә': // CYRILLIC SMALL SCHWA (0x04d9) + case u'ӛ': // CYRILLIC SMALL SCHWA W/ DIAERESIS (0x04db) + case u'ӝ': // CYRILLIC SMALL ZHE W/ DIAERESIS (0x04dd) + case u'ӟ': // CYRILLIC SMALL ZE W/ DIAERESIS (0x04df) + case u'ӡ': // CYRILLIC SMALL ABKHASIAN DZE (0x04e1) + case u'ӣ': // CYRILLIC SMALL I W/ MACRON (0x04e3) + case u'ӥ': // CYRILLIC SMALL I W/ DIAERESIS (0x04e5) + case u'ӧ': // CYRILLIC SMALL O W/ DIAERESIS (0x04e7) + case u'ө': // CYRILLIC SMALL BARRED O (0x04e9) + case u'ӫ': // CYRILLIC SMALL BARRED O W/ DIAERESIS (0x04eb) + case u'ӭ': // CYRILLIC SMALL E W/ DIAERESIS (0x04ed) + case u'ӯ': // CYRILLIC SMALL U W/ MACRON (0x04ef) + case u'ӱ': // CYRILLIC SMALL U W/ DIAERESIS (0x04f1) + case u'ӳ': // CYRILLIC SMALL U W/ DOUBLE ACUTE (0x04f3) + case u'ӵ': // CYRILLIC SMALL CHE W/ DIAERESIS (0x04f5) + case u'ӷ': // CYRILLIC SMALL GHE W/ DESCENDER (0x04f7) + case u'ӹ': // CYRILLIC SMALL YERU W/ DIAERESIS (0x04f9) + case u'ӻ': // CYRILLIC SMALL GHE W/ STROKE AND HOOK (0x04fb) + case u'ӽ': // CYRILLIC SMALL HA W/ HOOK (0x04fd) + case u'ӿ': // CYRILLIC SMALL HA W/ STROKE (0x04ff) + case u'ԁ': // CYRILLIC SMALL KOMI DE (0x0501) + case u'ԃ': // CYRILLIC SMALL KOMI DJE (0x0503) + case u'ԅ': // CYRILLIC SMALL KOMI ZJE (0x0505) + case u'ԇ': // CYRILLIC SMALL KOMI DZJE (0x0507) + case u'ԉ': // CYRILLIC SMALL KOMI LJE (0x0509) + case u'ԋ': // CYRILLIC SMALL KOMI NJE (0x050b) + case u'ԍ': // CYRILLIC SMALL KOMI SJE (0x050d) + case u'ԏ': // CYRILLIC SMALL KOMI TJE (0x050f) + case u'ԑ': // CYRILLIC SMALL REVERSED ZE (0x0511) + case u'ԓ': // CYRILLIC SMALL EL W/ HOOK (0x0513) + case u'ԕ': // CYRILLIC SMALL LHA (0x0515) + case u'ԗ': // CYRILLIC SMALL RHA (0x0517) + case u'ԙ': // CYRILLIC SMALL YAE (0x0519) + case u'ԛ': // CYRILLIC SMALL QA (0x051b) + case u'ԝ': // CYRILLIC SMALL WE (0x051d) + case u'ԟ': // CYRILLIC SMALL ALEUT KA (0x051f) + case u'ԡ': // CYRILLIC SMALL EL W/ MIDDLE HOOK (0x0521) + case u'ԣ': // CYRILLIC SMALL EN W/ MIDDLE HOOK (0x0523) + case u'ԥ': // CYRILLIC SMALL PE W/ DESCENDER (0x0525) + case u'ԧ': // CYRILLIC SMALL SHHA W/ DESCENDER (0x0527) + case u'ԩ': // CYRILLIC SMALL EN W/ LEFT HOOK (0x0529) + case u'ԫ': // CYRILLIC SMALL DZZHE (0x052b) + case u'ԭ': // CYRILLIC SMALL DCHE (0x052d) + case u'ԯ': // CYRILLIC SMALL EL W/ DESCENDER (0x052f) + case u'ՠ': // ARMENIAN SMALL TURNED AYB (0x0560) + case u'ա': // ARMENIAN SMALL AYB (0x0561) + case u'բ': // ARMENIAN SMALL BEN (0x0562) + case u'գ': // ARMENIAN SMALL GIM (0x0563) + case u'դ': // ARMENIAN SMALL DA (0x0564) + case u'ե': // ARMENIAN SMALL ECH (0x0565) + case u'զ': // ARMENIAN SMALL ZA (0x0566) + case u'է': // ARMENIAN SMALL EH (0x0567) + case u'ը': // ARMENIAN SMALL ET (0x0568) + case u'թ': // ARMENIAN SMALL TO (0x0569) + case u'ժ': // ARMENIAN SMALL ZHE (0x056a) + case u'ի': // ARMENIAN SMALL INI (0x056b) + case u'լ': // ARMENIAN SMALL LIWN (0x056c) + case u'խ': // ARMENIAN SMALL XEH (0x056d) + case u'ծ': // ARMENIAN SMALL CA (0x056e) + case u'կ': // ARMENIAN SMALL KEN (0x056f) + case u'հ': // ARMENIAN SMALL HO (0x0570) + case u'ձ': // ARMENIAN SMALL JA (0x0571) + case u'ղ': // ARMENIAN SMALL GHAD (0x0572) + case u'ճ': // ARMENIAN SMALL CHEH (0x0573) + case u'մ': // ARMENIAN SMALL MEN (0x0574) + case u'յ': // ARMENIAN SMALL YI (0x0575) + case u'ն': // ARMENIAN SMALL NOW (0x0576) + case u'շ': // ARMENIAN SMALL SHA (0x0577) + case u'ո': // ARMENIAN SMALL VO (0x0578) + case u'չ': // ARMENIAN SMALL CHA (0x0579) + case u'պ': // ARMENIAN SMALL PEH (0x057a) + case u'ջ': // ARMENIAN SMALL JHEH (0x057b) + case u'ռ': // ARMENIAN SMALL RA (0x057c) + case u'ս': // ARMENIAN SMALL SEH (0x057d) + case u'վ': // ARMENIAN SMALL VEW (0x057e) + case u'տ': // ARMENIAN SMALL TIWN (0x057f) + case u'ր': // ARMENIAN SMALL REH (0x0580) + case u'ց': // ARMENIAN SMALL CO (0x0581) + case u'ւ': // ARMENIAN SMALL YIWN (0x0582) + case u'փ': // ARMENIAN SMALL PIWR (0x0583) + case u'ք': // ARMENIAN SMALL KEH (0x0584) + case u'օ': // ARMENIAN SMALL OH (0x0585) + case u'ֆ': // ARMENIAN SMALL FEH (0x0586) + case u'և': // ARMENIAN SMALL LIGATURE ECH YIWN (0x0587) + case u'ֈ': // ARMENIAN SMALL YI W/ STROKE (0x0588) + case u'ა': // GEORGIAN LETTER AN (0x10d0) + case u'ბ': // GEORGIAN LETTER BAN (0x10d1) + case u'გ': // GEORGIAN LETTER GAN (0x10d2) + case u'დ': // GEORGIAN LETTER DON (0x10d3) + case u'ე': // GEORGIAN LETTER EN (0x10d4) + case u'ვ': // GEORGIAN LETTER VIN (0x10d5) + case u'ზ': // GEORGIAN LETTER ZEN (0x10d6) + case u'თ': // GEORGIAN LETTER TAN (0x10d7) + case u'ი': // GEORGIAN LETTER IN (0x10d8) + case u'კ': // GEORGIAN LETTER KAN (0x10d9) + case u'ლ': // GEORGIAN LETTER LAS (0x10da) + case u'მ': // GEORGIAN LETTER MAN (0x10db) + case u'ნ': // GEORGIAN LETTER NAR (0x10dc) + case u'ო': // GEORGIAN LETTER ON (0x10dd) + case u'პ': // GEORGIAN LETTER PAR (0x10de) + case u'ჟ': // GEORGIAN LETTER ZHAR (0x10df) + case u'რ': // GEORGIAN LETTER RAE (0x10e0) + case u'ს': // GEORGIAN LETTER SAN (0x10e1) + case u'ტ': // GEORGIAN LETTER TAR (0x10e2) + case u'უ': // GEORGIAN LETTER UN (0x10e3) + case u'ფ': // GEORGIAN LETTER PHAR (0x10e4) + case u'ქ': // GEORGIAN LETTER KHAR (0x10e5) + case u'ღ': // GEORGIAN LETTER GHAN (0x10e6) + case u'ყ': // GEORGIAN LETTER QAR (0x10e7) + case u'შ': // GEORGIAN LETTER SHIN (0x10e8) + case u'ჩ': // GEORGIAN LETTER CHIN (0x10e9) + case u'ც': // GEORGIAN LETTER CAN (0x10ea) + case u'ძ': // GEORGIAN LETTER JIL (0x10eb) + case u'წ': // GEORGIAN LETTER CIL (0x10ec) + case u'ჭ': // GEORGIAN LETTER CHAR (0x10ed) + case u'ხ': // GEORGIAN LETTER XAN (0x10ee) + case u'ჯ': // GEORGIAN LETTER JHAN (0x10ef) + case u'ჰ': // GEORGIAN LETTER HAE (0x10f0) + case u'ჱ': // GEORGIAN LETTER HE (0x10f1) + case u'ჲ': // GEORGIAN LETTER HIE (0x10f2) + case u'ჳ': // GEORGIAN LETTER WE (0x10f3) + case u'ჴ': // GEORGIAN LETTER HAR (0x10f4) + case u'ჵ': // GEORGIAN LETTER HOE (0x10f5) + case u'ჶ': // GEORGIAN LETTER FI (0x10f6) + case u'ჷ': // GEORGIAN LETTER YN (0x10f7) + case u'ჸ': // GEORGIAN LETTER ELIFI (0x10f8) + case u'ჹ': // GEORGIAN LETTER TURNED GAN (0x10f9) + case u'ჺ': // GEORGIAN LETTER AIN (0x10fa) + case u'ჽ': // GEORGIAN LETTER AEN (0x10fd) + case u'ჾ': // GEORGIAN LETTER HARD SIGN (0x10fe) + case u'ჿ': // GEORGIAN LETTER LABIAL SIGN (0x10ff) + case u'ᏸ': // CHEROKEE SMALL YE (0x13f8) + case u'ᏹ': // CHEROKEE SMALL YI (0x13f9) + case u'ᏺ': // CHEROKEE SMALL YO (0x13fa) + case u'ᏻ': // CHEROKEE SMALL YU (0x13fb) + case u'ᏼ': // CHEROKEE SMALL YV (0x13fc) + case u'ᏽ': // CHEROKEE SMALL MV (0x13fd) + case u'ᲀ': // CYRILLIC SMALL ROUNDED VE (0x1c80) + case u'ᲁ': // CYRILLIC SMALL LONG-LEGGED DE (0x1c81) + case u'ᲂ': // CYRILLIC SMALL NARROW O (0x1c82) + case u'ᲃ': // CYRILLIC SMALL WIDE ES (0x1c83) + case u'ᲄ': // CYRILLIC SMALL TALL TE (0x1c84) + case u'ᲅ': // CYRILLIC SMALL THREE-LEGGED TE (0x1c85) + case u'ᲆ': // CYRILLIC SMALL TALL HARD SIGN (0x1c86) + case u'ᲇ': // CYRILLIC SMALL TALL YAT (0x1c87) + case u'ᲈ': // CYRILLIC SMALL UNBLENDED UK (0x1c88) + case u'ᴀ': // LATIN LETTER SMALL CAPITAL A (0x1d00) + case u'ᴁ': // LATIN LETTER SMALL CAPITAL AE (0x1d01) + case u'ᴂ': // LATIN SMALL TURNED AE (0x1d02) + case u'ᴃ': // LATIN LETTER SMALL CAPITAL BARRED B (0x1d03) + case u'ᴄ': // LATIN LETTER SMALL CAPITAL C (0x1d04) + case u'ᴅ': // LATIN LETTER SMALL CAPITAL D (0x1d05) + case u'ᴆ': // LATIN LETTER SMALL CAPITAL ETH (0x1d06) + case u'ᴇ': // LATIN LETTER SMALL CAPITAL E (0x1d07) + case u'ᴈ': // LATIN SMALL TURNED OPEN E (0x1d08) + case u'ᴉ': // LATIN SMALL TURNED I (0x1d09) + case u'ᴊ': // LATIN LETTER SMALL CAPITAL J (0x1d0a) + case u'ᴋ': // LATIN LETTER SMALL CAPITAL K (0x1d0b) + case u'ᴌ': // LATIN LETTER SMALL CAPITAL L W/ STROKE (0x1d0c) + case u'ᴍ': // LATIN LETTER SMALL CAPITAL M (0x1d0d) + case u'ᴎ': // LATIN LETTER SMALL CAPITAL REVERSED N (0x1d0e) + case u'ᴏ': // LATIN LETTER SMALL CAPITAL O (0x1d0f) + case u'ᴐ': // LATIN LETTER SMALL CAPITAL OPEN O (0x1d10) + case u'ᴑ': // LATIN SMALL SIDEWAYS O (0x1d11) + case u'ᴒ': // LATIN SMALL SIDEWAYS OPEN O (0x1d12) + case u'ᴓ': // LATIN SMALL SIDEWAYS O W/ STROKE (0x1d13) + case u'ᴔ': // LATIN SMALL TURNED OE (0x1d14) + case u'ᴕ': // LATIN LETTER SMALL CAPITAL OU (0x1d15) + case u'ᴖ': // LATIN SMALL TOP HALF O (0x1d16) + case u'ᴗ': // LATIN SMALL BOTTOM HALF O (0x1d17) + case u'ᴘ': // LATIN LETTER SMALL CAPITAL P (0x1d18) + case u'ᴙ': // LATIN LETTER SMALL CAPITAL REVERSED R (0x1d19) + case u'ᴚ': // LATIN LETTER SMALL CAPITAL TURNED R (0x1d1a) + case u'ᴛ': // LATIN LETTER SMALL CAPITAL T (0x1d1b) + case u'ᴜ': // LATIN LETTER SMALL CAPITAL U (0x1d1c) + case u'ᴝ': // LATIN SMALL SIDEWAYS U (0x1d1d) + case u'ᴞ': // LATIN SMALL SIDEWAYS DIAERESIZED U (0x1d1e) + case u'ᴟ': // LATIN SMALL SIDEWAYS TURNED M (0x1d1f) + case u'ᴠ': // LATIN LETTER SMALL CAPITAL V (0x1d20) + case u'ᴡ': // LATIN LETTER SMALL CAPITAL W (0x1d21) + case u'ᴢ': // LATIN LETTER SMALL CAPITAL Z (0x1d22) + case u'ᴣ': // LATIN LETTER SMALL CAPITAL EZH (0x1d23) + case u'ᴤ': // LATIN LETTER VOICED LARYNGEAL SPIRANT (0x1d24) + case u'ᴥ': // LATIN LETTER AIN (0x1d25) + case u'ᴦ': // GREEK LETTER SMALL CAPITAL GAMMA (0x1d26) + case u'ᴧ': // GREEK LETTER SMALL CAPITAL LAMDA (0x1d27) + case u'ᴨ': // GREEK LETTER SMALL CAPITAL PI (0x1d28) + case u'ᴩ': // GREEK LETTER SMALL CAPITAL RHO (0x1d29) + case u'ᴪ': // GREEK LETTER SMALL CAPITAL PSI (0x1d2a) + case u'ᴫ': // CYRILLIC LETTER SMALL CAPITAL EL (0x1d2b) + case u'ᵫ': // LATIN SMALL UE (0x1d6b) + case u'ᵬ': // LATIN SMALL B W/ MIDDLE TILDE (0x1d6c) + case u'ᵭ': // LATIN SMALL D W/ MIDDLE TILDE (0x1d6d) + case u'ᵮ': // LATIN SMALL F W/ MIDDLE TILDE (0x1d6e) + case u'ᵯ': // LATIN SMALL M W/ MIDDLE TILDE (0x1d6f) + case u'ᵰ': // LATIN SMALL N W/ MIDDLE TILDE (0x1d70) + case u'ᵱ': // LATIN SMALL P W/ MIDDLE TILDE (0x1d71) + case u'ᵲ': // LATIN SMALL R W/ MIDDLE TILDE (0x1d72) + case u'ᵳ': // LATIN SMALL R W/ FISHHOOK AND MIDDLE TILDE (0x1d73) + case u'ᵴ': // LATIN SMALL S W/ MIDDLE TILDE (0x1d74) + case u'ᵵ': // LATIN SMALL T W/ MIDDLE TILDE (0x1d75) + case u'ᵶ': // LATIN SMALL Z W/ MIDDLE TILDE (0x1d76) + case u'ᵷ': // LATIN SMALL TURNED G (0x1d77) + case u'ᵹ': // LATIN SMALL INSULAR G (0x1d79) + case u'ᵺ': // LATIN SMALL TH W/ STRIKETHROUGH (0x1d7a) + case u'ᵻ': // LATIN SMALL CAPITAL LETTER I W/ STROKE (0x1d7b) + case u'ᵼ': // LATIN SMALL IOTA W/ STROKE (0x1d7c) + case u'ᵽ': // LATIN SMALL P W/ STROKE (0x1d7d) + case u'ᵾ': // LATIN SMALL CAPITAL LETTER U W/ STROKE (0x1d7e) + case u'ᵿ': // LATIN SMALL UPSILON W/ STROKE (0x1d7f) + case u'ᶀ': // LATIN SMALL B W/ PALATAL HOOK (0x1d80) + case u'ᶁ': // LATIN SMALL D W/ PALATAL HOOK (0x1d81) + case u'ᶂ': // LATIN SMALL F W/ PALATAL HOOK (0x1d82) + case u'ᶃ': // LATIN SMALL G W/ PALATAL HOOK (0x1d83) + case u'ᶄ': // LATIN SMALL K W/ PALATAL HOOK (0x1d84) + case u'ᶅ': // LATIN SMALL L W/ PALATAL HOOK (0x1d85) + case u'ᶆ': // LATIN SMALL M W/ PALATAL HOOK (0x1d86) + case u'ᶇ': // LATIN SMALL N W/ PALATAL HOOK (0x1d87) + case u'ᶈ': // LATIN SMALL P W/ PALATAL HOOK (0x1d88) + case u'ᶉ': // LATIN SMALL R W/ PALATAL HOOK (0x1d89) + case u'ᶊ': // LATIN SMALL S W/ PALATAL HOOK (0x1d8a) + case u'ᶋ': // LATIN SMALL ESH W/ PALATAL HOOK (0x1d8b) + case u'ᶌ': // LATIN SMALL V W/ PALATAL HOOK (0x1d8c) + case u'ᶍ': // LATIN SMALL X W/ PALATAL HOOK (0x1d8d) + case u'ᶎ': // LATIN SMALL Z W/ PALATAL HOOK (0x1d8e) + case u'ᶏ': // LATIN SMALL A W/ RETROFLEX HOOK (0x1d8f) + case u'ᶐ': // LATIN SMALL ALPHA W/ RETROFLEX HOOK (0x1d90) + case u'ᶑ': // LATIN SMALL D W/ HOOK AND TAIL (0x1d91) + case u'ᶒ': // LATIN SMALL E W/ RETROFLEX HOOK (0x1d92) + case u'ᶓ': // LATIN SMALL OPEN E W/ RETROFLEX HOOK (0x1d93) + case u'ᶔ': // LATIN SMALL REVERSED OPEN E W/ RETROFLEX HOOK (0x1d94) + case u'ᶕ': // LATIN SMALL SCHWA W/ RETROFLEX HOOK (0x1d95) + case u'ᶖ': // LATIN SMALL I W/ RETROFLEX HOOK (0x1d96) + case u'ᶗ': // LATIN SMALL OPEN O W/ RETROFLEX HOOK (0x1d97) + case u'ᶘ': // LATIN SMALL ESH W/ RETROFLEX HOOK (0x1d98) + case u'ᶙ': // LATIN SMALL U W/ RETROFLEX HOOK (0x1d99) + case u'ᶚ': // LATIN SMALL EZH W/ RETROFLEX HOOK (0x1d9a) + case u'ḁ': // LATIN SMALL A W/ RING BELOW (0x1e01) + case u'ḃ': // LATIN SMALL B W/ DOT ABOVE (0x1e03) + case u'ḅ': // LATIN SMALL B W/ DOT BELOW (0x1e05) + case u'ḇ': // LATIN SMALL B W/ LINE BELOW (0x1e07) + case u'ḉ': // LATIN SMALL C W/ CEDILLA AND ACUTE (0x1e09) + case u'ḋ': // LATIN SMALL D W/ DOT ABOVE (0x1e0b) + case u'ḍ': // LATIN SMALL D W/ DOT BELOW (0x1e0d) + case u'ḏ': // LATIN SMALL D W/ LINE BELOW (0x1e0f) + case u'ḑ': // LATIN SMALL D W/ CEDILLA (0x1e11) + case u'ḓ': // LATIN SMALL D W/ CIRCUMFLEX BELOW (0x1e13) + case u'ḕ': // LATIN SMALL E W/ MACRON AND GRAVE (0x1e15) + case u'ḗ': // LATIN SMALL E W/ MACRON AND ACUTE (0x1e17) + case u'ḙ': // LATIN SMALL E W/ CIRCUMFLEX BELOW (0x1e19) + case u'ḛ': // LATIN SMALL E W/ TILDE BELOW (0x1e1b) + case u'ḝ': // LATIN SMALL E W/ CEDILLA AND BREVE (0x1e1d) + case u'ḟ': // LATIN SMALL F W/ DOT ABOVE (0x1e1f) + case u'ḡ': // LATIN SMALL G W/ MACRON (0x1e21) + case u'ḣ': // LATIN SMALL H W/ DOT ABOVE (0x1e23) + case u'ḥ': // LATIN SMALL H W/ DOT BELOW (0x1e25) + case u'ḧ': // LATIN SMALL H W/ DIAERESIS (0x1e27) + case u'ḩ': // LATIN SMALL H W/ CEDILLA (0x1e29) + case u'ḫ': // LATIN SMALL H W/ BREVE BELOW (0x1e2b) + case u'ḭ': // LATIN SMALL I W/ TILDE BELOW (0x1e2d) + case u'ḯ': // LATIN SMALL I W/ DIAERESIS AND ACUTE (0x1e2f) + case u'ḱ': // LATIN SMALL K W/ ACUTE (0x1e31) + case u'ḳ': // LATIN SMALL K W/ DOT BELOW (0x1e33) + case u'ḵ': // LATIN SMALL K W/ LINE BELOW (0x1e35) + case u'ḷ': // LATIN SMALL L W/ DOT BELOW (0x1e37) + case u'ḹ': // LATIN SMALL L W/ DOT BELOW AND MACRON (0x1e39) + case u'ḻ': // LATIN SMALL L W/ LINE BELOW (0x1e3b) + case u'ḽ': // LATIN SMALL L W/ CIRCUMFLEX BELOW (0x1e3d) + case u'ḿ': // LATIN SMALL M W/ ACUTE (0x1e3f) + case u'ṁ': // LATIN SMALL M W/ DOT ABOVE (0x1e41) + case u'ṃ': // LATIN SMALL M W/ DOT BELOW (0x1e43) + case u'ṅ': // LATIN SMALL N W/ DOT ABOVE (0x1e45) + case u'ṇ': // LATIN SMALL N W/ DOT BELOW (0x1e47) + case u'ṉ': // LATIN SMALL N W/ LINE BELOW (0x1e49) + case u'ṋ': // LATIN SMALL N W/ CIRCUMFLEX BELOW (0x1e4b) + case u'ṍ': // LATIN SMALL O W/ TILDE AND ACUTE (0x1e4d) + case u'ṏ': // LATIN SMALL O W/ TILDE AND DIAERESIS (0x1e4f) + case u'ṑ': // LATIN SMALL O W/ MACRON AND GRAVE (0x1e51) + case u'ṓ': // LATIN SMALL O W/ MACRON AND ACUTE (0x1e53) + case u'ṕ': // LATIN SMALL P W/ ACUTE (0x1e55) + case u'ṗ': // LATIN SMALL P W/ DOT ABOVE (0x1e57) + case u'ṙ': // LATIN SMALL R W/ DOT ABOVE (0x1e59) + case u'ṛ': // LATIN SMALL R W/ DOT BELOW (0x1e5b) + case u'ṝ': // LATIN SMALL R W/ DOT BELOW AND MACRON (0x1e5d) + case u'ṟ': // LATIN SMALL R W/ LINE BELOW (0x1e5f) + case u'ṡ': // LATIN SMALL S W/ DOT ABOVE (0x1e61) + case u'ṣ': // LATIN SMALL S W/ DOT BELOW (0x1e63) + case u'ṥ': // LATIN SMALL S W/ ACUTE AND DOT ABOVE (0x1e65) + case u'ṧ': // LATIN SMALL S W/ CARON AND DOT ABOVE (0x1e67) + case u'ṩ': // LATIN SMALL S W/ DOT BELOW AND DOT ABOVE (0x1e69) + case u'ṫ': // LATIN SMALL T W/ DOT ABOVE (0x1e6b) + case u'ṭ': // LATIN SMALL T W/ DOT BELOW (0x1e6d) + case u'ṯ': // LATIN SMALL T W/ LINE BELOW (0x1e6f) + case u'ṱ': // LATIN SMALL T W/ CIRCUMFLEX BELOW (0x1e71) + case u'ṳ': // LATIN SMALL U W/ DIAERESIS BELOW (0x1e73) + case u'ṵ': // LATIN SMALL U W/ TILDE BELOW (0x1e75) + case u'ṷ': // LATIN SMALL U W/ CIRCUMFLEX BELOW (0x1e77) + case u'ṹ': // LATIN SMALL U W/ TILDE AND ACUTE (0x1e79) + case u'ṻ': // LATIN SMALL U W/ MACRON AND DIAERESIS (0x1e7b) + case u'ṽ': // LATIN SMALL V W/ TILDE (0x1e7d) + case u'ṿ': // LATIN SMALL V W/ DOT BELOW (0x1e7f) + case u'ẁ': // LATIN SMALL W W/ GRAVE (0x1e81) + case u'ẃ': // LATIN SMALL W W/ ACUTE (0x1e83) + case u'ẅ': // LATIN SMALL W W/ DIAERESIS (0x1e85) + case u'ẇ': // LATIN SMALL W W/ DOT ABOVE (0x1e87) + case u'ẉ': // LATIN SMALL W W/ DOT BELOW (0x1e89) + case u'ẋ': // LATIN SMALL X W/ DOT ABOVE (0x1e8b) + case u'ẍ': // LATIN SMALL X W/ DIAERESIS (0x1e8d) + case u'ẏ': // LATIN SMALL Y W/ DOT ABOVE (0x1e8f) + case u'ẑ': // LATIN SMALL Z W/ CIRCUMFLEX (0x1e91) + case u'ẓ': // LATIN SMALL Z W/ DOT BELOW (0x1e93) + case u'ẕ': // LATIN SMALL Z W/ LINE BELOW (0x1e95) + case u'ẖ': // LATIN SMALL H W/ LINE BELOW (0x1e96) + case u'ẗ': // LATIN SMALL T W/ DIAERESIS (0x1e97) + case u'ẘ': // LATIN SMALL W W/ RING ABOVE (0x1e98) + case u'ẙ': // LATIN SMALL Y W/ RING ABOVE (0x1e99) + case u'ẚ': // LATIN SMALL A W/ RIGHT HALF RING (0x1e9a) + case u'ẛ': // LATIN SMALL LONG S W/ DOT ABOVE (0x1e9b) + case u'ẜ': // LATIN SMALL LONG S W/ DIAGONAL STROKE (0x1e9c) + case u'ẝ': // LATIN SMALL LONG S W/ HIGH STROKE (0x1e9d) + case u'ẟ': // LATIN SMALL DELTA (0x1e9f) + case u'ạ': // LATIN SMALL A W/ DOT BELOW (0x1ea1) + case u'ả': // LATIN SMALL A W/ HOOK ABOVE (0x1ea3) + case u'ấ': // LATIN SMALL A W/ CIRCUMFLEX AND ACUTE (0x1ea5) + case u'ầ': // LATIN SMALL A W/ CIRCUMFLEX AND GRAVE (0x1ea7) + case u'ẩ': // LATIN SMALL A W/ CIRCUMFLEX AND HOOK ABOVE (0x1ea9) + case u'ẫ': // LATIN SMALL A W/ CIRCUMFLEX AND TILDE (0x1eab) + case u'ậ': // LATIN SMALL A W/ CIRCUMFLEX AND DOT BELOW (0x1ead) + case u'ắ': // LATIN SMALL A W/ BREVE AND ACUTE (0x1eaf) + case u'ằ': // LATIN SMALL A W/ BREVE AND GRAVE (0x1eb1) + case u'ẳ': // LATIN SMALL A W/ BREVE AND HOOK ABOVE (0x1eb3) + case u'ẵ': // LATIN SMALL A W/ BREVE AND TILDE (0x1eb5) + case u'ặ': // LATIN SMALL A W/ BREVE AND DOT BELOW (0x1eb7) + case u'ẹ': // LATIN SMALL E W/ DOT BELOW (0x1eb9) + case u'ẻ': // LATIN SMALL E W/ HOOK ABOVE (0x1ebb) + case u'ẽ': // LATIN SMALL E W/ TILDE (0x1ebd) + case u'ế': // LATIN SMALL E W/ CIRCUMFLEX AND ACUTE (0x1ebf) + case u'ề': // LATIN SMALL E W/ CIRCUMFLEX AND GRAVE (0x1ec1) + case u'ể': // LATIN SMALL E W/ CIRCUMFLEX AND HOOK ABOVE (0x1ec3) + case u'ễ': // LATIN SMALL E W/ CIRCUMFLEX AND TILDE (0x1ec5) + case u'ệ': // LATIN SMALL E W/ CIRCUMFLEX AND DOT BELOW (0x1ec7) + case u'ỉ': // LATIN SMALL I W/ HOOK ABOVE (0x1ec9) + case u'ị': // LATIN SMALL I W/ DOT BELOW (0x1ecb) + case u'ọ': // LATIN SMALL O W/ DOT BELOW (0x1ecd) + case u'ỏ': // LATIN SMALL O W/ HOOK ABOVE (0x1ecf) + case u'ố': // LATIN SMALL O W/ CIRCUMFLEX AND ACUTE (0x1ed1) + case u'ồ': // LATIN SMALL O W/ CIRCUMFLEX AND GRAVE (0x1ed3) + case u'ổ': // LATIN SMALL O W/ CIRCUMFLEX AND HOOK ABOVE (0x1ed5) + case u'ỗ': // LATIN SMALL O W/ CIRCUMFLEX AND TILDE (0x1ed7) + case u'ộ': // LATIN SMALL O W/ CIRCUMFLEX AND DOT BELOW (0x1ed9) + case u'ớ': // LATIN SMALL O W/ HORN AND ACUTE (0x1edb) + case u'ờ': // LATIN SMALL O W/ HORN AND GRAVE (0x1edd) + case u'ở': // LATIN SMALL O W/ HORN AND HOOK ABOVE (0x1edf) + case u'ỡ': // LATIN SMALL O W/ HORN AND TILDE (0x1ee1) + case u'ợ': // LATIN SMALL O W/ HORN AND DOT BELOW (0x1ee3) + case u'ụ': // LATIN SMALL U W/ DOT BELOW (0x1ee5) + case u'ủ': // LATIN SMALL U W/ HOOK ABOVE (0x1ee7) + case u'ứ': // LATIN SMALL U W/ HORN AND ACUTE (0x1ee9) + case u'ừ': // LATIN SMALL U W/ HORN AND GRAVE (0x1eeb) + case u'ử': // LATIN SMALL U W/ HORN AND HOOK ABOVE (0x1eed) + case u'ữ': // LATIN SMALL U W/ HORN AND TILDE (0x1eef) + case u'ự': // LATIN SMALL U W/ HORN AND DOT BELOW (0x1ef1) + case u'ỳ': // LATIN SMALL Y W/ GRAVE (0x1ef3) + case u'ỵ': // LATIN SMALL Y W/ DOT BELOW (0x1ef5) + case u'ỷ': // LATIN SMALL Y W/ HOOK ABOVE (0x1ef7) + case u'ỹ': // LATIN SMALL Y W/ TILDE (0x1ef9) + case u'ỻ': // LATIN SMALL MIDDLE-WELSH LL (0x1efb) + case u'ỽ': // LATIN SMALL MIDDLE-WELSH V (0x1efd) + case u'ỿ': // LATIN SMALL Y W/ LOOP (0x1eff) + case u'ἀ': // GREEK SMALL ALPHA W/ PSILI (0x1f00) + case u'ἁ': // GREEK SMALL ALPHA W/ DASIA (0x1f01) + case u'ἂ': // GREEK SMALL ALPHA W/ PSILI AND VARIA (0x1f02) + case u'ἃ': // GREEK SMALL ALPHA W/ DASIA AND VARIA (0x1f03) + case u'ἄ': // GREEK SMALL ALPHA W/ PSILI AND OXIA (0x1f04) + case u'ἅ': // GREEK SMALL ALPHA W/ DASIA AND OXIA (0x1f05) + case u'ἆ': // GREEK SMALL ALPHA W/ PSILI AND PERISPOMENI (0x1f06) + case u'ἇ': // GREEK SMALL ALPHA W/ DASIA AND PERISPOMENI (0x1f07) + case u'ἐ': // GREEK SMALL EPSILON W/ PSILI (0x1f10) + case u'ἑ': // GREEK SMALL EPSILON W/ DASIA (0x1f11) + case u'ἒ': // GREEK SMALL EPSILON W/ PSILI AND VARIA (0x1f12) + case u'ἓ': // GREEK SMALL EPSILON W/ DASIA AND VARIA (0x1f13) + case u'ἔ': // GREEK SMALL EPSILON W/ PSILI AND OXIA (0x1f14) + case u'ἕ': // GREEK SMALL EPSILON W/ DASIA AND OXIA (0x1f15) + case u'ἠ': // GREEK SMALL ETA W/ PSILI (0x1f20) + case u'ἡ': // GREEK SMALL ETA W/ DASIA (0x1f21) + case u'ἢ': // GREEK SMALL ETA W/ PSILI AND VARIA (0x1f22) + case u'ἣ': // GREEK SMALL ETA W/ DASIA AND VARIA (0x1f23) + case u'ἤ': // GREEK SMALL ETA W/ PSILI AND OXIA (0x1f24) + case u'ἥ': // GREEK SMALL ETA W/ DASIA AND OXIA (0x1f25) + case u'ἦ': // GREEK SMALL ETA W/ PSILI AND PERISPOMENI (0x1f26) + case u'ἧ': // GREEK SMALL ETA W/ DASIA AND PERISPOMENI (0x1f27) + case u'ἰ': // GREEK SMALL IOTA W/ PSILI (0x1f30) + case u'ἱ': // GREEK SMALL IOTA W/ DASIA (0x1f31) + case u'ἲ': // GREEK SMALL IOTA W/ PSILI AND VARIA (0x1f32) + case u'ἳ': // GREEK SMALL IOTA W/ DASIA AND VARIA (0x1f33) + case u'ἴ': // GREEK SMALL IOTA W/ PSILI AND OXIA (0x1f34) + case u'ἵ': // GREEK SMALL IOTA W/ DASIA AND OXIA (0x1f35) + case u'ἶ': // GREEK SMALL IOTA W/ PSILI AND PERISPOMENI (0x1f36) + case u'ἷ': // GREEK SMALL IOTA W/ DASIA AND PERISPOMENI (0x1f37) + case u'ὀ': // GREEK SMALL OMICRON W/ PSILI (0x1f40) + case u'ὁ': // GREEK SMALL OMICRON W/ DASIA (0x1f41) + case u'ὂ': // GREEK SMALL OMICRON W/ PSILI AND VARIA (0x1f42) + case u'ὃ': // GREEK SMALL OMICRON W/ DASIA AND VARIA (0x1f43) + case u'ὄ': // GREEK SMALL OMICRON W/ PSILI AND OXIA (0x1f44) + case u'ὅ': // GREEK SMALL OMICRON W/ DASIA AND OXIA (0x1f45) + case u'ὐ': // GREEK SMALL UPSILON W/ PSILI (0x1f50) + case u'ὑ': // GREEK SMALL UPSILON W/ DASIA (0x1f51) + case u'ὒ': // GREEK SMALL UPSILON W/ PSILI AND VARIA (0x1f52) + case u'ὓ': // GREEK SMALL UPSILON W/ DASIA AND VARIA (0x1f53) + case u'ὔ': // GREEK SMALL UPSILON W/ PSILI AND OXIA (0x1f54) + case u'ὕ': // GREEK SMALL UPSILON W/ DASIA AND OXIA (0x1f55) + case u'ὖ': // GREEK SMALL UPSILON W/ PSILI AND PERISPOMENI (0x1f56) + case u'ὗ': // GREEK SMALL UPSILON W/ DASIA AND PERISPOMENI (0x1f57) + case u'ὠ': // GREEK SMALL OMEGA W/ PSILI (0x1f60) + case u'ὡ': // GREEK SMALL OMEGA W/ DASIA (0x1f61) + case u'ὢ': // GREEK SMALL OMEGA W/ PSILI AND VARIA (0x1f62) + case u'ὣ': // GREEK SMALL OMEGA W/ DASIA AND VARIA (0x1f63) + case u'ὤ': // GREEK SMALL OMEGA W/ PSILI AND OXIA (0x1f64) + case u'ὥ': // GREEK SMALL OMEGA W/ DASIA AND OXIA (0x1f65) + case u'ὦ': // GREEK SMALL OMEGA W/ PSILI AND PERISPOMENI (0x1f66) + case u'ὧ': // GREEK SMALL OMEGA W/ DASIA AND PERISPOMENI (0x1f67) + case u'ὰ': // GREEK SMALL ALPHA W/ VARIA (0x1f70) + case u'ά': // GREEK SMALL ALPHA W/ OXIA (0x1f71) + case u'ὲ': // GREEK SMALL EPSILON W/ VARIA (0x1f72) + case u'έ': // GREEK SMALL EPSILON W/ OXIA (0x1f73) + case u'ὴ': // GREEK SMALL ETA W/ VARIA (0x1f74) + case u'ή': // GREEK SMALL ETA W/ OXIA (0x1f75) + case u'ὶ': // GREEK SMALL IOTA W/ VARIA (0x1f76) + case u'ί': // GREEK SMALL IOTA W/ OXIA (0x1f77) + case u'ὸ': // GREEK SMALL OMICRON W/ VARIA (0x1f78) + case u'ό': // GREEK SMALL OMICRON W/ OXIA (0x1f79) + case u'ὺ': // GREEK SMALL UPSILON W/ VARIA (0x1f7a) + case u'ύ': // GREEK SMALL UPSILON W/ OXIA (0x1f7b) + case u'ὼ': // GREEK SMALL OMEGA W/ VARIA (0x1f7c) + case u'ώ': // GREEK SMALL OMEGA W/ OXIA (0x1f7d) + case u'ᾀ': // GREEK SMALL ALPHA W/ PSILI YPOGEGRAMMENI (0x1f80) + case u'ᾁ': // GREEK SMALL ALPHA W/ DASIA YPOGEGRAMMENI (0x1f81) + case u'ᾂ': // GREEK SMALL ALPHA W/ PSILI VARIA YPOGEGRAMMENI (0x1f82) + case u'ᾃ': // GREEK SMALL ALPHA W/ DASIA VARIA YPOGEGRAMMENI (0x1f83) + case u'ᾄ': // GREEK SMALL ALPHA W/ PSILI OXIA YPOGEGRAMMENI (0x1f84) + case u'ᾅ': // GREEK SMALL ALPHA W/ DASIA OXIA YPOGEGRAMMENI (0x1f85) + case u'ᾆ': // GREEK SMALL ALPHA W/ PSILI PERISPOMENI YPOGEGRAMMENI (0x1f86) + case u'ᾇ': // GREEK SMALL ALPHA W/ DASIA PERISPOMENI YPOGEGRAMMENI (0x1f87) + case u'ᾐ': // GREEK SMALL ETA W/ PSILI YPOGEGRAMMENI (0x1f90) + case u'ᾑ': // GREEK SMALL ETA W/ DASIA YPOGEGRAMMENI (0x1f91) + case u'ᾒ': // GREEK SMALL ETA W/ PSILI VARIA YPOGEGRAMMENI 0x1f92) + case u'ᾓ': // GREEK SMALL ETA W/ DASIA VARIA YPOGEGRAMMENI 0x1f93) + case u'ᾔ': // GREEK SMALL ETA W/ PSILI OXIA YPOGEGRAMMENI 0x1f94) + case u'ᾕ': // GREEK SMALL ETA W/ DASIA OXIA YPOGEGRAMMENI 0x1f95) + case u'ᾖ': // GREEK SMALL ETA W/ PSILI PERISPOMENI YPOGEGRAMMENI (0x1f96) + case u'ᾗ': // GREEK SMALL ETA W/ DASIA PERISPOMENI YPOGEGRAMMENI (0x1f97) + case u'ᾠ': // GREEK SMALL OMEGA W/ PSILI YPOGEGRAMMENI (0x1fa0) + case u'ᾡ': // GREEK SMALL OMEGA W/ DASIA YPOGEGRAMMENI (0x1fa1) + case u'ᾢ': // GREEK SMALL OMEGA W/ PSILI VARIA YPOGEGRAMMENI (0x1fa2) + case u'ᾣ': // GREEK SMALL OMEGA W/ DASIA VARIA YPOGEGRAMMENI (0x1fa3) + case u'ᾤ': // GREEK SMALL OMEGA W/ PSILI OXIA YPOGEGRAMMENI (0x1fa4) + case u'ᾥ': // GREEK SMALL OMEGA W/ DASIA OXIA YPOGEGRAMMENI (0x1fa5) + case u'ᾦ': // GREEK SMALL OMEGA W/ PSILI PERISPOMENI YPOGEGRAMMENI (0x1fa6) + case u'ᾧ': // GREEK SMALL OMEGA W/ DASIA PERISPOMENI YPOGEGRAMMENI (0x1fa7) + case u'ᾰ': // GREEK SMALL ALPHA W/ VRACHY (0x1fb0) + case u'ᾱ': // GREEK SMALL ALPHA W/ MACRON (0x1fb1) + case u'ᾲ': // GREEK SMALL ALPHA W/ VARIA YPOGEGRAMMENI (0x1fb2) + case u'ᾳ': // GREEK SMALL ALPHA W/ YPOGEGRAMMENI (0x1fb3) + case u'ᾴ': // GREEK SMALL ALPHA W/ OXIA AND YPOGEGRAMMENI (0x1fb4) + case u'ᾶ': // GREEK SMALL ALPHA W/ PERISPOMENI (0x1fb6) + case u'ᾷ': // GREEK SMALL ALPHA W/ PERISPOMENI AND YPOGEGRAMMENI (0x1fb7) + case u'ι': // GREEK PROSGEGRAMMENI (0x1fbe) + case u'ῂ': // GREEK SMALL ETA W/ VARIA AND YPOGEGRAMMENI (0x1fc2) + case u'ῃ': // GREEK SMALL ETA W/ YPOGEGRAMMENI (0x1fc3) + case u'ῄ': // GREEK SMALL ETA W/ OXIA AND YPOGEGRAMMENI (0x1fc4) + case u'ῆ': // GREEK SMALL ETA W/ PERISPOMENI (0x1fc6) + case u'ῇ': // GREEK SMALL ETA W/ PERISPOMENI AND YPOGEGRAMMENI (0x1fc7) + case u'ῐ': // GREEK SMALL IOTA W/ VRACHY (0x1fd0) + case u'ῑ': // GREEK SMALL IOTA W/ MACRON (0x1fd1) + case u'ῒ': // GREEK SMALL IOTA W/ DIALYTIKA AND VARIA (0x1fd2) + case u'ΐ': // GREEK SMALL IOTA W/ DIALYTIKA AND OXIA (0x1fd3) + case u'ῖ': // GREEK SMALL IOTA W/ PERISPOMENI (0x1fd6) + case u'ῗ': // GREEK SMALL IOTA W/ DIALYTIKA AND PERISPOMENI (0x1fd7) + case u'ῠ': // GREEK SMALL UPSILON W/ VRACHY (0x1fe0) + case u'ῡ': // GREEK SMALL UPSILON W/ MACRON (0x1fe1) + case u'ῢ': // GREEK SMALL UPSILON W/ DIALYTIKA AND VARIA (0x1fe2) + case u'ΰ': // GREEK SMALL UPSILON W/ DIALYTIKA AND OXIA (0x1fe3) + case u'ῤ': // GREEK SMALL RHO W/ PSILI (0x1fe4) + case u'ῥ': // GREEK SMALL RHO W/ DASIA (0x1fe5) + case u'ῦ': // GREEK SMALL UPSILON W/ PERISPOMENI (0x1fe6) + case u'ῧ': // GREEK SMALL UPSILON W/ DIALYTIKA AND PERISPOMENI (0x1fe7) + case u'ῲ': // GREEK SMALL OMEGA W/ VARIA AND YPOGEGRAMMENI (0x1ff2) + case u'ῳ': // GREEK SMALL OMEGA W/ YPOGEGRAMMENI (0x1ff3) + case u'ῴ': // GREEK SMALL OMEGA W/ OXIA AND YPOGEGRAMMENI (0x1ff4) + case u'ῶ': // GREEK SMALL OMEGA W/ PERISPOMENI (0x1ff6) + case u'ῷ': // GREEK SMALL OMEGA W/ PERISPOMENI AND YPOGEGRAMMENI (0x1ff7) + case u'ℊ': // SCRIPT SMALL G (0x210a) + case u'ℎ': // PLANCK CONSTANT (0x210e) + case u'ℏ': // PLANCK CONSTANT OVER TWO PI (0x210f) + case u'ℓ': // SCRIPT SMALL L (0x2113) + case u'ℯ': // SCRIPT SMALL E (0x212f) + case u'ℴ': // SCRIPT SMALL O (0x2134) + case u'ℹ': // INFORMATION SOURCE (0x2139) + case u'ℼ': // DOUBLE-STRUCK SMALL PI (0x213c) + case u'ℽ': // DOUBLE-STRUCK SMALL GAMMA (0x213d) + case u'ⅆ': // DOUBLE-STRUCK ITALIC SMALL D (0x2146) + case u'ⅇ': // DOUBLE-STRUCK ITALIC SMALL E (0x2147) + case u'ⅈ': // DOUBLE-STRUCK ITALIC SMALL I (0x2148) + case u'ⅉ': // DOUBLE-STRUCK ITALIC SMALL J (0x2149) + case u'ⅎ': // TURNED SMALL F (0x214e) + case u'ↄ': // LATIN SMALL REVERSED C (0x2184) + case u'ⰰ': // GLAGOLITIC SMALL AZU (0x2c30) + case u'ⰱ': // GLAGOLITIC SMALL BUKY (0x2c31) + case u'ⰲ': // GLAGOLITIC SMALL VEDE (0x2c32) + case u'ⰳ': // GLAGOLITIC SMALL GLAGOLI (0x2c33) + case u'ⰴ': // GLAGOLITIC SMALL DOBRO (0x2c34) + case u'ⰵ': // GLAGOLITIC SMALL YESTU (0x2c35) + case u'ⰶ': // GLAGOLITIC SMALL ZHIVETE (0x2c36) + case u'ⰷ': // GLAGOLITIC SMALL DZELO (0x2c37) + case u'ⰸ': // GLAGOLITIC SMALL ZEMLJA (0x2c38) + case u'ⰹ': // GLAGOLITIC SMALL IZHE (0x2c39) + case u'ⰺ': // GLAGOLITIC SMALL INITIAL IZHE (0x2c3a) + case u'ⰻ': // GLAGOLITIC SMALL I (0x2c3b) + case u'ⰼ': // GLAGOLITIC SMALL DJERVI (0x2c3c) + case u'ⰽ': // GLAGOLITIC SMALL KAKO (0x2c3d) + case u'ⰾ': // GLAGOLITIC SMALL LJUDIJE (0x2c3e) + case u'ⰿ': // GLAGOLITIC SMALL MYSLITE (0x2c3f) + case u'ⱀ': // GLAGOLITIC SMALL NASHI (0x2c40) + case u'ⱁ': // GLAGOLITIC SMALL ONU (0x2c41) + case u'ⱂ': // GLAGOLITIC SMALL POKOJI (0x2c42) + case u'ⱃ': // GLAGOLITIC SMALL RITSI (0x2c43) + case u'ⱄ': // GLAGOLITIC SMALL SLOVO (0x2c44) + case u'ⱅ': // GLAGOLITIC SMALL TVRIDO (0x2c45) + case u'ⱆ': // GLAGOLITIC SMALL UKU (0x2c46) + case u'ⱇ': // GLAGOLITIC SMALL FRITU (0x2c47) + case u'ⱈ': // GLAGOLITIC SMALL HERU (0x2c48) + case u'ⱉ': // GLAGOLITIC SMALL OTU (0x2c49) + case u'ⱊ': // GLAGOLITIC SMALL PE (0x2c4a) + case u'ⱋ': // GLAGOLITIC SMALL SHTA (0x2c4b) + case u'ⱌ': // GLAGOLITIC SMALL TSI (0x2c4c) + case u'ⱍ': // GLAGOLITIC SMALL CHRIVI (0x2c4d) + case u'ⱎ': // GLAGOLITIC SMALL SHA (0x2c4e) + case u'ⱏ': // GLAGOLITIC SMALL YERU (0x2c4f) + case u'ⱐ': // GLAGOLITIC SMALL YERI (0x2c50) + case u'ⱑ': // GLAGOLITIC SMALL YATI (0x2c51) + case u'ⱒ': // GLAGOLITIC SMALL SPIDERY HA (0x2c52) + case u'ⱓ': // GLAGOLITIC SMALL YU (0x2c53) + case u'ⱔ': // GLAGOLITIC SMALL SMALL YUS (0x2c54) + case u'ⱕ': // GLAGOLITIC SMALL SMALL YUS W/ TAIL (0x2c55) + case u'ⱖ': // GLAGOLITIC SMALL YO (0x2c56) + case u'ⱗ': // GLAGOLITIC SMALL IOTATED SMALL YUS (0x2c57) + case u'ⱘ': // GLAGOLITIC SMALL BIG YUS (0x2c58) + case u'ⱙ': // GLAGOLITIC SMALL IOTATED BIG YUS (0x2c59) + case u'ⱚ': // GLAGOLITIC SMALL FITA (0x2c5a) + case u'ⱛ': // GLAGOLITIC SMALL IZHITSA (0x2c5b) + case u'ⱜ': // GLAGOLITIC SMALL SHTAPIC (0x2c5c) + case u'ⱝ': // GLAGOLITIC SMALL TROKUTASTI A (0x2c5d) + case u'ⱞ': // GLAGOLITIC SMALL LATINATE MYSLITE (0x2c5e) + case u'ⱡ': // LATIN SMALL L W/ DOUBLE BAR (0x2c61) + case u'ⱥ': // LATIN SMALL A W/ STROKE (0x2c65) + case u'ⱦ': // LATIN SMALL T W/ DIAGONAL STROKE (0x2c66) + case u'ⱨ': // LATIN SMALL H W/ DESCENDER (0x2c68) + case u'ⱪ': // LATIN SMALL K W/ DESCENDER (0x2c6a) + case u'ⱬ': // LATIN SMALL Z W/ DESCENDER (0x2c6c) + case u'ⱱ': // LATIN SMALL V W/ RIGHT HOOK (0x2c71) + case u'ⱳ': // LATIN SMALL W W/ HOOK (0x2c73) + case u'ⱴ': // LATIN SMALL V W/ CURL (0x2c74) + case u'ⱶ': // LATIN SMALL HALF H (0x2c76) + case u'ⱷ': // LATIN SMALL TAILLESS PHI (0x2c77) + case u'ⱸ': // LATIN SMALL E W/ NOTCH (0x2c78) + case u'ⱹ': // LATIN SMALL TURNED R W/ TAIL (0x2c79) + case u'ⱺ': // LATIN SMALL O W/ LOW RING INSIDE (0x2c7a) + case u'ⱻ': // LATIN LETTER SMALL CAPITAL TURNED E (0x2c7b) + case u'ⲁ': // COPTIC SMALL ALFA (0x2c81) + case u'ⲃ': // COPTIC SMALL VIDA (0x2c83) + case u'ⲅ': // COPTIC SMALL GAMMA (0x2c85) + case u'ⲇ': // COPTIC SMALL DALDA (0x2c87) + case u'ⲉ': // COPTIC SMALL EIE (0x2c89) + case u'ⲋ': // COPTIC SMALL SOU (0x2c8b) + case u'ⲍ': // COPTIC SMALL ZATA (0x2c8d) + case u'ⲏ': // COPTIC SMALL HATE (0x2c8f) + case u'ⲑ': // COPTIC SMALL THETHE (0x2c91) + case u'ⲓ': // COPTIC SMALL IAUDA (0x2c93) + case u'ⲕ': // COPTIC SMALL KAPA (0x2c95) + case u'ⲗ': // COPTIC SMALL LAULA (0x2c97) + case u'ⲙ': // COPTIC SMALL MI (0x2c99) + case u'ⲛ': // COPTIC SMALL NI (0x2c9b) + case u'ⲝ': // COPTIC SMALL KSI (0x2c9d) + case u'ⲟ': // COPTIC SMALL O (0x2c9f) + case u'ⲡ': // COPTIC SMALL PI (0x2ca1) + case u'ⲣ': // COPTIC SMALL RO (0x2ca3) + case u'ⲥ': // COPTIC SMALL SIMA (0x2ca5) + case u'ⲧ': // COPTIC SMALL TAU (0x2ca7) + case u'ⲩ': // COPTIC SMALL UA (0x2ca9) + case u'ⲫ': // COPTIC SMALL FI (0x2cab) + case u'ⲭ': // COPTIC SMALL KHI (0x2cad) + case u'ⲯ': // COPTIC SMALL PSI (0x2caf) + case u'ⲱ': // COPTIC SMALL OOU (0x2cb1) + case u'ⲳ': // COPTIC SMALL DIALECT-P ALEF (0x2cb3) + case u'ⲵ': // COPTIC SMALL OLD COPTIC AIN (0x2cb5) + case u'ⲷ': // COPTIC SMALL CRYPTOGRAMMIC EIE (0x2cb7) + case u'ⲹ': // COPTIC SMALL DIALECT-P KAPA (0x2cb9) + case u'ⲻ': // COPTIC SMALL DIALECT-P NI (0x2cbb) + case u'ⲽ': // COPTIC SMALL CRYPTOGRAMMIC NI (0x2cbd) + case u'ⲿ': // COPTIC SMALL OLD COPTIC OOU (0x2cbf) + case u'ⳁ': // COPTIC SMALL SAMPI (0x2cc1) + case u'ⳃ': // COPTIC SMALL CROSSED SHEI (0x2cc3) + case u'ⳅ': // COPTIC SMALL OLD COPTIC SHEI (0x2cc5) + case u'ⳇ': // COPTIC SMALL OLD COPTIC ESH (0x2cc7) + case u'ⳉ': // COPTIC SMALL AKHMIMIC KHEI (0x2cc9) + case u'ⳋ': // COPTIC SMALL DIALECT-P HORI (0x2ccb) + case u'ⳍ': // COPTIC SMALL OLD COPTIC HORI (0x2ccd) + case u'ⳏ': // COPTIC SMALL OLD COPTIC HA (0x2ccf) + case u'ⳑ': // COPTIC SMALL L-SHAPED HA (0x2cd1) + case u'ⳓ': // COPTIC SMALL OLD COPTIC HEI (0x2cd3) + case u'ⳕ': // COPTIC SMALL OLD COPTIC HAT (0x2cd5) + case u'ⳗ': // COPTIC SMALL OLD COPTIC GANGIA (0x2cd7) + case u'ⳙ': // COPTIC SMALL OLD COPTIC DJA (0x2cd9) + case u'ⳛ': // COPTIC SMALL OLD COPTIC SHIMA (0x2cdb) + case u'ⳝ': // COPTIC SMALL OLD NUBIAN SHIMA (0x2cdd) + case u'ⳟ': // COPTIC SMALL OLD NUBIAN NGI (0x2cdf) + case u'ⳡ': // COPTIC SMALL OLD NUBIAN NYI (0x2ce1) + case u'ⳣ': // COPTIC SMALL OLD NUBIAN WAU (0x2ce3) + case u'ⳤ': // COPTIC SYMBOL KAI (0x2ce4) + case u'ⳬ': // COPTIC SMALL CRYPTOGRAMMIC SHEI (0x2cec) + case u'ⳮ': // COPTIC SMALL CRYPTOGRAMMIC GANGIA (0x2cee) + case u'ⳳ': // COPTIC SMALL BOHAIRIC KHEI (0x2cf3) + case u'ⴀ': // GEORGIAN SMALL AN (0x2d00) + case u'ⴁ': // GEORGIAN SMALL BAN (0x2d01) + case u'ⴂ': // GEORGIAN SMALL GAN (0x2d02) + case u'ⴃ': // GEORGIAN SMALL DON (0x2d03) + case u'ⴄ': // GEORGIAN SMALL EN (0x2d04) + case u'ⴅ': // GEORGIAN SMALL VIN (0x2d05) + case u'ⴆ': // GEORGIAN SMALL ZEN (0x2d06) + case u'ⴇ': // GEORGIAN SMALL TAN (0x2d07) + case u'ⴈ': // GEORGIAN SMALL IN (0x2d08) + case u'ⴉ': // GEORGIAN SMALL KAN (0x2d09) + case u'ⴊ': // GEORGIAN SMALL LAS (0x2d0a) + case u'ⴋ': // GEORGIAN SMALL MAN (0x2d0b) + case u'ⴌ': // GEORGIAN SMALL NAR (0x2d0c) + case u'ⴍ': // GEORGIAN SMALL ON (0x2d0d) + case u'ⴎ': // GEORGIAN SMALL PAR (0x2d0e) + case u'ⴏ': // GEORGIAN SMALL ZHAR (0x2d0f) + case u'ⴐ': // GEORGIAN SMALL RAE (0x2d10) + case u'ⴑ': // GEORGIAN SMALL SAN (0x2d11) + case u'ⴒ': // GEORGIAN SMALL TAR (0x2d12) + case u'ⴓ': // GEORGIAN SMALL UN (0x2d13) + case u'ⴔ': // GEORGIAN SMALL PHAR (0x2d14) + case u'ⴕ': // GEORGIAN SMALL KHAR (0x2d15) + case u'ⴖ': // GEORGIAN SMALL GHAN (0x2d16) + case u'ⴗ': // GEORGIAN SMALL QAR (0x2d17) + case u'ⴘ': // GEORGIAN SMALL SHIN (0x2d18) + case u'ⴙ': // GEORGIAN SMALL CHIN (0x2d19) + case u'ⴚ': // GEORGIAN SMALL CAN (0x2d1a) + case u'ⴛ': // GEORGIAN SMALL JIL (0x2d1b) + case u'ⴜ': // GEORGIAN SMALL CIL (0x2d1c) + case u'ⴝ': // GEORGIAN SMALL CHAR (0x2d1d) + case u'ⴞ': // GEORGIAN SMALL XAN (0x2d1e) + case u'ⴟ': // GEORGIAN SMALL JHAN (0x2d1f) + case u'ⴠ': // GEORGIAN SMALL HAE (0x2d20) + case u'ⴡ': // GEORGIAN SMALL HE (0x2d21) + case u'ⴢ': // GEORGIAN SMALL HIE (0x2d22) + case u'ⴣ': // GEORGIAN SMALL WE (0x2d23) + case u'ⴤ': // GEORGIAN SMALL HAR (0x2d24) + case u'ⴥ': // GEORGIAN SMALL HOE (0x2d25) + case u'ⴧ': // GEORGIAN SMALL YN (0x2d27) + case u'ⴭ': // GEORGIAN SMALL AEN (0x2d2d) + case u'ꙁ': // CYRILLIC SMALL ZEMLYA (0xa641) + case u'ꙃ': // CYRILLIC SMALL DZELO (0xa643) + case u'ꙅ': // CYRILLIC SMALL REVERSED DZE (0xa645) + case u'ꙇ': // CYRILLIC SMALL IOTA (0xa647) + case u'ꙉ': // CYRILLIC SMALL DJERV (0xa649) + case u'ꙋ': // CYRILLIC SMALL MONOGRAPH UK (0xa64b) + case u'ꙍ': // CYRILLIC SMALL BROAD OMEGA (0xa64d) + case u'ꙏ': // CYRILLIC SMALL NEUTRAL YER (0xa64f) + case u'ꙑ': // CYRILLIC SMALL YERU W/ BACK YER (0xa651) + case u'ꙓ': // CYRILLIC SMALL IOTIFIED YAT (0xa653) + case u'ꙕ': // CYRILLIC SMALL REVERSED YU (0xa655) + case u'ꙗ': // CYRILLIC SMALL IOTIFIED A (0xa657) + case u'ꙙ': // CYRILLIC SMALL CLOSED LITTLE YUS (0xa659) + case u'ꙛ': // CYRILLIC SMALL BLENDED YUS (0xa65b) + case u'ꙝ': // CYRILLIC SMALL IOTIFIED CLOSED LITTLE YUS (0xa65d) + case u'ꙟ': // CYRILLIC SMALL YN (0xa65f) + case u'ꙡ': // CYRILLIC SMALL REVERSED TSE (0xa661) + case u'ꙣ': // CYRILLIC SMALL SOFT DE (0xa663) + case u'ꙥ': // CYRILLIC SMALL SOFT EL (0xa665) + case u'ꙧ': // CYRILLIC SMALL SOFT EM (0xa667) + case u'ꙩ': // CYRILLIC SMALL MONOCULAR O (0xa669) + case u'ꙫ': // CYRILLIC SMALL BINOCULAR O (0xa66b) + case u'ꙭ': // CYRILLIC SMALL DOUBLE MONOCULAR O (0xa66d) + case 0xa681: // CYRILLIC SMALL DWE (0xa681) + case 0xa683: // CYRILLIC SMALL DZWE (0xa683) + case 0xa685: // CYRILLIC SMALL ZHWE (0xa685) + case 0xa687: // CYRILLIC SMALL CCHE (0xa687) + case 0xa689: // CYRILLIC SMALL DZZE (0xa689) + case 0xa68b: // CYRILLIC SMALL TE W/ MIDDLE HOOK (0xa68b) + case 0xa68d: // CYRILLIC SMALL TWE (0xa68d) + case 0xa68f: // CYRILLIC SMALL TSWE (0xa68f) + case 0xa691: // CYRILLIC SMALL TSSE (0xa691) + case 0xa693: // CYRILLIC SMALL TCHE (0xa693) + case 0xa695: // CYRILLIC SMALL HWE (0xa695) + case 0xa697: // CYRILLIC SMALL SHWE (0xa697) + case 0xa699: // CYRILLIC SMALL DOUBLE O (0xa699) + case 0xa69b: // CYRILLIC SMALL CROSSED O (0xa69b) + case u'ꜣ': // LATIN SMALL EGYPTOLOGICAL ALEF (0xa723) + case u'ꜥ': // LATIN SMALL EGYPTOLOGICAL AIN (0xa725) + case u'ꜧ': // LATIN SMALL HENG (0xa727) + case u'ꜩ': // LATIN SMALL TZ (0xa729) + case u'ꜫ': // LATIN SMALL TRESILLO (0xa72b) + case u'ꜭ': // LATIN SMALL CUATRILLO (0xa72d) + case u'ꜯ': // LATIN SMALL CUATRILLO W/ COMMA (0xa72f) + case u'ꜰ': // LATIN LETTER SMALL CAPITAL F (0xa730) + case u'ꜱ': // LATIN LETTER SMALL CAPITAL S (0xa731) + case u'ꜳ': // LATIN SMALL AA (0xa733) + case u'ꜵ': // LATIN SMALL AO (0xa735) + case u'ꜷ': // LATIN SMALL AU (0xa737) + case u'ꜹ': // LATIN SMALL AV (0xa739) + case u'ꜻ': // LATIN SMALL AV W/ HORIZONTAL BAR (0xa73b) + case u'ꜽ': // LATIN SMALL AY (0xa73d) + case u'ꜿ': // LATIN SMALL REVERSED C W/ DOT (0xa73f) + case u'ꝁ': // LATIN SMALL K W/ STROKE (0xa741) + case u'ꝃ': // LATIN SMALL K W/ DIAGONAL STROKE (0xa743) + case u'ꝅ': // LATIN SMALL K W/ STROKE AND DIAGONAL STROKE (0xa745) + case u'ꝇ': // LATIN SMALL BROKEN L (0xa747) + case u'ꝉ': // LATIN SMALL L W/ HIGH STROKE (0xa749) + case u'ꝋ': // LATIN SMALL O W/ LONG STROKE OVERLAY (0xa74b) + case u'ꝍ': // LATIN SMALL O W/ LOOP (0xa74d) + case u'ꝏ': // LATIN SMALL OO (0xa74f) + case u'ꝑ': // LATIN SMALL P W/ STROKE THROUGH DESCENDER (0xa751) + case u'ꝓ': // LATIN SMALL P W/ FLOURISH (0xa753) + case u'ꝕ': // LATIN SMALL P W/ SQUIRREL TAIL (0xa755) + case u'ꝗ': // LATIN SMALL Q W/ STROKE THROUGH DESCENDER (0xa757) + case u'ꝙ': // LATIN SMALL Q W/ DIAGONAL STROKE (0xa759) + case u'ꝛ': // LATIN SMALL R ROTUNDA (0xa75b) + case u'ꝝ': // LATIN SMALL RUM ROTUNDA (0xa75d) + case u'ꝟ': // LATIN SMALL V W/ DIAGONAL STROKE (0xa75f) + case u'ꝡ': // LATIN SMALL VY (0xa761) + case u'ꝣ': // LATIN SMALL VISIGOTHIC Z (0xa763) + case u'ꝥ': // LATIN SMALL THORN W/ STROKE (0xa765) + case u'ꝧ': // LATIN SMALL THORN W/ STROKE THROUGH DESCENDER (0xa767) + case u'ꝩ': // LATIN SMALL VEND (0xa769) + case u'ꝫ': // LATIN SMALL ET (0xa76b) + case u'ꝭ': // LATIN SMALL IS (0xa76d) + case u'ꝯ': // LATIN SMALL CON (0xa76f) + case u'ꝱ': // LATIN SMALL DUM (0xa771) + case u'ꝲ': // LATIN SMALL LUM (0xa772) + case u'ꝳ': // LATIN SMALL MUM (0xa773) + case u'ꝴ': // LATIN SMALL NUM (0xa774) + case u'ꝵ': // LATIN SMALL RUM (0xa775) + case u'ꝶ': // LATIN LETTER SMALL CAPITAL RUM (0xa776) + case u'ꝷ': // LATIN SMALL TUM (0xa777) + case u'ꝸ': // LATIN SMALL UM (0xa778) + case u'ꝺ': // LATIN SMALL INSULAR D (0xa77a) + case u'ꝼ': // LATIN SMALL INSULAR F (0xa77c) + case u'ꝿ': // LATIN SMALL TURNED INSULAR G (0xa77f) + case u'ꞁ': // LATIN SMALL TURNED L (0xa781) + case u'ꞃ': // LATIN SMALL INSULAR R (0xa783) + case u'ꞅ': // LATIN SMALL INSULAR S (0xa785) + case u'ꞇ': // LATIN SMALL INSULAR T (0xa787) + case u'ꞌ': // LATIN SMALL SALTILLO (0xa78c) + case u'ꞎ': // LATIN SMALL L W/ RETROFLEX HOOK AND BELT (0xa78e) + case u'ꞑ': // LATIN SMALL N W/ DESCENDER (0xa791) + case u'ꞓ': // LATIN SMALL C W/ BAR (0xa793) + case u'ꞔ': // LATIN SMALL C W/ PALATAL HOOK (0xa794) + case u'ꞕ': // LATIN SMALL H W/ PALATAL HOOK (0xa795) + case u'ꞗ': // LATIN SMALL B W/ FLOURISH (0xa797) + case u'ꞙ': // LATIN SMALL F W/ STROKE (0xa799) + case u'ꞛ': // LATIN SMALL VOLAPUK AE (0xa79b) + case u'ꞝ': // LATIN SMALL VOLAPUK OE (0xa79d) + case u'ꞟ': // LATIN SMALL VOLAPUK UE (0xa79f) + case u'ꞡ': // LATIN SMALL G W/ OBLIQUE STROKE (0xa7a1) + case u'ꞣ': // LATIN SMALL K W/ OBLIQUE STROKE (0xa7a3) + case u'ꞥ': // LATIN SMALL N W/ OBLIQUE STROKE (0xa7a5) + case u'ꞧ': // LATIN SMALL R W/ OBLIQUE STROKE (0xa7a7) + case u'ꞩ': // LATIN SMALL S W/ OBLIQUE STROKE (0xa7a9) + case u'ꞯ': // LATIN LETTER SMALL CAPITAL Q (0xa7af) + case u'ꞵ': // LATIN SMALL BETA (0xa7b5) + case u'ꞷ': // LATIN SMALL OMEGA (0xa7b7) + case u'ꞹ': // LATIN SMALL U W/ STROKE (0xa7b9) + case u'ꞻ': // LATIN SMALL GLOTTAL A (0xa7bb) + case u'ꞽ': // LATIN SMALL GLOTTAL I (0xa7bd) + case u'ꞿ': // LATIN SMALL GLOTTAL U (0xa7bf) + case u'ꟃ': // LATIN SMALL ANGLICANA W (0xa7c3) + case u'ꟺ': // LATIN LETTER SMALL CAPITAL TURNED M (0xa7fa) + case u'ꬰ': // LATIN SMALL BARRED ALPHA (0xab30) + case u'ꬱ': // LATIN SMALL A REVERSED-SCHWA (0xab31) + case u'ꬲ': // LATIN SMALL BLACKLETTER E (0xab32) + case u'ꬳ': // LATIN SMALL BARRED E (0xab33) + case u'ꬴ': // LATIN SMALL E W/ FLOURISH (0xab34) + case u'ꬵ': // LATIN SMALL LENIS F (0xab35) + case u'ꬶ': // LATIN SMALL SCRIPT G W/ CROSSED-TAIL (0xab36) + case u'ꬷ': // LATIN SMALL L W/ INVERTED LAZY S (0xab37) + case u'ꬸ': // LATIN SMALL L W/ DOUBLE MIDDLE TILDE (0xab38) + case u'ꬹ': // LATIN SMALL L W/ MIDDLE RING (0xab39) + case u'ꬺ': // LATIN SMALL M W/ CROSSED-TAIL (0xab3a) + case u'ꬻ': // LATIN SMALL N W/ CROSSED-TAIL (0xab3b) + case u'ꬼ': // LATIN SMALL ENG W/ CROSSED-TAIL (0xab3c) + case u'ꬽ': // LATIN SMALL BLACKLETTER O (0xab3d) + case u'ꬾ': // LATIN SMALL BLACKLETTER O W/ STROKE (0xab3e) + case u'ꬿ': // LATIN SMALL OPEN O W/ STROKE (0xab3f) + case u'ꭀ': // LATIN SMALL INVERTED OE (0xab40) + case u'ꭁ': // LATIN SMALL TURNED OE W/ STROKE (0xab41) + case u'ꭂ': // LATIN SMALL TURNED OE W/ HORIZONTAL STROKE (0xab42) + case u'ꭃ': // LATIN SMALL TURNED O OPEN-O (0xab43) + case u'ꭄ': // LATIN SMALL TURNED O OPEN-O W/ STROKE (0xab44) + case u'ꭅ': // LATIN SMALL STIRRUP R (0xab45) + case u'ꭆ': // LATIN LETTER SMALL CAPITAL R W/ RIGHT LEG (0xab46) + case u'ꭇ': // LATIN SMALL R WITHOUT HANDLE (0xab47) + case u'ꭈ': // LATIN SMALL DOUBLE R (0xab48) + case u'ꭉ': // LATIN SMALL R W/ CROSSED-TAIL (0xab49) + case u'ꭊ': // LATIN SMALL DOUBLE R W/ CROSSED-TAIL (0xab4a) + case u'ꭋ': // LATIN SMALL SCRIPT R (0xab4b) + case u'ꭌ': // LATIN SMALL SCRIPT R W/ RING (0xab4c) + case u'ꭍ': // LATIN SMALL BASELINE ESH (0xab4d) + case u'ꭎ': // LATIN SMALL U W/ SHORT RIGHT LEG (0xab4e) + case u'ꭏ': // LATIN SMALL U BAR W/ SHORT RIGHT LEG (0xab4f) + case u'ꭐ': // LATIN SMALL UI (0xab50) + case u'ꭑ': // LATIN SMALL TURNED UI (0xab51) + case u'ꭒ': // LATIN SMALL U W/ LEFT HOOK (0xab52) + case u'ꭓ': // LATIN SMALL CHI (0xab53) + case u'ꭔ': // LATIN SMALL CHI W/ LOW RIGHT RING (0xab54) + case u'ꭕ': // LATIN SMALL CHI W/ LOW LEFT SERIF (0xab55) + case u'ꭖ': // LATIN SMALL X W/ LOW RIGHT RING (0xab56) + case u'ꭗ': // LATIN SMALL X W/ LONG LEFT LEG (0xab57) + case u'ꭘ': // LATIN SMALL X W/ LONG LEFT LEG AND LOW RIGHT RING (0xab58) + case u'ꭙ': // LATIN SMALL X W/ LONG LEFT LEG W/ SERIF (0xab59) + case u'ꭚ': // LATIN SMALL Y W/ SHORT RIGHT LEG (0xab5a) + case u'ꭠ': // LATIN SMALL SAKHA YAT (0xab60) + case u'ꭡ': // LATIN SMALL IOTIFIED E (0xab61) + case u'ꭢ': // LATIN SMALL OPEN OE (0xab62) + case u'ꭣ': // LATIN SMALL UO (0xab63) + case u'ꭤ': // LATIN SMALL INVERTED ALPHA (0xab64) + case u'ꭥ': // GREEK LETTER SMALL CAPITAL OMEGA (0xab65) + case u'ꭦ': // LATIN SMALL DZ DIGRAPH W/ RETROFLEX HOOK (0xab66) + case u'ꭧ': // LATIN SMALL TS DIGRAPH W/ RETROFLEX HOOK (0xab67) + case u'ꭰ': // CHEROKEE SMALL A (0xab70) + case u'ꭱ': // CHEROKEE SMALL E (0xab71) + case u'ꭲ': // CHEROKEE SMALL I (0xab72) + case u'ꭳ': // CHEROKEE SMALL O (0xab73) + case u'ꭴ': // CHEROKEE SMALL U (0xab74) + case u'ꭵ': // CHEROKEE SMALL V (0xab75) + case u'ꭶ': // CHEROKEE SMALL GA (0xab76) + case u'ꭷ': // CHEROKEE SMALL KA (0xab77) + case u'ꭸ': // CHEROKEE SMALL GE (0xab78) + case u'ꭹ': // CHEROKEE SMALL GI (0xab79) + case u'ꭺ': // CHEROKEE SMALL GO (0xab7a) + case u'ꭻ': // CHEROKEE SMALL GU (0xab7b) + case u'ꭼ': // CHEROKEE SMALL GV (0xab7c) + case u'ꭽ': // CHEROKEE SMALL HA (0xab7d) + case u'ꭾ': // CHEROKEE SMALL HE (0xab7e) + case u'ꭿ': // CHEROKEE SMALL HI (0xab7f) + case u'ꮀ': // CHEROKEE SMALL HO (0xab80) + case u'ꮁ': // CHEROKEE SMALL HU (0xab81) + case u'ꮂ': // CHEROKEE SMALL HV (0xab82) + case u'ꮃ': // CHEROKEE SMALL LA (0xab83) + case u'ꮄ': // CHEROKEE SMALL LE (0xab84) + case u'ꮅ': // CHEROKEE SMALL LI (0xab85) + case u'ꮆ': // CHEROKEE SMALL LO (0xab86) + case u'ꮇ': // CHEROKEE SMALL LU (0xab87) + case u'ꮈ': // CHEROKEE SMALL LV (0xab88) + case u'ꮉ': // CHEROKEE SMALL MA (0xab89) + case u'ꮊ': // CHEROKEE SMALL ME (0xab8a) + case u'ꮋ': // CHEROKEE SMALL MI (0xab8b) + case u'ꮌ': // CHEROKEE SMALL MO (0xab8c) + case u'ꮍ': // CHEROKEE SMALL MU (0xab8d) + case u'ꮎ': // CHEROKEE SMALL NA (0xab8e) + case u'ꮏ': // CHEROKEE SMALL HNA (0xab8f) + case u'ꮐ': // CHEROKEE SMALL NAH (0xab90) + case u'ꮑ': // CHEROKEE SMALL NE (0xab91) + case u'ꮒ': // CHEROKEE SMALL NI (0xab92) + case u'ꮓ': // CHEROKEE SMALL NO (0xab93) + case u'ꮔ': // CHEROKEE SMALL NU (0xab94) + case u'ꮕ': // CHEROKEE SMALL NV (0xab95) + case u'ꮖ': // CHEROKEE SMALL QUA (0xab96) + case u'ꮗ': // CHEROKEE SMALL QUE (0xab97) + case u'ꮘ': // CHEROKEE SMALL QUI (0xab98) + case u'ꮙ': // CHEROKEE SMALL QUO (0xab99) + case u'ꮚ': // CHEROKEE SMALL QUU (0xab9a) + case u'ꮛ': // CHEROKEE SMALL QUV (0xab9b) + case u'ꮜ': // CHEROKEE SMALL SA (0xab9c) + case u'ꮝ': // CHEROKEE SMALL S (0xab9d) + case u'ꮞ': // CHEROKEE SMALL SE (0xab9e) + case u'ꮟ': // CHEROKEE SMALL SI (0xab9f) + case u'ꮠ': // CHEROKEE SMALL SO (0xaba0) + case u'ꮡ': // CHEROKEE SMALL SU (0xaba1) + case u'ꮢ': // CHEROKEE SMALL SV (0xaba2) + case u'ꮣ': // CHEROKEE SMALL DA (0xaba3) + case u'ꮤ': // CHEROKEE SMALL TA (0xaba4) + case u'ꮥ': // CHEROKEE SMALL DE (0xaba5) + case u'ꮦ': // CHEROKEE SMALL TE (0xaba6) + case u'ꮧ': // CHEROKEE SMALL DI (0xaba7) + case u'ꮨ': // CHEROKEE SMALL TI (0xaba8) + case u'ꮩ': // CHEROKEE SMALL DO (0xaba9) + case u'ꮪ': // CHEROKEE SMALL DU (0xabaa) + case u'ꮫ': // CHEROKEE SMALL DV (0xabab) + case u'ꮬ': // CHEROKEE SMALL DLA (0xabac) + case u'ꮭ': // CHEROKEE SMALL TLA (0xabad) + case u'ꮮ': // CHEROKEE SMALL TLE (0xabae) + case u'ꮯ': // CHEROKEE SMALL TLI (0xabaf) + case u'ꮰ': // CHEROKEE SMALL TLO (0xabb0) + case u'ꮱ': // CHEROKEE SMALL TLU (0xabb1) + case u'ꮲ': // CHEROKEE SMALL TLV (0xabb2) + case u'ꮳ': // CHEROKEE SMALL TSA (0xabb3) + case u'ꮴ': // CHEROKEE SMALL TSE (0xabb4) + case u'ꮵ': // CHEROKEE SMALL TSI (0xabb5) + case u'ꮶ': // CHEROKEE SMALL TSO (0xabb6) + case u'ꮷ': // CHEROKEE SMALL TSU (0xabb7) + case u'ꮸ': // CHEROKEE SMALL TSV (0xabb8) + case u'ꮹ': // CHEROKEE SMALL WA (0xabb9) + case u'ꮺ': // CHEROKEE SMALL WE (0xabba) + case u'ꮻ': // CHEROKEE SMALL WI (0xabbb) + case u'ꮼ': // CHEROKEE SMALL WO (0xabbc) + case u'ꮽ': // CHEROKEE SMALL WU (0xabbd) + case u'ꮾ': // CHEROKEE SMALL WV (0xabbe) + case u'ꮿ': // CHEROKEE SMALL YA (0xabbf) + case u'ff': // LATIN SMALL LIGATURE FF (0xfb00) + case u'fi': // LATIN SMALL LIGATURE FI (0xfb01) + case u'fl': // LATIN SMALL LIGATURE FL (0xfb02) + case u'ffi': // LATIN SMALL LIGATURE FFI (0xfb03) + case u'ffl': // LATIN SMALL LIGATURE FFL (0xfb04) + case u'ſt': // LATIN SMALL LIGATURE LONG S T (0xfb05) + case u'st': // LATIN SMALL LIGATURE ST (0xfb06) + case u'ﬓ': // ARMENIAN SMALL LIGATURE MEN NOW (0xfb13) + case u'ﬔ': // ARMENIAN SMALL LIGATURE MEN ECH (0xfb14) + case u'ﬕ': // ARMENIAN SMALL LIGATURE MEN INI (0xfb15) + case u'ﬖ': // ARMENIAN SMALL LIGATURE VEW NOW (0xfb16) + case u'ﬗ': // ARMENIAN SMALL LIGATURE MEN XEH (0xfb17) + case u'ａ': // FULLWIDTH LATIN SMALL A (0xff41) + case u'ｂ': // FULLWIDTH LATIN SMALL B (0xff42) + case u'ｃ': // FULLWIDTH LATIN SMALL C (0xff43) + case u'ｄ': // FULLWIDTH LATIN SMALL D (0xff44) + case u'ｅ': // FULLWIDTH LATIN SMALL E (0xff45) + case u'ｆ': // FULLWIDTH LATIN SMALL F (0xff46) + case u'ｇ': // FULLWIDTH LATIN SMALL G (0xff47) + case u'ｈ': // FULLWIDTH LATIN SMALL H (0xff48) + case u'ｉ': // FULLWIDTH LATIN SMALL I (0xff49) + case u'ｊ': // FULLWIDTH LATIN SMALL J (0xff4a) + case u'ｋ': // FULLWIDTH LATIN SMALL K (0xff4b) + case u'ｌ': // FULLWIDTH LATIN SMALL L (0xff4c) + case u'ｍ': // FULLWIDTH LATIN SMALL M (0xff4d) + case u'ｎ': // FULLWIDTH LATIN SMALL N (0xff4e) + case u'ｏ': // FULLWIDTH LATIN SMALL O (0xff4f) + case u'ｐ': // FULLWIDTH LATIN SMALL P (0xff50) + case u'ｑ': // FULLWIDTH LATIN SMALL Q (0xff51) + case u'ｒ': // FULLWIDTH LATIN SMALL R (0xff52) + case u'ｓ': // FULLWIDTH LATIN SMALL S (0xff53) + case u'ｔ': // FULLWIDTH LATIN SMALL T (0xff54) + case u'ｕ': // FULLWIDTH LATIN SMALL U (0xff55) + case u'ｖ': // FULLWIDTH LATIN SMALL V (0xff56) + case u'ｗ': // FULLWIDTH LATIN SMALL W (0xff57) + case u'ｘ': // FULLWIDTH LATIN SMALL X (0xff58) + case u'ｙ': // FULLWIDTH LATIN SMALL Y (0xff59) + case u'ｚ': // FULLWIDTH LATIN SMALL Z (0xff5a) + case L'𐐨': // DESERET SMALL LONG I (0x10428) + case L'𐐩': // DESERET SMALL LONG E (0x10429) + case L'𐐪': // DESERET SMALL LONG A (0x1042a) + case L'𐐫': // DESERET SMALL LONG AH (0x1042b) + case L'𐐬': // DESERET SMALL LONG O (0x1042c) + case L'𐐭': // DESERET SMALL LONG OO (0x1042d) + case L'𐐮': // DESERET SMALL SHORT I (0x1042e) + case L'𐐯': // DESERET SMALL SHORT E (0x1042f) + case L'𐐰': // DESERET SMALL SHORT A (0x10430) + case L'𐐱': // DESERET SMALL SHORT AH (0x10431) + case L'𐐲': // DESERET SMALL SHORT O (0x10432) + case L'𐐳': // DESERET SMALL SHORT OO (0x10433) + case L'𐐴': // DESERET SMALL AY (0x10434) + case L'𐐵': // DESERET SMALL OW (0x10435) + case L'𐐶': // DESERET SMALL WU (0x10436) + case L'𐐷': // DESERET SMALL YEE (0x10437) + case L'𐐸': // DESERET SMALL H (0x10438) + case L'𐐹': // DESERET SMALL PEE (0x10439) + case L'𐐺': // DESERET SMALL BEE (0x1043a) + case L'𐐻': // DESERET SMALL TEE (0x1043b) + case L'𐐼': // DESERET SMALL DEE (0x1043c) + case L'𐐽': // DESERET SMALL CHEE (0x1043d) + case L'𐐾': // DESERET SMALL JEE (0x1043e) + case L'𐐿': // DESERET SMALL KAY (0x1043f) + case L'𐑀': // DESERET SMALL GAY (0x10440) + case L'𐑁': // DESERET SMALL EF (0x10441) + case L'𐑂': // DESERET SMALL VEE (0x10442) + case L'𐑃': // DESERET SMALL ETH (0x10443) + case L'𐑄': // DESERET SMALL THEE (0x10444) + case L'𐑅': // DESERET SMALL ES (0x10445) + case L'𐑆': // DESERET SMALL ZEE (0x10446) + case L'𐑇': // DESERET SMALL ESH (0x10447) + case L'𐑈': // DESERET SMALL ZHEE (0x10448) + case L'𐑉': // DESERET SMALL ER (0x10449) + case L'𐑊': // DESERET SMALL EL (0x1044a) + case L'𐑋': // DESERET SMALL EM (0x1044b) + case L'𐑌': // DESERET SMALL EN (0x1044c) + case L'𐑍': // DESERET SMALL ENG (0x1044d) + case L'𐑎': // DESERET SMALL OI (0x1044e) + case L'𐑏': // DESERET SMALL EW (0x1044f) + case L'𐓘': // OSAGE SMALL A (0x104d8) + case L'𐓙': // OSAGE SMALL AI (0x104d9) + case L'𐓚': // OSAGE SMALL AIN (0x104da) + case L'𐓛': // OSAGE SMALL AH (0x104db) + case L'𐓜': // OSAGE SMALL BRA (0x104dc) + case L'𐓝': // OSAGE SMALL CHA (0x104dd) + case L'𐓞': // OSAGE SMALL EHCHA (0x104de) + case L'𐓟': // OSAGE SMALL E (0x104df) + case L'𐓠': // OSAGE SMALL EIN (0x104e0) + case L'𐓡': // OSAGE SMALL HA (0x104e1) + case L'𐓢': // OSAGE SMALL HYA (0x104e2) + case L'𐓣': // OSAGE SMALL I (0x104e3) + case L'𐓤': // OSAGE SMALL KA (0x104e4) + case L'𐓥': // OSAGE SMALL EHKA (0x104e5) + case L'𐓦': // OSAGE SMALL KYA (0x104e6) + case L'𐓧': // OSAGE SMALL LA (0x104e7) + case L'𐓨': // OSAGE SMALL MA (0x104e8) + case L'𐓩': // OSAGE SMALL NA (0x104e9) + case L'𐓪': // OSAGE SMALL O (0x104ea) + case L'𐓫': // OSAGE SMALL OIN (0x104eb) + case L'𐓬': // OSAGE SMALL PA (0x104ec) + case L'𐓭': // OSAGE SMALL EHPA (0x104ed) + case L'𐓮': // OSAGE SMALL SA (0x104ee) + case L'𐓯': // OSAGE SMALL SHA (0x104ef) + case L'𐓰': // OSAGE SMALL TA (0x104f0) + case L'𐓱': // OSAGE SMALL EHTA (0x104f1) + case L'𐓲': // OSAGE SMALL TSA (0x104f2) + case L'𐓳': // OSAGE SMALL EHTSA (0x104f3) + case L'𐓴': // OSAGE SMALL TSHA (0x104f4) + case L'𐓵': // OSAGE SMALL DHA (0x104f5) + case L'𐓶': // OSAGE SMALL U (0x104f6) + case L'𐓷': // OSAGE SMALL WA (0x104f7) + case L'𐓸': // OSAGE SMALL KHA (0x104f8) + case L'𐓹': // OSAGE SMALL GHA (0x104f9) + case L'𐓺': // OSAGE SMALL ZA (0x104fa) + case L'𐓻': // OSAGE SMALL ZHA (0x104fb) + case L'𐳀': // OLD HUNGARIAN SMALL A (0x10cc0) + case L'𐳁': // OLD HUNGARIAN SMALL AA (0x10cc1) + case L'𐳂': // OLD HUNGARIAN SMALL EB (0x10cc2) + case L'𐳃': // OLD HUNGARIAN SMALL AMB (0x10cc3) + case L'𐳄': // OLD HUNGARIAN SMALL EC (0x10cc4) + case L'𐳅': // OLD HUNGARIAN SMALL ENC (0x10cc5) + case L'𐳆': // OLD HUNGARIAN SMALL ECS (0x10cc6) + case L'𐳇': // OLD HUNGARIAN SMALL ED (0x10cc7) + case L'𐳈': // OLD HUNGARIAN SMALL AND (0x10cc8) + case L'𐳉': // OLD HUNGARIAN SMALL E (0x10cc9) + case L'𐳊': // OLD HUNGARIAN SMALL CLOSE E (0x10cca) + case L'𐳋': // OLD HUNGARIAN SMALL EE (0x10ccb) + case L'𐳌': // OLD HUNGARIAN SMALL EF (0x10ccc) + case L'𐳍': // OLD HUNGARIAN SMALL EG (0x10ccd) + case L'𐳎': // OLD HUNGARIAN SMALL EGY (0x10cce) + case L'𐳏': // OLD HUNGARIAN SMALL EH (0x10ccf) + case L'𐳐': // OLD HUNGARIAN SMALL I (0x10cd0) + case L'𐳑': // OLD HUNGARIAN SMALL II (0x10cd1) + case L'𐳒': // OLD HUNGARIAN SMALL EJ (0x10cd2) + case L'𐳓': // OLD HUNGARIAN SMALL EK (0x10cd3) + case L'𐳔': // OLD HUNGARIAN SMALL AK (0x10cd4) + case L'𐳕': // OLD HUNGARIAN SMALL UNK (0x10cd5) + case L'𐳖': // OLD HUNGARIAN SMALL EL (0x10cd6) + case L'𐳗': // OLD HUNGARIAN SMALL ELY (0x10cd7) + case L'𐳘': // OLD HUNGARIAN SMALL EM (0x10cd8) + case L'𐳙': // OLD HUNGARIAN SMALL EN (0x10cd9) + case L'𐳚': // OLD HUNGARIAN SMALL ENY (0x10cda) + case L'𐳛': // OLD HUNGARIAN SMALL O (0x10cdb) + case L'𐳜': // OLD HUNGARIAN SMALL OO (0x10cdc) + case L'𐳝': // OLD HUNGARIAN SMALL NIKOLSBURG OE (0x10cdd) + case L'𐳞': // OLD HUNGARIAN SMALL RUDIMENTA OE (0x10cde) + case L'𐳟': // OLD HUNGARIAN SMALL OEE (0x10cdf) + case L'𐳠': // OLD HUNGARIAN SMALL EP (0x10ce0) + case L'𐳡': // OLD HUNGARIAN SMALL EMP (0x10ce1) + case L'𐳢': // OLD HUNGARIAN SMALL ER (0x10ce2) + case L'𐳣': // OLD HUNGARIAN SMALL SHORT ER (0x10ce3) + case L'𐳤': // OLD HUNGARIAN SMALL ES (0x10ce4) + case L'𐳥': // OLD HUNGARIAN SMALL ESZ (0x10ce5) + case L'𐳦': // OLD HUNGARIAN SMALL ET (0x10ce6) + case L'𐳧': // OLD HUNGARIAN SMALL ENT (0x10ce7) + case L'𐳨': // OLD HUNGARIAN SMALL ETY (0x10ce8) + case L'𐳩': // OLD HUNGARIAN SMALL ECH (0x10ce9) + case L'𐳪': // OLD HUNGARIAN SMALL U (0x10cea) + case L'𐳫': // OLD HUNGARIAN SMALL UU (0x10ceb) + case L'𐳬': // OLD HUNGARIAN SMALL NIKOLSBURG UE (0x10cec) + case L'𐳭': // OLD HUNGARIAN SMALL RUDIMENTA UE (0x10ced) + case L'𐳮': // OLD HUNGARIAN SMALL EV (0x10cee) + case L'𐳯': // OLD HUNGARIAN SMALL EZ (0x10cef) + case L'𐳰': // OLD HUNGARIAN SMALL EZS (0x10cf0) + case L'𐳱': // OLD HUNGARIAN SMALL ENT-SHAPED SIGN (0x10cf1) + case L'𐳲': // OLD HUNGARIAN SMALL US (0x10cf2) + case L'𑣀': // WARANG CITI SMALL NGAA (0x118c0) + case L'𑣁': // WARANG CITI SMALL A (0x118c1) + case L'𑣂': // WARANG CITI SMALL WI (0x118c2) + case L'𑣃': // WARANG CITI SMALL YU (0x118c3) + case L'𑣄': // WARANG CITI SMALL YA (0x118c4) + case L'𑣅': // WARANG CITI SMALL YO (0x118c5) + case L'𑣆': // WARANG CITI SMALL II (0x118c6) + case L'𑣇': // WARANG CITI SMALL UU (0x118c7) + case L'𑣈': // WARANG CITI SMALL E (0x118c8) + case L'𑣉': // WARANG CITI SMALL O (0x118c9) + case L'𑣊': // WARANG CITI SMALL ANG (0x118ca) + case L'𑣋': // WARANG CITI SMALL GA (0x118cb) + case L'𑣌': // WARANG CITI SMALL KO (0x118cc) + case L'𑣍': // WARANG CITI SMALL ENY (0x118cd) + case L'𑣎': // WARANG CITI SMALL YUJ (0x118ce) + case L'𑣏': // WARANG CITI SMALL UC (0x118cf) + case L'𑣐': // WARANG CITI SMALL ENN (0x118d0) + case L'𑣑': // WARANG CITI SMALL ODD (0x118d1) + case L'𑣒': // WARANG CITI SMALL TTE (0x118d2) + case L'𑣓': // WARANG CITI SMALL NUNG (0x118d3) + case L'𑣔': // WARANG CITI SMALL DA (0x118d4) + case L'𑣕': // WARANG CITI SMALL AT (0x118d5) + case L'𑣖': // WARANG CITI SMALL AM (0x118d6) + case L'𑣗': // WARANG CITI SMALL BU (0x118d7) + case L'𑣘': // WARANG CITI SMALL PU (0x118d8) + case L'𑣙': // WARANG CITI SMALL HIYO (0x118d9) + case L'𑣚': // WARANG CITI SMALL HOLO (0x118da) + case L'𑣛': // WARANG CITI SMALL HORR (0x118db) + case L'𑣜': // WARANG CITI SMALL HAR (0x118dc) + case L'𑣝': // WARANG CITI SMALL SSUU (0x118dd) + case L'𑣞': // WARANG CITI SMALL SII (0x118de) + case L'𑣟': // WARANG CITI SMALL VIYO (0x118df) + case L'𖹠': // MEDEFAIDRIN SMALL M (0x16e60) + case L'𖹡': // MEDEFAIDRIN SMALL S (0x16e61) + case L'𖹢': // MEDEFAIDRIN SMALL V (0x16e62) + case L'𖹣': // MEDEFAIDRIN SMALL W (0x16e63) + case L'𖹤': // MEDEFAIDRIN SMALL ATIU (0x16e64) + case L'𖹥': // MEDEFAIDRIN SMALL Z (0x16e65) + case L'𖹦': // MEDEFAIDRIN SMALL KP (0x16e66) + case L'𖹧': // MEDEFAIDRIN SMALL P (0x16e67) + case L'𖹨': // MEDEFAIDRIN SMALL T (0x16e68) + case L'𖹩': // MEDEFAIDRIN SMALL G (0x16e69) + case L'𖹪': // MEDEFAIDRIN SMALL F (0x16e6a) + case L'𖹫': // MEDEFAIDRIN SMALL I (0x16e6b) + case L'𖹬': // MEDEFAIDRIN SMALL K (0x16e6c) + case L'𖹭': // MEDEFAIDRIN SMALL A (0x16e6d) + case L'𖹮': // MEDEFAIDRIN SMALL J (0x16e6e) + case L'𖹯': // MEDEFAIDRIN SMALL E (0x16e6f) + case L'𖹰': // MEDEFAIDRIN SMALL B (0x16e70) + case L'𖹱': // MEDEFAIDRIN SMALL C (0x16e71) + case L'𖹲': // MEDEFAIDRIN SMALL U (0x16e72) + case L'𖹳': // MEDEFAIDRIN SMALL YU (0x16e73) + case L'𖹴': // MEDEFAIDRIN SMALL L (0x16e74) + case L'𖹵': // MEDEFAIDRIN SMALL Q (0x16e75) + case L'𖹶': // MEDEFAIDRIN SMALL HP (0x16e76) + case L'𖹷': // MEDEFAIDRIN SMALL NY (0x16e77) + case L'𖹸': // MEDEFAIDRIN SMALL X (0x16e78) + case L'𖹹': // MEDEFAIDRIN SMALL D (0x16e79) + case L'𖹺': // MEDEFAIDRIN SMALL OE (0x16e7a) + case L'𖹻': // MEDEFAIDRIN SMALL N (0x16e7b) + case L'𖹼': // MEDEFAIDRIN SMALL R (0x16e7c) + case L'𖹽': // MEDEFAIDRIN SMALL O (0x16e7d) + case L'𖹾': // MEDEFAIDRIN SMALL AI (0x16e7e) + case L'𖹿': // MEDEFAIDRIN SMALL Y (0x16e7f) + case L'𝐚': // MATHEMATICAL BOLD SMALL A (0x1d41a) + case L'𝐛': // MATHEMATICAL BOLD SMALL B (0x1d41b) + case L'𝐜': // MATHEMATICAL BOLD SMALL C (0x1d41c) + case L'𝐝': // MATHEMATICAL BOLD SMALL D (0x1d41d) + case L'𝐞': // MATHEMATICAL BOLD SMALL E (0x1d41e) + case L'𝐟': // MATHEMATICAL BOLD SMALL F (0x1d41f) + case L'𝐠': // MATHEMATICAL BOLD SMALL G (0x1d420) + case L'𝐡': // MATHEMATICAL BOLD SMALL H (0x1d421) + case L'𝐢': // MATHEMATICAL BOLD SMALL I (0x1d422) + case L'𝐣': // MATHEMATICAL BOLD SMALL J (0x1d423) + case L'𝐤': // MATHEMATICAL BOLD SMALL K (0x1d424) + case L'𝐥': // MATHEMATICAL BOLD SMALL L (0x1d425) + case L'𝐦': // MATHEMATICAL BOLD SMALL M (0x1d426) + case L'𝐧': // MATHEMATICAL BOLD SMALL N (0x1d427) + case L'𝐨': // MATHEMATICAL BOLD SMALL O (0x1d428) + case L'𝐩': // MATHEMATICAL BOLD SMALL P (0x1d429) + case L'𝐪': // MATHEMATICAL BOLD SMALL Q (0x1d42a) + case L'𝐫': // MATHEMATICAL BOLD SMALL R (0x1d42b) + case L'𝐬': // MATHEMATICAL BOLD SMALL S (0x1d42c) + case L'𝐭': // MATHEMATICAL BOLD SMALL T (0x1d42d) + case L'𝐮': // MATHEMATICAL BOLD SMALL U (0x1d42e) + case L'𝐯': // MATHEMATICAL BOLD SMALL V (0x1d42f) + case L'𝐰': // MATHEMATICAL BOLD SMALL W (0x1d430) + case L'𝐱': // MATHEMATICAL BOLD SMALL X (0x1d431) + case L'𝐲': // MATHEMATICAL BOLD SMALL Y (0x1d432) + case L'𝐳': // MATHEMATICAL BOLD SMALL Z (0x1d433) + case L'𝑎': // MATHEMATICAL ITALIC SMALL A (0x1d44e) + case L'𝑏': // MATHEMATICAL ITALIC SMALL B (0x1d44f) + case L'𝑐': // MATHEMATICAL ITALIC SMALL C (0x1d450) + case L'𝑑': // MATHEMATICAL ITALIC SMALL D (0x1d451) + case L'𝑒': // MATHEMATICAL ITALIC SMALL E (0x1d452) + case L'𝑓': // MATHEMATICAL ITALIC SMALL F (0x1d453) + case L'𝑔': // MATHEMATICAL ITALIC SMALL G (0x1d454) + case L'𝑖': // MATHEMATICAL ITALIC SMALL I (0x1d456) + case L'𝑗': // MATHEMATICAL ITALIC SMALL J (0x1d457) + case L'𝑘': // MATHEMATICAL ITALIC SMALL K (0x1d458) + case L'𝑙': // MATHEMATICAL ITALIC SMALL L (0x1d459) + case L'𝑚': // MATHEMATICAL ITALIC SMALL M (0x1d45a) + case L'𝑛': // MATHEMATICAL ITALIC SMALL N (0x1d45b) + case L'𝑜': // MATHEMATICAL ITALIC SMALL O (0x1d45c) + case L'𝑝': // MATHEMATICAL ITALIC SMALL P (0x1d45d) + case L'𝑞': // MATHEMATICAL ITALIC SMALL Q (0x1d45e) + case L'𝑟': // MATHEMATICAL ITALIC SMALL R (0x1d45f) + case L'𝑠': // MATHEMATICAL ITALIC SMALL S (0x1d460) + case L'𝑡': // MATHEMATICAL ITALIC SMALL T (0x1d461) + case L'𝑢': // MATHEMATICAL ITALIC SMALL U (0x1d462) + case L'𝑣': // MATHEMATICAL ITALIC SMALL V (0x1d463) + case L'𝑤': // MATHEMATICAL ITALIC SMALL W (0x1d464) + case L'𝑥': // MATHEMATICAL ITALIC SMALL X (0x1d465) + case L'𝑦': // MATHEMATICAL ITALIC SMALL Y (0x1d466) + case L'𝑧': // MATHEMATICAL ITALIC SMALL Z (0x1d467) + case L'𝒂': // MATHEMATICAL BOLD ITALIC SMALL A (0x1d482) + case L'𝒃': // MATHEMATICAL BOLD ITALIC SMALL B (0x1d483) + case L'𝒄': // MATHEMATICAL BOLD ITALIC SMALL C (0x1d484) + case L'𝒅': // MATHEMATICAL BOLD ITALIC SMALL D (0x1d485) + case L'𝒆': // MATHEMATICAL BOLD ITALIC SMALL E (0x1d486) + case L'𝒇': // MATHEMATICAL BOLD ITALIC SMALL F (0x1d487) + case L'𝒈': // MATHEMATICAL BOLD ITALIC SMALL G (0x1d488) + case L'𝒉': // MATHEMATICAL BOLD ITALIC SMALL H (0x1d489) + case L'𝒊': // MATHEMATICAL BOLD ITALIC SMALL I (0x1d48a) + case L'𝒋': // MATHEMATICAL BOLD ITALIC SMALL J (0x1d48b) + case L'𝒌': // MATHEMATICAL BOLD ITALIC SMALL K (0x1d48c) + case L'𝒍': // MATHEMATICAL BOLD ITALIC SMALL L (0x1d48d) + case L'𝒎': // MATHEMATICAL BOLD ITALIC SMALL M (0x1d48e) + case L'𝒏': // MATHEMATICAL BOLD ITALIC SMALL N (0x1d48f) + case L'𝒐': // MATHEMATICAL BOLD ITALIC SMALL O (0x1d490) + case L'𝒑': // MATHEMATICAL BOLD ITALIC SMALL P (0x1d491) + case L'𝒒': // MATHEMATICAL BOLD ITALIC SMALL Q (0x1d492) + case L'𝒓': // MATHEMATICAL BOLD ITALIC SMALL R (0x1d493) + case L'𝒔': // MATHEMATICAL BOLD ITALIC SMALL S (0x1d494) + case L'𝒕': // MATHEMATICAL BOLD ITALIC SMALL T (0x1d495) + case L'𝒖': // MATHEMATICAL BOLD ITALIC SMALL U (0x1d496) + case L'𝒗': // MATHEMATICAL BOLD ITALIC SMALL V (0x1d497) + case L'𝒘': // MATHEMATICAL BOLD ITALIC SMALL W (0x1d498) + case L'𝒙': // MATHEMATICAL BOLD ITALIC SMALL X (0x1d499) + case L'𝒚': // MATHEMATICAL BOLD ITALIC SMALL Y (0x1d49a) + case L'𝒛': // MATHEMATICAL BOLD ITALIC SMALL Z (0x1d49b) + case L'𝒶': // MATHEMATICAL SCRIPT SMALL A (0x1d4b6) + case L'𝒷': // MATHEMATICAL SCRIPT SMALL B (0x1d4b7) + case L'𝒸': // MATHEMATICAL SCRIPT SMALL C (0x1d4b8) + case L'𝒹': // MATHEMATICAL SCRIPT SMALL D (0x1d4b9) + case L'𝒻': // MATHEMATICAL SCRIPT SMALL F (0x1d4bb) + case L'𝒽': // MATHEMATICAL SCRIPT SMALL H (0x1d4bd) + case L'𝒾': // MATHEMATICAL SCRIPT SMALL I (0x1d4be) + case L'𝒿': // MATHEMATICAL SCRIPT SMALL J (0x1d4bf) + case L'𝓀': // MATHEMATICAL SCRIPT SMALL K (0x1d4c0) + case L'𝓁': // MATHEMATICAL SCRIPT SMALL L (0x1d4c1) + case L'𝓂': // MATHEMATICAL SCRIPT SMALL M (0x1d4c2) + case L'𝓃': // MATHEMATICAL SCRIPT SMALL N (0x1d4c3) + case L'𝓅': // MATHEMATICAL SCRIPT SMALL P (0x1d4c5) + case L'𝓆': // MATHEMATICAL SCRIPT SMALL Q (0x1d4c6) + case L'𝓇': // MATHEMATICAL SCRIPT SMALL R (0x1d4c7) + case L'𝓈': // MATHEMATICAL SCRIPT SMALL S (0x1d4c8) + case L'𝓉': // MATHEMATICAL SCRIPT SMALL T (0x1d4c9) + case L'𝓊': // MATHEMATICAL SCRIPT SMALL U (0x1d4ca) + case L'𝓋': // MATHEMATICAL SCRIPT SMALL V (0x1d4cb) + case L'𝓌': // MATHEMATICAL SCRIPT SMALL W (0x1d4cc) + case L'𝓍': // MATHEMATICAL SCRIPT SMALL X (0x1d4cd) + case L'𝓎': // MATHEMATICAL SCRIPT SMALL Y (0x1d4ce) + case L'𝓏': // MATHEMATICAL SCRIPT SMALL Z (0x1d4cf) + case L'𝓪': // MATHEMATICAL BOLD SCRIPT SMALL A (0x1d4ea) + case L'𝓫': // MATHEMATICAL BOLD SCRIPT SMALL B (0x1d4eb) + case L'𝓬': // MATHEMATICAL BOLD SCRIPT SMALL C (0x1d4ec) + case L'𝓭': // MATHEMATICAL BOLD SCRIPT SMALL D (0x1d4ed) + case L'𝓮': // MATHEMATICAL BOLD SCRIPT SMALL E (0x1d4ee) + case L'𝓯': // MATHEMATICAL BOLD SCRIPT SMALL F (0x1d4ef) + case L'𝓰': // MATHEMATICAL BOLD SCRIPT SMALL G (0x1d4f0) + case L'𝓱': // MATHEMATICAL BOLD SCRIPT SMALL H (0x1d4f1) + case L'𝓲': // MATHEMATICAL BOLD SCRIPT SMALL I (0x1d4f2) + case L'𝓳': // MATHEMATICAL BOLD SCRIPT SMALL J (0x1d4f3) + case L'𝓴': // MATHEMATICAL BOLD SCRIPT SMALL K (0x1d4f4) + case L'𝓵': // MATHEMATICAL BOLD SCRIPT SMALL L (0x1d4f5) + case L'𝓶': // MATHEMATICAL BOLD SCRIPT SMALL M (0x1d4f6) + case L'𝓷': // MATHEMATICAL BOLD SCRIPT SMALL N (0x1d4f7) + case L'𝓸': // MATHEMATICAL BOLD SCRIPT SMALL O (0x1d4f8) + case L'𝓹': // MATHEMATICAL BOLD SCRIPT SMALL P (0x1d4f9) + case L'𝓺': // MATHEMATICAL BOLD SCRIPT SMALL Q (0x1d4fa) + case L'𝓻': // MATHEMATICAL BOLD SCRIPT SMALL R (0x1d4fb) + case L'𝓼': // MATHEMATICAL BOLD SCRIPT SMALL S (0x1d4fc) + case L'𝓽': // MATHEMATICAL BOLD SCRIPT SMALL T (0x1d4fd) + case L'𝓾': // MATHEMATICAL BOLD SCRIPT SMALL U (0x1d4fe) + case L'𝓿': // MATHEMATICAL BOLD SCRIPT SMALL V (0x1d4ff) + case L'𝔀': // MATHEMATICAL BOLD SCRIPT SMALL W (0x1d500) + case L'𝔁': // MATHEMATICAL BOLD SCRIPT SMALL X (0x1d501) + case L'𝔂': // MATHEMATICAL BOLD SCRIPT SMALL Y (0x1d502) + case L'𝔃': // MATHEMATICAL BOLD SCRIPT SMALL Z (0x1d503) + case L'𝔞': // MATHEMATICAL FRAKTUR SMALL A (0x1d51e) + case L'𝔟': // MATHEMATICAL FRAKTUR SMALL B (0x1d51f) + case L'𝔠': // MATHEMATICAL FRAKTUR SMALL C (0x1d520) + case L'𝔡': // MATHEMATICAL FRAKTUR SMALL D (0x1d521) + case L'𝔢': // MATHEMATICAL FRAKTUR SMALL E (0x1d522) + case L'𝔣': // MATHEMATICAL FRAKTUR SMALL F (0x1d523) + case L'𝔤': // MATHEMATICAL FRAKTUR SMALL G (0x1d524) + case L'𝔥': // MATHEMATICAL FRAKTUR SMALL H (0x1d525) + case L'𝔦': // MATHEMATICAL FRAKTUR SMALL I (0x1d526) + case L'𝔧': // MATHEMATICAL FRAKTUR SMALL J (0x1d527) + case L'𝔨': // MATHEMATICAL FRAKTUR SMALL K (0x1d528) + case L'𝔩': // MATHEMATICAL FRAKTUR SMALL L (0x1d529) + case L'𝔪': // MATHEMATICAL FRAKTUR SMALL M (0x1d52a) + case L'𝔫': // MATHEMATICAL FRAKTUR SMALL N (0x1d52b) + case L'𝔬': // MATHEMATICAL FRAKTUR SMALL O (0x1d52c) + case L'𝔭': // MATHEMATICAL FRAKTUR SMALL P (0x1d52d) + case L'𝔮': // MATHEMATICAL FRAKTUR SMALL Q (0x1d52e) + case L'𝔯': // MATHEMATICAL FRAKTUR SMALL R (0x1d52f) + case L'𝔰': // MATHEMATICAL FRAKTUR SMALL S (0x1d530) + case L'𝔱': // MATHEMATICAL FRAKTUR SMALL T (0x1d531) + case L'𝔲': // MATHEMATICAL FRAKTUR SMALL U (0x1d532) + case L'𝔳': // MATHEMATICAL FRAKTUR SMALL V (0x1d533) + case L'𝔴': // MATHEMATICAL FRAKTUR SMALL W (0x1d534) + case L'𝔵': // MATHEMATICAL FRAKTUR SMALL X (0x1d535) + case L'𝔶': // MATHEMATICAL FRAKTUR SMALL Y (0x1d536) + case L'𝔷': // MATHEMATICAL FRAKTUR SMALL Z (0x1d537) + case L'𝕒': // MATHEMATICAL DOUBLE-STRUCK SMALL A (0x1d552) + case L'𝕓': // MATHEMATICAL DOUBLE-STRUCK SMALL B (0x1d553) + case L'𝕔': // MATHEMATICAL DOUBLE-STRUCK SMALL C (0x1d554) + case L'𝕕': // MATHEMATICAL DOUBLE-STRUCK SMALL D (0x1d555) + case L'𝕖': // MATHEMATICAL DOUBLE-STRUCK SMALL E (0x1d556) + case L'𝕗': // MATHEMATICAL DOUBLE-STRUCK SMALL F (0x1d557) + case L'𝕘': // MATHEMATICAL DOUBLE-STRUCK SMALL G (0x1d558) + case L'𝕙': // MATHEMATICAL DOUBLE-STRUCK SMALL H (0x1d559) + case L'𝕚': // MATHEMATICAL DOUBLE-STRUCK SMALL I (0x1d55a) + case L'𝕛': // MATHEMATICAL DOUBLE-STRUCK SMALL J (0x1d55b) + case L'𝕜': // MATHEMATICAL DOUBLE-STRUCK SMALL K (0x1d55c) + case L'𝕝': // MATHEMATICAL DOUBLE-STRUCK SMALL L (0x1d55d) + case L'𝕞': // MATHEMATICAL DOUBLE-STRUCK SMALL M (0x1d55e) + case L'𝕟': // MATHEMATICAL DOUBLE-STRUCK SMALL N (0x1d55f) + case L'𝕠': // MATHEMATICAL DOUBLE-STRUCK SMALL O (0x1d560) + case L'𝕡': // MATHEMATICAL DOUBLE-STRUCK SMALL P (0x1d561) + case L'𝕢': // MATHEMATICAL DOUBLE-STRUCK SMALL Q (0x1d562) + case L'𝕣': // MATHEMATICAL DOUBLE-STRUCK SMALL R (0x1d563) + case L'𝕤': // MATHEMATICAL DOUBLE-STRUCK SMALL S (0x1d564) + case L'𝕥': // MATHEMATICAL DOUBLE-STRUCK SMALL T (0x1d565) + case L'𝕦': // MATHEMATICAL DOUBLE-STRUCK SMALL U (0x1d566) + case L'𝕧': // MATHEMATICAL DOUBLE-STRUCK SMALL V (0x1d567) + case L'𝕨': // MATHEMATICAL DOUBLE-STRUCK SMALL W (0x1d568) + case L'𝕩': // MATHEMATICAL DOUBLE-STRUCK SMALL X (0x1d569) + case L'𝕪': // MATHEMATICAL DOUBLE-STRUCK SMALL Y (0x1d56a) + case L'𝕫': // MATHEMATICAL DOUBLE-STRUCK SMALL Z (0x1d56b) + case L'𝖆': // MATHEMATICAL BOLD FRAKTUR SMALL A (0x1d586) + case L'𝖇': // MATHEMATICAL BOLD FRAKTUR SMALL B (0x1d587) + case L'𝖈': // MATHEMATICAL BOLD FRAKTUR SMALL C (0x1d588) + case L'𝖉': // MATHEMATICAL BOLD FRAKTUR SMALL D (0x1d589) + case L'𝖊': // MATHEMATICAL BOLD FRAKTUR SMALL E (0x1d58a) + case L'𝖋': // MATHEMATICAL BOLD FRAKTUR SMALL F (0x1d58b) + case L'𝖌': // MATHEMATICAL BOLD FRAKTUR SMALL G (0x1d58c) + case L'𝖍': // MATHEMATICAL BOLD FRAKTUR SMALL H (0x1d58d) + case L'𝖎': // MATHEMATICAL BOLD FRAKTUR SMALL I (0x1d58e) + case L'𝖏': // MATHEMATICAL BOLD FRAKTUR SMALL J (0x1d58f) + case L'𝖐': // MATHEMATICAL BOLD FRAKTUR SMALL K (0x1d590) + case L'𝖑': // MATHEMATICAL BOLD FRAKTUR SMALL L (0x1d591) + case L'𝖒': // MATHEMATICAL BOLD FRAKTUR SMALL M (0x1d592) + case L'𝖓': // MATHEMATICAL BOLD FRAKTUR SMALL N (0x1d593) + case L'𝖔': // MATHEMATICAL BOLD FRAKTUR SMALL O (0x1d594) + case L'𝖕': // MATHEMATICAL BOLD FRAKTUR SMALL P (0x1d595) + case L'𝖖': // MATHEMATICAL BOLD FRAKTUR SMALL Q (0x1d596) + case L'𝖗': // MATHEMATICAL BOLD FRAKTUR SMALL R (0x1d597) + case L'𝖘': // MATHEMATICAL BOLD FRAKTUR SMALL S (0x1d598) + case L'𝖙': // MATHEMATICAL BOLD FRAKTUR SMALL T (0x1d599) + case L'𝖚': // MATHEMATICAL BOLD FRAKTUR SMALL U (0x1d59a) + case L'𝖛': // MATHEMATICAL BOLD FRAKTUR SMALL V (0x1d59b) + case L'𝖜': // MATHEMATICAL BOLD FRAKTUR SMALL W (0x1d59c) + case L'𝖝': // MATHEMATICAL BOLD FRAKTUR SMALL X (0x1d59d) + case L'𝖞': // MATHEMATICAL BOLD FRAKTUR SMALL Y (0x1d59e) + case L'𝖟': // MATHEMATICAL BOLD FRAKTUR SMALL Z (0x1d59f) + case L'𝖺': // MATHEMATICAL SANS SMALL A (0x1d5ba) + case L'𝖻': // MATHEMATICAL SANS SMALL B (0x1d5bb) + case L'𝖼': // MATHEMATICAL SANS SMALL C (0x1d5bc) + case L'𝖽': // MATHEMATICAL SANS SMALL D (0x1d5bd) + case L'𝖾': // MATHEMATICAL SANS SMALL E (0x1d5be) + case L'𝖿': // MATHEMATICAL SANS SMALL F (0x1d5bf) + case L'𝗀': // MATHEMATICAL SANS SMALL G (0x1d5c0) + case L'𝗁': // MATHEMATICAL SANS SMALL H (0x1d5c1) + case L'𝗂': // MATHEMATICAL SANS SMALL I (0x1d5c2) + case L'𝗃': // MATHEMATICAL SANS SMALL J (0x1d5c3) + case L'𝗄': // MATHEMATICAL SANS SMALL K (0x1d5c4) + case L'𝗅': // MATHEMATICAL SANS SMALL L (0x1d5c5) + case L'𝗆': // MATHEMATICAL SANS SMALL M (0x1d5c6) + case L'𝗇': // MATHEMATICAL SANS SMALL N (0x1d5c7) + case L'𝗈': // MATHEMATICAL SANS SMALL O (0x1d5c8) + case L'𝗉': // MATHEMATICAL SANS SMALL P (0x1d5c9) + case L'𝗊': // MATHEMATICAL SANS SMALL Q (0x1d5ca) + case L'𝗋': // MATHEMATICAL SANS SMALL R (0x1d5cb) + case L'𝗌': // MATHEMATICAL SANS SMALL S (0x1d5cc) + case L'𝗍': // MATHEMATICAL SANS SMALL T (0x1d5cd) + case L'𝗎': // MATHEMATICAL SANS SMALL U (0x1d5ce) + case L'𝗏': // MATHEMATICAL SANS SMALL V (0x1d5cf) + case L'𝗐': // MATHEMATICAL SANS SMALL W (0x1d5d0) + case L'𝗑': // MATHEMATICAL SANS SMALL X (0x1d5d1) + case L'𝗒': // MATHEMATICAL SANS SMALL Y (0x1d5d2) + case L'𝗓': // MATHEMATICAL SANS SMALL Z (0x1d5d3) + case L'𝗮': // MATHEMATICAL SANS BOLD SMALL A (0x1d5ee) + case L'𝗯': // MATHEMATICAL SANS BOLD SMALL B (0x1d5ef) + case L'𝗰': // MATHEMATICAL SANS BOLD SMALL C (0x1d5f0) + case L'𝗱': // MATHEMATICAL SANS BOLD SMALL D (0x1d5f1) + case L'𝗲': // MATHEMATICAL SANS BOLD SMALL E (0x1d5f2) + case L'𝗳': // MATHEMATICAL SANS BOLD SMALL F (0x1d5f3) + case L'𝗴': // MATHEMATICAL SANS BOLD SMALL G (0x1d5f4) + case L'𝗵': // MATHEMATICAL SANS BOLD SMALL H (0x1d5f5) + case L'𝗶': // MATHEMATICAL SANS BOLD SMALL I (0x1d5f6) + case L'𝗷': // MATHEMATICAL SANS BOLD SMALL J (0x1d5f7) + case L'𝗸': // MATHEMATICAL SANS BOLD SMALL K (0x1d5f8) + case L'𝗹': // MATHEMATICAL SANS BOLD SMALL L (0x1d5f9) + case L'𝗺': // MATHEMATICAL SANS BOLD SMALL M (0x1d5fa) + case L'𝗻': // MATHEMATICAL SANS BOLD SMALL N (0x1d5fb) + case L'𝗼': // MATHEMATICAL SANS BOLD SMALL O (0x1d5fc) + case L'𝗽': // MATHEMATICAL SANS BOLD SMALL P (0x1d5fd) + case L'𝗾': // MATHEMATICAL SANS BOLD SMALL Q (0x1d5fe) + case L'𝗿': // MATHEMATICAL SANS BOLD SMALL R (0x1d5ff) + case L'𝘀': // MATHEMATICAL SANS BOLD SMALL S (0x1d600) + case L'𝘁': // MATHEMATICAL SANS BOLD SMALL T (0x1d601) + case L'𝘂': // MATHEMATICAL SANS BOLD SMALL U (0x1d602) + case L'𝘃': // MATHEMATICAL SANS BOLD SMALL V (0x1d603) + case L'𝘄': // MATHEMATICAL SANS BOLD SMALL W (0x1d604) + case L'𝘅': // MATHEMATICAL SANS BOLD SMALL X (0x1d605) + case L'𝘆': // MATHEMATICAL SANS BOLD SMALL Y (0x1d606) + case L'𝘇': // MATHEMATICAL SANS BOLD SMALL Z (0x1d607) + case L'𝘢': // MATHEMATICAL SANS ITALIC SMALL A (0x1d622) + case L'𝘣': // MATHEMATICAL SANS ITALIC SMALL B (0x1d623) + case L'𝘤': // MATHEMATICAL SANS ITALIC SMALL C (0x1d624) + case L'𝘥': // MATHEMATICAL SANS ITALIC SMALL D (0x1d625) + case L'𝘦': // MATHEMATICAL SANS ITALIC SMALL E (0x1d626) + case L'𝘧': // MATHEMATICAL SANS ITALIC SMALL F (0x1d627) + case L'𝘨': // MATHEMATICAL SANS ITALIC SMALL G (0x1d628) + case L'𝘩': // MATHEMATICAL SANS ITALIC SMALL H (0x1d629) + case L'𝘪': // MATHEMATICAL SANS ITALIC SMALL I (0x1d62a) + case L'𝘫': // MATHEMATICAL SANS ITALIC SMALL J (0x1d62b) + case L'𝘬': // MATHEMATICAL SANS ITALIC SMALL K (0x1d62c) + case L'𝘭': // MATHEMATICAL SANS ITALIC SMALL L (0x1d62d) + case L'𝘮': // MATHEMATICAL SANS ITALIC SMALL M (0x1d62e) + case L'𝘯': // MATHEMATICAL SANS ITALIC SMALL N (0x1d62f) + case L'𝘰': // MATHEMATICAL SANS ITALIC SMALL O (0x1d630) + case L'𝘱': // MATHEMATICAL SANS ITALIC SMALL P (0x1d631) + case L'𝘲': // MATHEMATICAL SANS ITALIC SMALL Q (0x1d632) + case L'𝘳': // MATHEMATICAL SANS ITALIC SMALL R (0x1d633) + case L'𝘴': // MATHEMATICAL SANS ITALIC SMALL S (0x1d634) + case L'𝘵': // MATHEMATICAL SANS ITALIC SMALL T (0x1d635) + case L'𝘶': // MATHEMATICAL SANS ITALIC SMALL U (0x1d636) + case L'𝘷': // MATHEMATICAL SANS ITALIC SMALL V (0x1d637) + case L'𝘸': // MATHEMATICAL SANS ITALIC SMALL W (0x1d638) + case L'𝘹': // MATHEMATICAL SANS ITALIC SMALL X (0x1d639) + case L'𝘺': // MATHEMATICAL SANS ITALIC SMALL Y (0x1d63a) + case L'𝘻': // MATHEMATICAL SANS ITALIC SMALL Z (0x1d63b) + case L'𝙖': // MATHEMATICAL SANS BOLD ITALIC SMALL A (0x1d656) + case L'𝙗': // MATHEMATICAL SANS BOLD ITALIC SMALL B (0x1d657) + case L'𝙘': // MATHEMATICAL SANS BOLD ITALIC SMALL C (0x1d658) + case L'𝙙': // MATHEMATICAL SANS BOLD ITALIC SMALL D (0x1d659) + case L'𝙚': // MATHEMATICAL SANS BOLD ITALIC SMALL E (0x1d65a) + case L'𝙛': // MATHEMATICAL SANS BOLD ITALIC SMALL F (0x1d65b) + case L'𝙜': // MATHEMATICAL SANS BOLD ITALIC SMALL G (0x1d65c) + case L'𝙝': // MATHEMATICAL SANS BOLD ITALIC SMALL H (0x1d65d) + case L'𝙞': // MATHEMATICAL SANS BOLD ITALIC SMALL I (0x1d65e) + case L'𝙟': // MATHEMATICAL SANS BOLD ITALIC SMALL J (0x1d65f) + case L'𝙠': // MATHEMATICAL SANS BOLD ITALIC SMALL K (0x1d660) + case L'𝙡': // MATHEMATICAL SANS BOLD ITALIC SMALL L (0x1d661) + case L'𝙢': // MATHEMATICAL SANS BOLD ITALIC SMALL M (0x1d662) + case L'𝙣': // MATHEMATICAL SANS BOLD ITALIC SMALL N (0x1d663) + case L'𝙤': // MATHEMATICAL SANS BOLD ITALIC SMALL O (0x1d664) + case L'𝙥': // MATHEMATICAL SANS BOLD ITALIC SMALL P (0x1d665) + case L'𝙦': // MATHEMATICAL SANS BOLD ITALIC SMALL Q (0x1d666) + case L'𝙧': // MATHEMATICAL SANS BOLD ITALIC SMALL R (0x1d667) + case L'𝙨': // MATHEMATICAL SANS BOLD ITALIC SMALL S (0x1d668) + case L'𝙩': // MATHEMATICAL SANS BOLD ITALIC SMALL T (0x1d669) + case L'𝙪': // MATHEMATICAL SANS BOLD ITALIC SMALL U (0x1d66a) + case L'𝙫': // MATHEMATICAL SANS BOLD ITALIC SMALL V (0x1d66b) + case L'𝙬': // MATHEMATICAL SANS BOLD ITALIC SMALL W (0x1d66c) + case L'𝙭': // MATHEMATICAL SANS BOLD ITALIC SMALL X (0x1d66d) + case L'𝙮': // MATHEMATICAL SANS BOLD ITALIC SMALL Y (0x1d66e) + case L'𝙯': // MATHEMATICAL SANS BOLD ITALIC SMALL Z (0x1d66f) + case L'𝚊': // MATHEMATICAL MONOSPACE SMALL A (0x1d68a) + case L'𝚋': // MATHEMATICAL MONOSPACE SMALL B (0x1d68b) + case L'𝚌': // MATHEMATICAL MONOSPACE SMALL C (0x1d68c) + case L'𝚍': // MATHEMATICAL MONOSPACE SMALL D (0x1d68d) + case L'𝚎': // MATHEMATICAL MONOSPACE SMALL E (0x1d68e) + case L'𝚏': // MATHEMATICAL MONOSPACE SMALL F (0x1d68f) + case L'𝚐': // MATHEMATICAL MONOSPACE SMALL G (0x1d690) + case L'𝚑': // MATHEMATICAL MONOSPACE SMALL H (0x1d691) + case L'𝚒': // MATHEMATICAL MONOSPACE SMALL I (0x1d692) + case L'𝚓': // MATHEMATICAL MONOSPACE SMALL J (0x1d693) + case L'𝚔': // MATHEMATICAL MONOSPACE SMALL K (0x1d694) + case L'𝚕': // MATHEMATICAL MONOSPACE SMALL L (0x1d695) + case L'𝚖': // MATHEMATICAL MONOSPACE SMALL M (0x1d696) + case L'𝚗': // MATHEMATICAL MONOSPACE SMALL N (0x1d697) + case L'𝚘': // MATHEMATICAL MONOSPACE SMALL O (0x1d698) + case L'𝚙': // MATHEMATICAL MONOSPACE SMALL P (0x1d699) + case L'𝚚': // MATHEMATICAL MONOSPACE SMALL Q (0x1d69a) + case L'𝚛': // MATHEMATICAL MONOSPACE SMALL R (0x1d69b) + case L'𝚜': // MATHEMATICAL MONOSPACE SMALL S (0x1d69c) + case L'𝚝': // MATHEMATICAL MONOSPACE SMALL T (0x1d69d) + case L'𝚞': // MATHEMATICAL MONOSPACE SMALL U (0x1d69e) + case L'𝚟': // MATHEMATICAL MONOSPACE SMALL V (0x1d69f) + case L'𝚠': // MATHEMATICAL MONOSPACE SMALL W (0x1d6a0) + case L'𝚡': // MATHEMATICAL MONOSPACE SMALL X (0x1d6a1) + case L'𝚢': // MATHEMATICAL MONOSPACE SMALL Y (0x1d6a2) + case L'𝚣': // MATHEMATICAL MONOSPACE SMALL Z (0x1d6a3) + case L'𝚤': // MATHEMATICAL ITALIC SMALL DOTLESS I (0x1d6a4) + case L'𝚥': // MATHEMATICAL ITALIC SMALL DOTLESS J (0x1d6a5) + case L'𝛂': // MATHEMATICAL BOLD SMALL ALPHA (0x1d6c2) + case L'𝛃': // MATHEMATICAL BOLD SMALL BETA (0x1d6c3) + case L'𝛄': // MATHEMATICAL BOLD SMALL GAMMA (0x1d6c4) + case L'𝛅': // MATHEMATICAL BOLD SMALL DELTA (0x1d6c5) + case L'𝛆': // MATHEMATICAL BOLD SMALL EPSILON (0x1d6c6) + case L'𝛇': // MATHEMATICAL BOLD SMALL ZETA (0x1d6c7) + case L'𝛈': // MATHEMATICAL BOLD SMALL ETA (0x1d6c8) + case L'𝛉': // MATHEMATICAL BOLD SMALL THETA (0x1d6c9) + case L'𝛊': // MATHEMATICAL BOLD SMALL IOTA (0x1d6ca) + case L'𝛋': // MATHEMATICAL BOLD SMALL KAPPA (0x1d6cb) + case L'𝛌': // MATHEMATICAL BOLD SMALL LAMDA (0x1d6cc) + case L'𝛍': // MATHEMATICAL BOLD SMALL MU (0x1d6cd) + case L'𝛎': // MATHEMATICAL BOLD SMALL NU (0x1d6ce) + case L'𝛏': // MATHEMATICAL BOLD SMALL XI (0x1d6cf) + case L'𝛐': // MATHEMATICAL BOLD SMALL OMICRON (0x1d6d0) + case L'𝛑': // MATHEMATICAL BOLD SMALL PI (0x1d6d1) + case L'𝛒': // MATHEMATICAL BOLD SMALL RHO (0x1d6d2) + case L'𝛓': // MATHEMATICAL BOLD SMALL FINAL SIGMA (0x1d6d3) + case L'𝛔': // MATHEMATICAL BOLD SMALL SIGMA (0x1d6d4) + case L'𝛕': // MATHEMATICAL BOLD SMALL TAU (0x1d6d5) + case L'𝛖': // MATHEMATICAL BOLD SMALL UPSILON (0x1d6d6) + case L'𝛗': // MATHEMATICAL BOLD SMALL PHI (0x1d6d7) + case L'𝛘': // MATHEMATICAL BOLD SMALL CHI (0x1d6d8) + case L'𝛙': // MATHEMATICAL BOLD SMALL PSI (0x1d6d9) + case L'𝛚': // MATHEMATICAL BOLD SMALL OMEGA (0x1d6da) + case L'𝛜': // MATHEMATICAL BOLD EPSILON SYMBOL (0x1d6dc) + case L'𝛝': // MATHEMATICAL BOLD THETA SYMBOL (0x1d6dd) + case L'𝛞': // MATHEMATICAL BOLD KAPPA SYMBOL (0x1d6de) + case L'𝛟': // MATHEMATICAL BOLD PHI SYMBOL (0x1d6df) + case L'𝛠': // MATHEMATICAL BOLD RHO SYMBOL (0x1d6e0) + case L'𝛡': // MATHEMATICAL BOLD PI SYMBOL (0x1d6e1) + case L'𝛼': // MATHEMATICAL ITALIC SMALL ALPHA (0x1d6fc) + case L'𝛽': // MATHEMATICAL ITALIC SMALL BETA (0x1d6fd) + case L'𝛾': // MATHEMATICAL ITALIC SMALL GAMMA (0x1d6fe) + case L'𝛿': // MATHEMATICAL ITALIC SMALL DELTA (0x1d6ff) + case L'𝜀': // MATHEMATICAL ITALIC SMALL EPSILON (0x1d700) + case L'𝜁': // MATHEMATICAL ITALIC SMALL ZETA (0x1d701) + case L'𝜂': // MATHEMATICAL ITALIC SMALL ETA (0x1d702) + case L'𝜃': // MATHEMATICAL ITALIC SMALL THETA (0x1d703) + case L'𝜄': // MATHEMATICAL ITALIC SMALL IOTA (0x1d704) + case L'𝜅': // MATHEMATICAL ITALIC SMALL KAPPA (0x1d705) + case L'𝜆': // MATHEMATICAL ITALIC SMALL LAMDA (0x1d706) + case L'𝜇': // MATHEMATICAL ITALIC SMALL MU (0x1d707) + case L'𝜈': // MATHEMATICAL ITALIC SMALL NU (0x1d708) + case L'𝜉': // MATHEMATICAL ITALIC SMALL XI (0x1d709) + case L'𝜊': // MATHEMATICAL ITALIC SMALL OMICRON (0x1d70a) + case L'𝜋': // MATHEMATICAL ITALIC SMALL PI (0x1d70b) + case L'𝜌': // MATHEMATICAL ITALIC SMALL RHO (0x1d70c) + case L'𝜍': // MATHEMATICAL ITALIC SMALL FINAL SIGMA (0x1d70d) + case L'𝜎': // MATHEMATICAL ITALIC SMALL SIGMA (0x1d70e) + case L'𝜏': // MATHEMATICAL ITALIC SMALL TAU (0x1d70f) + case L'𝜐': // MATHEMATICAL ITALIC SMALL UPSILON (0x1d710) + case L'𝜑': // MATHEMATICAL ITALIC SMALL PHI (0x1d711) + case L'𝜒': // MATHEMATICAL ITALIC SMALL CHI (0x1d712) + case L'𝜓': // MATHEMATICAL ITALIC SMALL PSI (0x1d713) + case L'𝜔': // MATHEMATICAL ITALIC SMALL OMEGA (0x1d714) + case L'𝜖': // MATHEMATICAL ITALIC EPSILON SYMBOL (0x1d716) + case L'𝜗': // MATHEMATICAL ITALIC THETA SYMBOL (0x1d717) + case L'𝜘': // MATHEMATICAL ITALIC KAPPA SYMBOL (0x1d718) + case L'𝜙': // MATHEMATICAL ITALIC PHI SYMBOL (0x1d719) + case L'𝜚': // MATHEMATICAL ITALIC RHO SYMBOL (0x1d71a) + case L'𝜛': // MATHEMATICAL ITALIC PI SYMBOL (0x1d71b) + case L'𝜶': // MATHEMATICAL BOLD ITALIC SMALL ALPHA (0x1d736) + case L'𝜷': // MATHEMATICAL BOLD ITALIC SMALL BETA (0x1d737) + case L'𝜸': // MATHEMATICAL BOLD ITALIC SMALL GAMMA (0x1d738) + case L'𝜹': // MATHEMATICAL BOLD ITALIC SMALL DELTA (0x1d739) + case L'𝜺': // MATHEMATICAL BOLD ITALIC SMALL EPSILON (0x1d73a) + case L'𝜻': // MATHEMATICAL BOLD ITALIC SMALL ZETA (0x1d73b) + case L'𝜼': // MATHEMATICAL BOLD ITALIC SMALL ETA (0x1d73c) + case L'𝜽': // MATHEMATICAL BOLD ITALIC SMALL THETA (0x1d73d) + case L'𝜾': // MATHEMATICAL BOLD ITALIC SMALL IOTA (0x1d73e) + case L'𝜿': // MATHEMATICAL BOLD ITALIC SMALL KAPPA (0x1d73f) + case L'𝝀': // MATHEMATICAL BOLD ITALIC SMALL LAMDA (0x1d740) + case L'𝝁': // MATHEMATICAL BOLD ITALIC SMALL MU (0x1d741) + case L'𝝂': // MATHEMATICAL BOLD ITALIC SMALL NU (0x1d742) + case L'𝝃': // MATHEMATICAL BOLD ITALIC SMALL XI (0x1d743) + case L'𝝄': // MATHEMATICAL BOLD ITALIC SMALL OMICRON (0x1d744) + case L'𝝅': // MATHEMATICAL BOLD ITALIC SMALL PI (0x1d745) + case L'𝝆': // MATHEMATICAL BOLD ITALIC SMALL RHO (0x1d746) + case L'𝝇': // MATHEMATICAL BOLD ITALIC SMALL FINAL SIGMA (0x1d747) + case L'𝝈': // MATHEMATICAL BOLD ITALIC SMALL SIGMA (0x1d748) + case L'𝝉': // MATHEMATICAL BOLD ITALIC SMALL TAU (0x1d749) + case L'𝝊': // MATHEMATICAL BOLD ITALIC SMALL UPSILON (0x1d74a) + case L'𝝋': // MATHEMATICAL BOLD ITALIC SMALL PHI (0x1d74b) + case L'𝝌': // MATHEMATICAL BOLD ITALIC SMALL CHI (0x1d74c) + case L'𝝍': // MATHEMATICAL BOLD ITALIC SMALL PSI (0x1d74d) + case L'𝝎': // MATHEMATICAL BOLD ITALIC SMALL OMEGA (0x1d74e) + case L'𝝐': // MATHEMATICAL BOLD ITALIC EPSILON SYMBOL (0x1d750) + case L'𝝑': // MATHEMATICAL BOLD ITALIC THETA SYMBOL (0x1d751) + case L'𝝒': // MATHEMATICAL BOLD ITALIC KAPPA SYMBOL (0x1d752) + case L'𝝓': // MATHEMATICAL BOLD ITALIC PHI SYMBOL (0x1d753) + case L'𝝔': // MATHEMATICAL BOLD ITALIC RHO SYMBOL (0x1d754) + case L'𝝕': // MATHEMATICAL BOLD ITALIC PI SYMBOL (0x1d755) + case L'𝝰': // MATHEMATICAL SANS BOLD SMALL ALPHA (0x1d770) + case L'𝝱': // MATHEMATICAL SANS BOLD SMALL BETA (0x1d771) + case L'𝝲': // MATHEMATICAL SANS BOLD SMALL GAMMA (0x1d772) + case L'𝝳': // MATHEMATICAL SANS BOLD SMALL DELTA (0x1d773) + case L'𝝴': // MATHEMATICAL SANS BOLD SMALL EPSILON (0x1d774) + case L'𝝵': // MATHEMATICAL SANS BOLD SMALL ZETA (0x1d775) + case L'𝝶': // MATHEMATICAL SANS BOLD SMALL ETA (0x1d776) + case L'𝝷': // MATHEMATICAL SANS BOLD SMALL THETA (0x1d777) + case L'𝝸': // MATHEMATICAL SANS BOLD SMALL IOTA (0x1d778) + case L'𝝹': // MATHEMATICAL SANS BOLD SMALL KAPPA (0x1d779) + case L'𝝺': // MATHEMATICAL SANS BOLD SMALL LAMDA (0x1d77a) + case L'𝝻': // MATHEMATICAL SANS BOLD SMALL MU (0x1d77b) + case L'𝝼': // MATHEMATICAL SANS BOLD SMALL NU (0x1d77c) + case L'𝝽': // MATHEMATICAL SANS BOLD SMALL XI (0x1d77d) + case L'𝝾': // MATHEMATICAL SANS BOLD SMALL OMICRON (0x1d77e) + case L'𝝿': // MATHEMATICAL SANS BOLD SMALL PI (0x1d77f) + case L'𝞀': // MATHEMATICAL SANS BOLD SMALL RHO (0x1d780) + case L'𝞁': // MATHEMATICAL SANS BOLD SMALL FINAL SIGMA (0x1d781) + case L'𝞂': // MATHEMATICAL SANS BOLD SMALL SIGMA (0x1d782) + case L'𝞃': // MATHEMATICAL SANS BOLD SMALL TAU (0x1d783) + case L'𝞄': // MATHEMATICAL SANS BOLD SMALL UPSILON (0x1d784) + case L'𝞅': // MATHEMATICAL SANS BOLD SMALL PHI (0x1d785) + case L'𝞆': // MATHEMATICAL SANS BOLD SMALL CHI (0x1d786) + case L'𝞇': // MATHEMATICAL SANS BOLD SMALL PSI (0x1d787) + case L'𝞈': // MATHEMATICAL SANS BOLD SMALL OMEGA (0x1d788) + case L'𝞊': // MATHEMATICAL SANS BOLD EPSILON SYMBOL (0x1d78a) + case L'𝞋': // MATHEMATICAL SANS BOLD THETA SYMBOL (0x1d78b) + case L'𝞌': // MATHEMATICAL SANS BOLD KAPPA SYMBOL (0x1d78c) + case L'𝞍': // MATHEMATICAL SANS BOLD PHI SYMBOL (0x1d78d) + case L'𝞎': // MATHEMATICAL SANS BOLD RHO SYMBOL (0x1d78e) + case L'𝞏': // MATHEMATICAL SANS BOLD PI SYMBOL (0x1d78f) + case L'𝞪': // MATHEMATICAL SANS BOLD ITALIC SMALL ALPHA (0x1d7aa) + case L'𝞫': // MATHEMATICAL SANS BOLD ITALIC SMALL BETA (0x1d7ab) + case L'𝞬': // MATHEMATICAL SANS BOLD ITALIC SMALL GAMMA (0x1d7ac) + case L'𝞭': // MATHEMATICAL SANS BOLD ITALIC SMALL DELTA (0x1d7ad) + case L'𝞮': // MATHEMATICAL SANS BOLD ITALIC SMALL EPSILON (0x1d7ae) + case L'𝞯': // MATHEMATICAL SANS BOLD ITALIC SMALL ZETA (0x1d7af) + case L'𝞰': // MATHEMATICAL SANS BOLD ITALIC SMALL ETA (0x1d7b0) + case L'𝞱': // MATHEMATICAL SANS BOLD ITALIC SMALL THETA (0x1d7b1) + case L'𝞲': // MATHEMATICAL SANS BOLD ITALIC SMALL IOTA (0x1d7b2) + case L'𝞳': // MATHEMATICAL SANS BOLD ITALIC SMALL KAPPA (0x1d7b3) + case L'𝞴': // MATHEMATICAL SANS BOLD ITALIC SMALL LAMDA (0x1d7b4) + case L'𝞵': // MATHEMATICAL SANS BOLD ITALIC SMALL MU (0x1d7b5) + case L'𝞶': // MATHEMATICAL SANS BOLD ITALIC SMALL NU (0x1d7b6) + case L'𝞷': // MATHEMATICAL SANS BOLD ITALIC SMALL XI (0x1d7b7) + case L'𝞸': // MATHEMATICAL SANS BOLD ITALIC SMALL OMICRON (0x1d7b8) + case L'𝞹': // MATHEMATICAL SANS BOLD ITALIC SMALL PI (0x1d7b9) + case L'𝞺': // MATHEMATICAL SANS BOLD ITALIC SMALL RHO (0x1d7ba) + case L'𝞻': // MATHEMATICAL SANS BOLD ITALIC SMALL FINAL SIGMA (0x1d7bb) + case L'𝞼': // MATHEMATICAL SANS BOLD ITALIC SMALL SIGMA (0x1d7bc) + case L'𝞽': // MATHEMATICAL SANS BOLD ITALIC SMALL TAU (0x1d7bd) + case L'𝞾': // MATHEMATICAL SANS BOLD ITALIC SMALL UPSILON (0x1d7be) + case L'𝞿': // MATHEMATICAL SANS BOLD ITALIC SMALL PHI (0x1d7bf) + case L'𝟀': // MATHEMATICAL SANS BOLD ITALIC SMALL CHI (0x1d7c0) + case L'𝟁': // MATHEMATICAL SANS BOLD ITALIC SMALL PSI (0x1d7c1) + case L'𝟂': // MATHEMATICAL SANS BOLD ITALIC SMALL OMEGA (0x1d7c2) + case L'𝟄': // MATHEMATICAL SANS BOLD ITALIC EPSILON SYMBOL (0x1d7c4) + case L'𝟅': // MATHEMATICAL SANS BOLD ITALIC THETA SYMBOL (0x1d7c5) + case L'𝟆': // MATHEMATICAL SANS BOLD ITALIC KAPPA SYMBOL (0x1d7c6) + case L'𝟇': // MATHEMATICAL SANS BOLD ITALIC PHI SYMBOL (0x1d7c7) + case L'𝟈': // MATHEMATICAL SANS BOLD ITALIC RHO SYMBOL (0x1d7c8) + case L'𝟉': // MATHEMATICAL SANS BOLD ITALIC PI SYMBOL (0x1d7c9) + case L'𝟋': // MATHEMATICAL BOLD SMALL DIGAMMA (0x1d7cb) + case L'𞤢': // ADLAM SMALL ALIF (0x1e922) + case L'𞤣': // ADLAM SMALL DAALI (0x1e923) + case L'𞤤': // ADLAM SMALL LAAM (0x1e924) + case L'𞤥': // ADLAM SMALL MIIM (0x1e925) + case L'𞤦': // ADLAM SMALL BA (0x1e926) + case L'𞤧': // ADLAM SMALL SINNYIIYHE (0x1e927) + case L'𞤨': // ADLAM SMALL PE (0x1e928) + case L'𞤩': // ADLAM SMALL BHE (0x1e929) + case L'𞤪': // ADLAM SMALL RA (0x1e92a) + case L'𞤫': // ADLAM SMALL E (0x1e92b) + case L'𞤬': // ADLAM SMALL FA (0x1e92c) + case L'𞤭': // ADLAM SMALL I (0x1e92d) + case L'𞤮': // ADLAM SMALL O (0x1e92e) + case L'𞤯': // ADLAM SMALL DHA (0x1e92f) + case L'𞤰': // ADLAM SMALL YHE (0x1e930) + case L'𞤱': // ADLAM SMALL WAW (0x1e931) + case L'𞤲': // ADLAM SMALL NUN (0x1e932) + case L'𞤳': // ADLAM SMALL KAF (0x1e933) + case L'𞤴': // ADLAM SMALL YA (0x1e934) + case L'𞤵': // ADLAM SMALL U (0x1e935) + case L'𞤶': // ADLAM SMALL JIIM (0x1e936) + case L'𞤷': // ADLAM SMALL CHI (0x1e937) + case L'𞤸': // ADLAM SMALL HA (0x1e938) + case L'𞤹': // ADLAM SMALL QAAF (0x1e939) + case L'𞤺': // ADLAM SMALL GA (0x1e93a) + case L'𞤻': // ADLAM SMALL NYA (0x1e93b) + case L'𞤼': // ADLAM SMALL TU (0x1e93c) + case L'𞤽': // ADLAM SMALL NHA (0x1e93d) + case L'𞤾': // ADLAM SMALL VA (0x1e93e) + case L'𞤿': // ADLAM SMALL KHA (0x1e93f) + case L'𞥀': // ADLAM SMALL GBE (0x1e940) + case L'𞥁': // ADLAM SMALL ZAL (0x1e941) + case L'𞥂': // ADLAM SMALL KPO (0x1e942) + case L'𞥃': // ADLAM SMALL SHA (0x1e943) + return 1; + default: + return 0; + } } diff --git a/libc/str/iswprint.c b/libc/str/iswprint.c index 3fd0c2550..bb794f612 100644 --- a/libc/str/iswprint.c +++ b/libc/str/iswprint.c @@ -18,6 +18,10 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswprint(wint_t wc) { - return !iswcntrl(wc); +/** + * Returns nonzero if c is printable. + */ +int iswprint(wint_t c) { + return !((0x00 <= c && c <= 0x1F) || (0x7F <= c && c <= 0x9F) || + (0xFFF9 <= c && c <= 0xFFFB) || c == 0x2028 || c == 0x2029); } diff --git a/libc/str/iswpunct.c b/libc/str/iswpunct.c index 63212cf6d..e0410a885 100644 --- a/libc/str/iswpunct.c +++ b/libc/str/iswpunct.c @@ -18,6 +18,524 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswpunct(wint_t wc) { - return ispunct(wc); +/** + * Returns nonzero if c is punctuation mark. + */ +int iswpunct(wint_t c) { + if (c < 0xa0) { + switch (c) { + case '!': + case '"': + case '#': + case '$': + case '%': + case '&': + case '\'': + case '(': + case ')': + case '*': + case '+': + case ',': + case '-': + case '.': + case '/': + case ':': + case ';': + case '<': + case '=': + case '>': + case '?': + case '@': + case '[': + case '\\': + case ']': + case '^': + case '_': + case '`': + case '{': + case '|': + case '}': + case '~': + return 1; + default: + return 0; + } + } + switch (c) { + case u'¡': // INVERTED EXCLAMATION MARK (0x00a1 Po) + case u'§': // SECTION SIGN (0x00a7 Po) + case u'«': // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK (0x00ab Pi) + case u'¶': // PILCROW SIGN (0x00b6 Po) + case u'·': // MIDDLE DOT (0x00b7 Po) + case u'»': // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK (0x00bb Pf) + case u'¿': // INVERTED QUESTION MARK (0x00bf Po) + case u';': // GREEK QUESTION MARK (0x037e Po) + case u'·': // GREEK ANO TELEIA (0x0387 Po) + case u'՚': // ARMENIAN APOSTROPHE (0x055a Po) + case u'՛': // ARMENIAN EMPHASIS MARK (0x055b Po) + case u'՜': // ARMENIAN EXCLAMATION MARK (0x055c Po) + case u'՝': // ARMENIAN COMMA (0x055d Po) + case u'՞': // ARMENIAN QUESTION MARK (0x055e Po) + case u'՟': // ARMENIAN ABBREVIATION MARK (0x055f Po) + case u'։': // ARMENIAN FULL STOP (0x0589 Po) + case u'֊': // ARMENIAN HYPHEN (0x058a Pd) + case 0x05be: // HEBREW PUNCTUATION MAQAF (0x05be Pd) + case 0x05c0: // HEBREW PUNCTUATION PASEQ (0x05c0 Po) + case 0x05c3: // HEBREW PUNCTUATION SOF PASUQ (0x05c3 Po) + case 0x05c6: // HEBREW PUNCTUATION NUN HAFUKHA (0x05c6 Po) + case 0x05f3: // HEBREW PUNCTUATION GERESH (0x05f3 Po) + case 0x05f4: // HEBREW PUNCTUATION GERSHAYIM (0x05f4 Po) + case 0x0609: // ARABIC-INDIC PER MILLE SIGN (0x0609 Po) + case 0x060a: // ARABIC-INDIC PER TEN THOUSAND SIGN (0x060a Po) + case 0x060c: // ARABIC COMMA (0x060c Po) + case 0x060d: // ARABIC DATE SEPARATOR (0x060d Po) + case 0x061b: // ARABIC SEMICOLON (0x061b Po) + case u'؞': // ARABIC TRIPLE DOT PUNCTUATION MARK (0x061e Po) + case u'؟': // ARABIC QUESTION MARK (0x061f Po) + case u'٪': // ARABIC PERCENT SIGN (0x066a Po) + case u'٫': // ARABIC DECIMAL SEPARATOR (0x066b Po) + case u'٬': // ARABIC THOUSANDS SEPARATOR (0x066c Po) + case u'٭': // ARABIC FIVE POINTED STAR (0x066d Po) + case u'۔': // ARABIC FULL STOP (0x06d4 Po) + case u'߷': // NKO SYMBOL GBAKURUNEN (0x07f7 Po) + case u'߸': // NKO COMMA (0x07f8 Po) + case u'߹': // NKO EXCLAMATION MARK (0x07f9 Po) + case u'।': // DEVANAGARI DANDA (0x0964 Po) + case u'॥': // DEVANAGARI DOUBLE DANDA (0x0965 Po) + case u'॰': // DEVANAGARI ABBREVIATION SIGN (0x0970 Po) + case 0x09fd: // BENGALI ABBREVIATION SIGN (0x09fd Po) + case 0x0a76: // GURMUKHI ABBREVIATION SIGN (0x0a76 Po) + case 0x0af0: // GUJARATI ABBREVIATION SIGN (0x0af0 Po) + case 0x0c77: // TELUGU SIGN SIDDHAM (0x0c77 Po) + case 0x0c84: // KANNADA SIGN SIDDHAM (0x0c84 Po) + case u'෴': // SINHALA PUNCTUATION KUNDDALIYA (0x0df4 Po) + case u'๏': // THAI CHARACTER FONGMAN (0x0e4f Po) + case u'๚': // THAI CHARACTER ANGKHANKHU (0x0e5a Po) + case u'๛': // THAI CHARACTER KHOMUT (0x0e5b Po) + case u'༄': // TIBETAN MARK INITIAL YIG MGO MDUN MA (0x0f04 Po) + case u'༅': // TIBETAN MARK CLOSING YIG MGO SGAB MA (0x0f05 Po) + case u'༆': // TIBETAN MARK CARET YIG MGO PHUR SHAD MA (0x0f06 Po) + case u'༇': // TIBETAN MARK YIG MGO TSHEG SHAD MA (0x0f07 Po) + case u'༈': // TIBETAN MARK SBRUL SHAD (0x0f08 Po) + case u'༉': // TIBETAN MARK BSKUR YIG MGO (0x0f09 Po) + case u'༊': // TIBETAN MARK BKA- SHOG YIG MGO (0x0f0a Po) + case u'་': // TIBETAN MARK INTERSYLLABIC TSHEG (0x0f0b Po) + case u'༌': // TIBETAN MARK DELIMITER TSHEG BSTAR (0x0f0c Po) + case u'།': // TIBETAN MARK SHAD (0x0f0d Po) + case u'༎': // TIBETAN MARK NYIS SHAD (0x0f0e Po) + case u'༏': // TIBETAN MARK TSHEG SHAD (0x0f0f Po) + case u'༐': // TIBETAN MARK NYIS TSHEG SHAD (0x0f10 Po) + case u'༑': // TIBETAN MARK RIN CHEN SPUNGS SHAD (0x0f11 Po) + case u'༒': // TIBETAN MARK RGYA GRAM SHAD (0x0f12 Po) + case u'༔': // TIBETAN MARK GTER TSHEG (0x0f14 Po) + case u'༺': // TIBETAN MARK GUG RTAGS GYON (0x0f3a Ps) + case u'༻': // TIBETAN MARK GUG RTAGS GYAS (0x0f3b Pe) + case u'༼': // TIBETAN MARK ANG KHANG GYON (0x0f3c Ps) + case u'༽': // TIBETAN MARK ANG KHANG GYAS (0x0f3d Pe) + case u'྅': // TIBETAN MARK PALUTA (0x0f85 Po) + case u'࿐': // TIBETAN MARK BSKA- SHOG GI MGO RGYAN (0x0fd0 Po) + case u'࿑': // TIBETAN MARK MNYAM YIG GI MGO RGYAN (0x0fd1 Po) + case u'࿒': // TIBETAN MARK NYIS TSHEG (0x0fd2 Po) + case u'࿓': // TIBETAN MARK INITIAL BRDA RNYING YIG MGO MDUN MA (0x0fd3 Po) + case u'࿔': // TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA (0x0fd4 Po) + case u'࿙': // TIBETAN MARK LEADING MCHAN RTAGS (0x0fd9 Po) + case u'࿚': // TIBETAN MARK TRAILING MCHAN RTAGS (0x0fda Po) + case u'၊': // MYANMAR SIGN LITTLE SECTION (0x104a Po) + case u'။': // MYANMAR SIGN SECTION (0x104b Po) + case u'၌': // MYANMAR SYMBOL LOCATIVE (0x104c Po) + case u'၍': // MYANMAR SYMBOL COMPLETED (0x104d Po) + case u'၎': // MYANMAR SYMBOL AFOREMENTIONED (0x104e Po) + case u'၏': // MYANMAR SYMBOL GENITIVE (0x104f Po) + case u'჻': // GEORGIAN PARAGRAPH SEPARATOR (0x10fb Po) + case u'፠': // ETHIOPIC SECTION MARK (0x1360 Po) + case u'፡': // ETHIOPIC WORDSPACE (0x1361 Po) + case u'።': // ETHIOPIC FULL STOP (0x1362 Po) + case u'፣': // ETHIOPIC COMMA (0x1363 Po) + case u'፤': // ETHIOPIC SEMICOLON (0x1364 Po) + case u'፥': // ETHIOPIC COLON (0x1365 Po) + case u'፦': // ETHIOPIC PREFACE COLON (0x1366 Po) + case u'፧': // ETHIOPIC QUESTION MARK (0x1367 Po) + case u'፨': // ETHIOPIC PARAGRAPH SEPARATOR (0x1368 Po) + case u'᐀': // CANADIAN SYLLABICS HYPHEN (0x1400 Pd) + case u'᙮': // CANADIAN SYLLABICS FULL STOP (0x166e Po) + case u'᚛': // OGHAM FEATHER MARK (0x169b Ps) + case u'᚜': // OGHAM REVERSED FEATHER MARK (0x169c Pe) + case u'᛫': // RUNIC SINGLE PUNCTUATION (0x16eb Po) + case u'᛬': // RUNIC MULTIPLE PUNCTUATION (0x16ec Po) + case u'᛭': // RUNIC CROSS PUNCTUATION (0x16ed Po) + case u'᜵': // PHILIPPINE SINGLE PUNCTUATION (0x1735 Po) + case u'᜶': // PHILIPPINE DOUBLE PUNCTUATION (0x1736 Po) + case u'។': // KHMER SIGN KHAN (0x17d4 Po) + case u'៕': // KHMER SIGN BARIYOOSAN (0x17d5 Po) + case u'៖': // KHMER SIGN CAMNUC PII KUUH (0x17d6 Po) + case u'៘': // KHMER SIGN BEYYAL (0x17d8 Po) + case u'៙': // KHMER SIGN PHNAEK MUAN (0x17d9 Po) + case u'៚': // KHMER SIGN KOOMUUT (0x17da Po) + case u'᠀': // MONGOLIAN BIRGA (0x1800 Po) + case u'᠁': // MONGOLIAN ELLIPSIS (0x1801 Po) + case u'᠂': // MONGOLIAN COMMA (0x1802 Po) + case u'᠃': // MONGOLIAN FULL STOP (0x1803 Po) + case u'᠄': // MONGOLIAN COLON (0x1804 Po) + case u'᠅': // MONGOLIAN FOUR DOTS (0x1805 Po) + case u'᠆': // MONGOLIAN TODO SOFT HYPHEN (0x1806 Pd) + case u'᠇': // MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER (0x1807 Po) + case u'᠈': // MONGOLIAN MANCHU COMMA (0x1808 Po) + case u'᠉': // MONGOLIAN MANCHU FULL STOP (0x1809 Po) + case u'᠊': // MONGOLIAN NIRUGU (0x180a Po) + case u'᥄': // LIMBU EXCLAMATION MARK (0x1944 Po) + case u'᥅': // LIMBU QUESTION MARK (0x1945 Po) + case u'᨞': // BUGINESE PALLAWA (0x1a1e Po) + case u'᨟': // BUGINESE END OF SECTION (0x1a1f Po) + case u'᱾': // OL CHIKI PUNCTUATION MUCAAD (0x1c7e Po) + case u'᱿': // OL CHIKI PUNCTUATION DOUBLE MUCAAD (0x1c7f Po) + case u'‐': // HYPHEN (0x2010 Pd) + case u'‑': // NON-BREAKING HYPHEN (0x2011 Pd) + case u'‒': // FIGURE DASH (0x2012 Pd) + case u'–': // EN DASH (0x2013 Pd) + case u'—': // EM DASH (0x2014 Pd) + case u'―': // HORIZONTAL BAR (0x2015 Pd) + case u'‖': // DOUBLE VERTICAL LINE (0x2016 Po) + case u'‗': // DOUBLE LOW LINE (0x2017 Po) + case u'‘': // LEFT SINGLE QUOTATION MARK (0x2018 Pi) + case u'’': // RIGHT SINGLE QUOTATION MARK (0x2019 Pf) + case u'‚': // SINGLE LOW-9 QUOTATION MARK (0x201a Ps) + case u'‛': // SINGLE HIGH-REVERSED-9 QUOTATION MARK (0x201b Pi) + case u'“': // LEFT DOUBLE QUOTATION MARK (0x201c Pi) + case u'”': // RIGHT DOUBLE QUOTATION MARK (0x201d Pf) + case u'„': // DOUBLE LOW-9 QUOTATION MARK (0x201e Ps) + case u'‟': // DOUBLE HIGH-REVERSED-9 QUOTATION MARK (0x201f Pi) + case u'†': // DAGGER (0x2020 Po) + case u'‡': // DOUBLE DAGGER (0x2021 Po) + case u'•': // BULLET (0x2022 Po) + case u'‣': // TRIANGULAR BULLET (0x2023 Po) + case u'․': // ONE DOT LEADER (0x2024 Po) + case u'‥': // TWO DOT LEADER (0x2025 Po) + case u'…': // HORIZONTAL ELLIPSIS (0x2026 Po) + case u'‧': // HYPHENATION POINT (0x2027 Po) + case u'‰': // PER MILLE SIGN (0x2030 Po) + case u'‱': // PER TEN THOUSAND SIGN (0x2031 Po) + case u'′': // PRIME (0x2032 Po) + case u'″': // DOUBLE PRIME (0x2033 Po) + case u'‴': // TRIPLE PRIME (0x2034 Po) + case u'‵': // REVERSED PRIME (0x2035 Po) + case u'‶': // REVERSED DOUBLE PRIME (0x2036 Po) + case u'‷': // REVERSED TRIPLE PRIME (0x2037 Po) + case u'‸': // CARET (0x2038 Po) + case u'‹': // SINGLE LEFT-POINTING ANGLE QUOTATION MARK (0x2039 Pi) + case u'›': // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK (0x203a Pf) + case u'※': // REFERENCE MARK (0x203b Po) + case u'‼': // DOUBLE EXCLAMATION MARK (0x203c Po) + case u'‽': // INTERROBANG (0x203d Po) + case u'‾': // OVERLINE (0x203e Po) + case u'‿': // UNDERTIE (0x203f Pc) + case u'⁀': // CHARACTER TIE (0x2040 Pc) + case u'⁁': // CARET INSERTION POINT (0x2041 Po) + case u'⁂': // ASTERISM (0x2042 Po) + case u'⁃': // HYPHEN BULLET (0x2043 Po) + case u'⁅': // LEFT SQUARE BRACKET WITH QUILL (0x2045 Ps) + case u'⁆': // RIGHT SQUARE BRACKET WITH QUILL (0x2046 Pe) + case u'⁇': // DOUBLE QUESTION MARK (0x2047 Po) + case u'⁈': // QUESTION EXCLAMATION MARK (0x2048 Po) + case u'⁉': // EXCLAMATION QUESTION MARK (0x2049 Po) + case u'⁊': // TIRONIAN SIGN ET (0x204a Po) + case u'⁋': // REVERSED PILCROW SIGN (0x204b Po) + case u'⁌': // BLACK LEFTWARDS BULLET (0x204c Po) + case u'⁍': // BLACK RIGHTWARDS BULLET (0x204d Po) + case u'⁎': // LOW ASTERISK (0x204e Po) + case u'⁏': // REVERSED SEMICOLON (0x204f Po) + case u'⁐': // CLOSE UP (0x2050 Po) + case u'⁑': // TWO ASTERISKS ALIGNED VERTICALLY (0x2051 Po) + case u'⁓': // SWUNG DASH (0x2053 Po) + case u'⁔': // INVERTED UNDERTIE (0x2054 Pc) + case u'⁕': // FLOWER PUNCTUATION MARK (0x2055 Po) + case u'⁖': // THREE DOT PUNCTUATION (0x2056 Po) + case u'⁗': // QUADRUPLE PRIME (0x2057 Po) + case u'⁘': // FOUR DOT PUNCTUATION (0x2058 Po) + case u'⁙': // FIVE DOT PUNCTUATION (0x2059 Po) + case u'⁚': // TWO DOT PUNCTUATION (0x205a Po) + case u'⁛': // FOUR DOT MARK (0x205b Po) + case u'⁜': // DOTTED CROSS (0x205c Po) + case u'⁝': // TRICOLON (0x205d Po) + case u'⁞': // VERTICAL FOUR DOTS (0x205e Po) + case u'⁽': // SUPERSCRIPT LEFT PARENTHESIS (0x207d Ps) + case u'⁾': // SUPERSCRIPT RIGHT PARENTHESIS (0x207e Pe) + case u'₍': // SUBSCRIPT LEFT PARENTHESIS (0x208d Ps) + case u'₎': // SUBSCRIPT RIGHT PARENTHESIS (0x208e Pe) + case u'⌈': // LEFT CEILING (0x2308 Ps) + case u'⌉': // RIGHT CEILING (0x2309 Pe) + case u'⌊': // LEFT FLOOR (0x230a Ps) + case u'⌋': // RIGHT FLOOR (0x230b Pe) + case u'〈': // LEFT-POINTING ANGLE BRACKET (0x2329 Ps) + case u'〉': // RIGHT-POINTING ANGLE BRACKET (0x232a Pe) + case u'❨': // MEDIUM LEFT PARENTHESIS ORNAMENT (0x2768 Ps) + case u'❩': // MEDIUM RIGHT PARENTHESIS ORNAMENT (0x2769 Pe) + case u'❪': // MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT (0x276a Ps) + case u'❫': // MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT (0x276b Pe) + case u'❬': // MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT (0x276c Ps) + case u'❭': // MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT (0x276d Pe) + case u'❮': // HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT (0x276e Ps) + case u'❯': // HEAVY RIGHT-POINTING ANGLE QUOT MARK ORNAMENT (0x276f Pe) + case u'❰': // HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT (0x2770 Ps) + case u'❱': // HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT (0x2771 Pe) + case u'❲': // LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT (0x2772 Ps) + case u'❳': // LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT (0x2773 Pe) + case u'❴': // MEDIUM LEFT CURLY BRACKET ORNAMENT (0x2774 Ps) + case u'❵': // MEDIUM RIGHT CURLY BRACKET ORNAMENT (0x2775 Pe) + case u'⟅': // LEFT S-SHAPED BAG DELIMITER (0x27c5 Ps) + case u'⟆': // RIGHT S-SHAPED BAG DELIMITER (0x27c6 Pe) + case u'⟦': // MATHEMATICAL LEFT WHITE SQUARE BRACKET (0x27e6 Ps) + case u'⟧': // MATHEMATICAL RIGHT WHITE SQUARE BRACKET (0x27e7 Pe) + case u'⟨': // MATHEMATICAL LEFT ANGLE BRACKET (0x27e8 Ps) + case u'⟩': // MATHEMATICAL RIGHT ANGLE BRACKET (0x27e9 Pe) + case u'⟪': // MATHEMATICAL LEFT DOUBLE ANGLE BRACKET (0x27ea Ps) + case u'⟫': // MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET (0x27eb Pe) + case u'⟬': // MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET (0x27ec Ps) + case u'⟭': // MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET (0x27ed Pe) + case u'⟮': // MATHEMATICAL LEFT FLATTENED PARENTHESIS (0x27ee Ps) + case u'⟯': // MATHEMATICAL RIGHT FLATTENED PARENTHESIS (0x27ef Pe) + case u'⦃': // LEFT WHITE CURLY BRACKET (0x2983 Ps) + case u'⦄': // RIGHT WHITE CURLY BRACKET (0x2984 Pe) + case u'⦅': // LEFT WHITE PARENTHESIS (0x2985 Ps) + case u'⦆': // RIGHT WHITE PARENTHESIS (0x2986 Pe) + case u'⦇': // Z NOTATION LEFT IMAGE BRACKET (0x2987 Ps) + case u'⦈': // Z NOTATION RIGHT IMAGE BRACKET (0x2988 Pe) + case u'⦉': // Z NOTATION LEFT BINDING BRACKET (0x2989 Ps) + case u'⦊': // Z NOTATION RIGHT BINDING BRACKET (0x298a Pe) + case u'⦋': // LEFT SQUARE BRACKET WITH UNDERBAR (0x298b Ps) + case u'⦌': // RIGHT SQUARE BRACKET WITH UNDERBAR (0x298c Pe) + case u'⦍': // LEFT SQUARE BRACKET WITH TICK IN TOP CORNER (0x298d Ps) + case u'⦎': // RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER (0x298e Pe) + case u'⦏': // LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER (0x298f Ps) + case u'⦐': // RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER (0x2990 Pe) + case u'⦑': // LEFT ANGLE BRACKET WITH DOT (0x2991 Ps) + case u'⦒': // RIGHT ANGLE BRACKET WITH DOT (0x2992 Pe) + case u'⦓': // LEFT ARC LESS-THAN BRACKET (0x2993 Ps) + case u'⦔': // RIGHT ARC GREATER-THAN BRACKET (0x2994 Pe) + case u'⦗': // LEFT BLACK TORTOISE SHELL BRACKET (0x2997 Ps) + case u'⦘': // RIGHT BLACK TORTOISE SHELL BRACKET (0x2998 Pe) + case u'⧘': // LEFT WIGGLY FENCE (0x29d8 Ps) + case u'⧙': // RIGHT WIGGLY FENCE (0x29d9 Pe) + case u'⧚': // LEFT DOUBLE WIGGLY FENCE (0x29da Ps) + case u'⧛': // RIGHT DOUBLE WIGGLY FENCE (0x29db Pe) + case u'⧼': // LEFT-POINTING CURVED ANGLE BRACKET (0x29fc Ps) + case u'⧽': // RIGHT-POINTING CURVED ANGLE BRACKET (0x29fd Pe) + case u'⵰': // TIFINAGH SEPARATOR MARK (0x2d70 Po) + case u'⸎': // EDITORIAL CORONIS (0x2e0e Po) + case u'⸏': // PARAGRAPHOS (0x2e0f Po) + case u'⸐': // FORKED PARAGRAPHOS (0x2e10 Po) + case u'⸑': // REVERSED FORKED PARAGRAPHOS (0x2e11 Po) + case u'⸒': // HYPODIASTOLE (0x2e12 Po) + case u'⸓': // DOTTED OBELOS (0x2e13 Po) + case u'⸔': // DOWNWARDS ANCORA (0x2e14 Po) + case u'⸕': // UPWARDS ANCORA (0x2e15 Po) + case u'⸖': // DOTTED RIGHT-POINTING ANGLE (0x2e16 Po) + case u'⸗': // DOUBLE OBLIQUE HYPHEN (0x2e17 Pd) + case u'⸙': // PALM BRANCH (0x2e19 Po) + case u'⸚': // HYPHEN WITH DIAERESIS (0x2e1a Pd) + case u'⸛': // TILDE WITH RING ABOVE (0x2e1b Po) + case u'⸞': // TILDE WITH DOT ABOVE (0x2e1e Po) + case u'⸟': // TILDE WITH DOT BELOW (0x2e1f Po) + case u'⸪': // TWO DOTS OVER ONE DOT PUNCTUATION (0x2e2a Po) + case u'⸫': // ONE DOT OVER TWO DOTS PUNCTUATION (0x2e2b Po) + case u'⸬': // SQUARED FOUR DOT PUNCTUATION (0x2e2c Po) + case u'⸭': // FIVE DOT MARK (0x2e2d Po) + case u'⸮': // REVERSED QUESTION MARK (0x2e2e Po) + case u'⸰': // RING POINT (0x2e30 Po) + case u'⸱': // WORD SEPARATOR MIDDLE DOT (0x2e31 Po) + case u'⸲': // TURNED COMMA (0x2e32 Po) + case u'⸳': // RAISED DOT (0x2e33 Po) + case u'⸴': // RAISED COMMA (0x2e34 Po) + case u'⸵': // TURNED SEMICOLON (0x2e35 Po) + case u'⸶': // DAGGER WITH LEFT GUARD (0x2e36 Po) + case u'⸷': // DAGGER WITH RIGHT GUARD (0x2e37 Po) + case u'⸸': // TURNED DAGGER (0x2e38 Po) + case u'⸹': // TOP HALF SECTION SIGN (0x2e39 Po) + case u'⸺': // TWO-EM DASH (0x2e3a Pd) + case u'⸻': // THREE-EM DASH (0x2e3b Pd) + case u'⸼': // STENOGRAPHIC FULL STOP (0x2e3c Po) + case u'⸽': // VERTICAL SIX DOTS (0x2e3d Po) + case u'⸾': // WIGGLY VERTICAL LINE (0x2e3e Po) + case u'⸿': // CAPITULUM (0x2e3f Po) + case u'⹀': // DOUBLE HYPHEN (0x2e40 Pd) + case u'⹁': // REVERSED COMMA (0x2e41 Po) + case u'⹂': // DOUBLE LOW-REVERSED-9 QUOTATION MARK (0x2e42 Ps) + case u'⹃': // DASH WITH LEFT UPTURN (0x2e43 Po) + case u'⹄': // DOUBLE SUSPENSION MARK (0x2e44 Po) + case u'⹅': // INVERTED LOW KAVYKA (0x2e45 Po) + case u'⹆': // INVERTED LOW KAVYKA WITH KAVYKA ABOVE (0x2e46 Po) + case u'⹇': // LOW KAVYKA (0x2e47 Po) + case u'⹈': // LOW KAVYKA WITH DOT (0x2e48 Po) + case u'⹉': // DOUBLE STACKED COMMA (0x2e49 Po) + case u'⹊': // DOTTED SOLIDUS (0x2e4a Po) + case u'⹋': // TRIPLE DAGGER (0x2e4b Po) + case u'⹌': // MEDIEVAL COMMA (0x2e4c Po) + case u'⹍': // PARAGRAPHUS MARK (0x2e4d Po) + case u'⹎': // PUNCTUS ELEVATUS MARK (0x2e4e Po) + case u'⹏': // CORNISH VERSE DIVIDER (0x2e4f Po) + case u'、': // IDEOGRAPHIC COMMA (0x3001 Po) + case u'。': // IDEOGRAPHIC FULL STOP (0x3002 Po) + case u'〃': // DITTO MARK (0x3003 Po) + case u'〈': // LEFT ANGLE BRACKET (0x3008 Ps) + case u'〉': // RIGHT ANGLE BRACKET (0x3009 Pe) + case u'《': // LEFT DOUBLE ANGLE BRACKET (0x300a Ps) + case u'》': // RIGHT DOUBLE ANGLE BRACKET (0x300b Pe) + case u'「': // LEFT CORNER BRACKET (0x300c Ps) + case u'」': // RIGHT CORNER BRACKET (0x300d Pe) + case u'『': // LEFT WHITE CORNER BRACKET (0x300e Ps) + case u'』': // RIGHT WHITE CORNER BRACKET (0x300f Pe) + case u'【': // LEFT BLACK LENTICULAR BRACKET (0x3010 Ps) + case u'】': // RIGHT BLACK LENTICULAR BRACKET (0x3011 Pe) + case u'〔': // LEFT TORTOISE SHELL BRACKET (0x3014 Ps) + case u'〕': // RIGHT TORTOISE SHELL BRACKET (0x3015 Pe) + case u'〖': // LEFT WHITE LENTICULAR BRACKET (0x3016 Ps) + case u'〗': // RIGHT WHITE LENTICULAR BRACKET (0x3017 Pe) + case u'〘': // LEFT WHITE TORTOISE SHELL BRACKET (0x3018 Ps) + case u'〙': // RIGHT WHITE TORTOISE SHELL BRACKET (0x3019 Pe) + case u'〚': // LEFT WHITE SQUARE BRACKET (0x301a Ps) + case u'〛': // RIGHT WHITE SQUARE BRACKET (0x301b Pe) + case u'〜': // WAVE DASH (0x301c Pd) + case u'〝': // REVERSED DOUBLE PRIME QUOTATION MARK (0x301d Ps) + case u'〞': // DOUBLE PRIME QUOTATION MARK (0x301e Pe) + case u'〟': // LOW DOUBLE PRIME QUOTATION MARK (0x301f Pe) + case u'〰': // WAVY DASH (0x3030 Pd) + case u'〽': // PART ALTERNATION MARK (0x303d Po) + case u'゠': // KATAKANA-HIRAGANA DOUBLE HYPHEN (0x30a0 Pd) + case u'・': // KATAKANA MIDDLE DOT (0x30fb Po) + case u'꓾': // LISU PUNCTUATION COMMA (0xa4fe Po) + case u'꓿': // LISU PUNCTUATION FULL STOP (0xa4ff Po) + case u'꘍': // VAI COMMA (0xa60d Po) + case u'꘎': // VAI FULL STOP (0xa60e Po) + case u'꘏': // VAI QUESTION MARK (0xa60f Po) + case u'꙾': // CYRILLIC KAVYKA (0xa67e Po) + case u'꡴': // PHAGS-PA SINGLE HEAD MARK (0xa874 Po) + case u'꡵': // PHAGS-PA DOUBLE HEAD MARK (0xa875 Po) + case u'꡶': // PHAGS-PA MARK SHAD (0xa876 Po) + case u'꡷': // PHAGS-PA MARK DOUBLE SHAD (0xa877 Po) + case u'꣎': // SAURASHTRA DANDA (0xa8ce Po) + case u'꣏': // SAURASHTRA DOUBLE DANDA (0xa8cf Po) + case u'꣸': // DEVANAGARI SIGN PUSHPIKA (0xa8f8 Po) + case u'꣹': // DEVANAGARI GAP FILLER (0xa8f9 Po) + case u'꣺': // DEVANAGARI CARET (0xa8fa Po) + case u'꣼': // DEVANAGARI SIGN SIDDHAM (0xa8fc Po) + case u'꧁': // JAVANESE LEFT RERENGGAN (0xa9c1 Po) + case u'꧂': // JAVANESE RIGHT RERENGGAN (0xa9c2 Po) + case u'꧃': // JAVANESE PADA ANDAP (0xa9c3 Po) + case u'꧄': // JAVANESE PADA MADYA (0xa9c4 Po) + case u'꧅': // JAVANESE PADA LUHUR (0xa9c5 Po) + case u'꧆': // JAVANESE PADA WINDU (0xa9c6 Po) + case u'꧇': // JAVANESE PADA PANGKAT (0xa9c7 Po) + case u'꧈': // JAVANESE PADA LINGSA (0xa9c8 Po) + case u'꧉': // JAVANESE PADA LUNGSI (0xa9c9 Po) + case u'꧊': // JAVANESE PADA ADEG (0xa9ca Po) + case u'꧋': // JAVANESE PADA ADEG ADEG (0xa9cb Po) + case u'꧌': // JAVANESE PADA PISELEH (0xa9cc Po) + case u'꧍': // JAVANESE TURNED PADA PISELEH (0xa9cd Po) + case u'꧞': // JAVANESE PADA TIRTA TUMETES (0xa9de Po) + case u'꧟': // JAVANESE PADA ISEN-ISEN (0xa9df Po) + case u'꩜': // CHAM PUNCTUATION SPIRAL (0xaa5c Po) + case u'꩝': // CHAM PUNCTUATION DANDA (0xaa5d Po) + case u'꩞': // CHAM PUNCTUATION DOUBLE DANDA (0xaa5e Po) + case u'꩟': // CHAM PUNCTUATION TRIPLE DANDA (0xaa5f Po) + case u'꫞': // TAI VIET SYMBOL HO HOI (0xaade Po) + case u'꫟': // TAI VIET SYMBOL KOI KOI (0xaadf Po) + case u'꫰': // MEETEI MAYEK CHEIKHAN (0xaaf0 Po) + case u'꫱': // MEETEI MAYEK AHANG KHUDAM (0xaaf1 Po) + case u'꯫': // MEETEI MAYEK CHEIKHEI (0xabeb Po) + case u'︐': // PRESENTATION FORM FOR VERTICAL COMMA (0xfe10 Po) + case u'︑': // PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA (0xfe11 Po) + case u'︒': // PRESENTATION FORM FOR VERTICAL IDEO FULL STOP (0xfe12 Po) + case u'︓': // PRESENTATION FORM FOR VERTICAL COLON (0xfe13 Po) + case u'︔': // PRESENTATION FORM FOR VERTICAL SEMICOLON (0xfe14 Po) + case u'︕': // PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK (0xfe15 Po) + case u'︖': // PRESENTATION FORM FOR VERTICAL QUESTION MARK (0xfe16 Po) + case u'︗': // PRESENTATION ... LEFT WHITE LENTICULAR BRACKET (0xfe17 Ps) + case u'︘': // PRESENTATION ... RIGHT WHITE LENTICULAR BRAKCET (0xfe18 Pe) + case u'︙': // PRESENTATION ... VERTICAL HORIZONTAL ELLIPSIS (0xfe19 Po) + case u'︰': // PRESENTATION FORM FOR VERTICAL TWO DOT LEADER (0xfe30 Po) + case u'︱': // PRESENTATION FORM FOR VERTICAL EM DASH (0xfe31 Pd) + case u'︲': // PRESENTATION FORM FOR VERTICAL EN DASH (0xfe32 Pd) + case u'︳': // PRESENTATION FORM FOR VERTICAL LOW LINE (0xfe33 Pc) + case u'︴': // PRESENTATION FORM FOR VERTICAL WAVY LOW LINE (0xfe34 Pc) + case u'︵': // PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS (0xfe35 Ps) + case u'︶': // PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS (0xfe36 Pe) + case u'︷': // PRESENTATION ... VERTICAL LEFT CURLY BRACKET (0xfe37 Ps) + case u'︸': // PRESENTATION ... VERTICAL RIGHT CURLY BRACKET (0xfe38 Pe) + case u'︹': // PRESENTATION ... LEFT TORTOISE SHELL BRACKET (0xfe39 Ps) + case u'︺': // PRESENTATION ... RIGHT TORTOISE SHELL BRACKET (0xfe3a Pe) + case u'︻': // PRESENTATION ... LEFT BLACK LENTICULAR BRACKET (0xfe3b Ps) + case u'︼': // PRESENTATION ... RIGHT BLACK LENTICULAR BRACKET (0xfe3c Pe) + case u'︽': // PRESENTATION ... LEFT DOUBLE ANGLE BRACKET (0xfe3d Ps) + case u'︾': // PRESENTATION ... RIGHT DOUBLE ANGLE BRACKET (0xfe3e Pe) + case u'︿': // PRESENTATION ... LEFT ANGLE BRACKET (0xfe3f Ps) + case u'﹀': // PRESENTATION ... RIGHT ANGLE BRACKET (0xfe40 Pe) + case u'﹁': // PRESENTATION ... LEFT CORNER BRACKET (0xfe41 Ps) + case u'﹂': // PRESENTATION ... RIGHT CORNER BRACKET (0xfe42 Pe) + case u'﹃': // PRESENTATION ... LEFT WHITE CORNER BRACKET (0xfe43 Ps) + case u'﹄': // PRESENTATION ... RIGHT WHITE CORNER BRACKET Pe) + case u'﹅': // SESAME DOT (0xfe45 Po) + case u'﹆': // WHITE SESAME DOT (0xfe46 Po) + case u'﹇': // PRESENTATION ... VERTICAL LEFT SQUARE BRACKET (0xfe47 Ps) + case u'﹈': // PRESENTATION ... VERTICAL RIGHT SQUARE BRACKET (0xfe48 Pe) + case u'﹉': // DASHED OVERLINE (0xfe49 Po) + case u'﹊': // CENTRELINE OVERLINE (0xfe4a Po) + case u'﹋': // WAVY OVERLINE (0xfe4b Po) + case u'﹌': // DOUBLE WAVY OVERLINE (0xfe4c Po) + case u'﹍': // DASHED LOW LINE (0xfe4d Pc) + case u'﹎': // CENTRELINE LOW LINE (0xfe4e Pc) + case u'﹏': // WAVY LOW LINE (0xfe4f Pc) + case u'﹐': // SMALL COMMA (0xfe50 Po) + case u'﹑': // SMALL IDEOGRAPHIC COMMA (0xfe51 Po) + case u'﹒': // SMALL FULL STOP (0xfe52 Po) + case u'﹔': // SMALL SEMICOLON (0xfe54 Po) + case u'﹕': // SMALL COLON (0xfe55 Po) + case u'﹖': // SMALL QUESTION MARK (0xfe56 Po) + case u'﹗': // SMALL EXCLAMATION MARK (0xfe57 Po) + case u'﹘': // SMALL EM DASH (0xfe58 Pd) + case u'﹙': // SMALL LEFT PARENTHESIS (0xfe59 Ps) + case u'﹚': // SMALL RIGHT PARENTHESIS (0xfe5a Pe) + case u'﹛': // SMALL LEFT CURLY BRACKET (0xfe5b Ps) + case u'﹜': // SMALL RIGHT CURLY BRACKET (0xfe5c Pe) + case u'﹝': // SMALL LEFT TORTOISE SHELL BRACKET (0xfe5d Ps) + case u'﹞': // SMALL RIGHT TORTOISE SHELL BRACKET (0xfe5e Pe) + case u'﹟': // SMALL NUMBER SIGN (0xfe5f Po) + case u'﹠': // SMALL AMPERSAND (0xfe60 Po) + case u'﹡': // SMALL ASTERISK (0xfe61 Po) + case u'﹣': // SMALL HYPHEN-MINUS (0xfe63 Pd) + case u'﹨': // SMALL REVERSE SOLIDUS (0xfe68 Po) + case u'﹪': // SMALL PERCENT SIGN (0xfe6a Po) + case u'﹫': // SMALL COMMERCIAL AT (0xfe6b Po) + case u'！': // FULLWIDTH EXCLAMATION MARK (0xff01 Po) + case u'＂': // FULLWIDTH QUOTATION MARK (0xff02 Po) + case u'＃': // FULLWIDTH NUMBER SIGN (0xff03 Po) + case u'％': // FULLWIDTH PERCENT SIGN (0xff05 Po) + case u'＆': // FULLWIDTH AMPERSAND (0xff06 Po) + case u'＇': // FULLWIDTH APOSTROPHE (0xff07 Po) + case u'（': // FULLWIDTH LEFT PARENTHESIS (0xff08 Ps) + case u'）': // FULLWIDTH RIGHT PARENTHESIS (0xff09 Pe) + case u'＊': // FULLWIDTH ASTERISK (0xff0a Po) + case u'，': // FULLWIDTH COMMA (0xff0c Po) + case u'－': // FULLWIDTH HYPHEN-MINUS (0xff0d Pd) + case u'．': // FULLWIDTH FULL STOP (0xff0e Po) + case u'／': // FULLWIDTH SOLIDUS (0xff0f Po) + case u'：': // FULLWIDTH COLON (0xff1a Po) + case u'；': // FULLWIDTH SEMICOLON (0xff1b Po) + case u'？': // FULLWIDTH QUESTION MARK (0xff1f Po) + case u'＠': // FULLWIDTH COMMERCIAL AT (0xff20 Po) + case u'［': // FULLWIDTH LEFT SQUARE BRACKET (0xff3b Ps) + case u'＼': // FULLWIDTH REVERSE SOLIDUS (0xff3c Po) + case u'］': // FULLWIDTH RIGHT SQUARE BRACKET (0xff3d Pe) + case u'＿': // FULLWIDTH LOW LINE (0xff3f Pc) + case u'｛': // FULLWIDTH LEFT CURLY BRACKET (0xff5b Ps) + case u'｝': // FULLWIDTH RIGHT CURLY BRACKET (0xff5d Pe) + case u'｟': // FULLWIDTH LEFT WHITE PARENTHESIS (0xff5f Ps) + case u'｠': // FULLWIDTH RIGHT WHITE PARENTHESIS (0xff60 Pe) + case u'｡': // HALFWIDTH IDEOGRAPHIC FULL STOP (0xff61 Po) + case u'｢': // HALFWIDTH LEFT CORNER BRACKET (0xff62 Ps) + case u'｣': // HALFWIDTH RIGHT CORNER BRACKET (0xff63 Pe) + case u'､': // HALFWIDTH IDEOGRAPHIC COMMA (0xff64 Po) + case u'･': // HALFWIDTH KATAKANA MIDDLE DOT (0xff65 Po) + return 1; + default: + return 0; + } } diff --git a/libc/str/iswspace.c b/libc/str/iswspace.c index 218f04835..b1b08cd7f 100644 --- a/libc/str/iswspace.c +++ b/libc/str/iswspace.c @@ -18,6 +18,39 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswspace(wint_t wc) { - return isspace(wc); +/** + * Returns nonzero if c is space character. + * + * We define this as invisible characters which move the cursor. That + * means `\t\r\n\f\v` and unicodes whose category begins with `Z` but + * not ogham since it's not invisible and non-breaking spaces neither + * since they're not invisible to emacs users. + */ +int iswspace(wint_t c) { + switch (c) { + case '\t': // CHARACTER TABULATION + case '\n': // LINE FEED + case '\f': // FORM FEED + case '\v': // LINE TABULATION + case '\r': // CARRIAGE RETURN + case ' ': // SPACE + case 0x2000: // EN QUAD (Zs) + case 0x2001: // EM QUAD (Zs) + case 0x2002: // EN SPACE (Zs) + case 0x2003: // EM SPACE (Zs) + case 0x2004: // THREE-PER-EM SPACE (Zs) + case 0x2005: // FOUR-PER-EM SPACE (Zs) + case 0x2006: // SIX-PER-EM SPACE (Zs) + case 0x2007: // FIGURE SPACE (Zs) + case 0x2008: // PUNCTUATION SPACE (Zs) + case 0x2009: // THIN SPACE (Zs) + case 0x200a: // HAIR SPACE (Zs) + case 0x2028: // LINE SEPARATOR (Zl) + case 0x2029: // PARAGRAPH SEPARATOR (Zp) + case 0x205f: // MEDIUM MATHEMATICAL SPACE (Zs) + case 0x3000: // IDEOGRAPHIC SPACE (Zs) + return 1; + default: + return 0; + } } diff --git a/libc/str/iswupper.c b/libc/str/iswupper.c index e40e63d74..21ff22af9 100644 --- a/libc/str/iswupper.c +++ b/libc/str/iswupper.c @@ -18,6 +18,1776 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswupper(wint_t wc) { - return isupper(wc); +/** + * Returns nonzero if c is uppercase letter. + */ +int iswupper(wint_t c) { + if (isascii(c)) return isupper(c); + switch (c) { + case u'À': // LATIN CAPITAL A W/ GRAVE 00c0 + case u'Á': // LATIN CAPITAL A W/ ACUTE 00c1 + case u'Â': // LATIN CAPITAL A W/ CIRCUMFLEX 00c2 + case u'Ã': // LATIN CAPITAL A W/ TILDE 00c3 + case u'Ä': // LATIN CAPITAL A W/ DIAERESIS 00c4 + case u'Å': // LATIN CAPITAL A W/ RING ABOVE 00c5 + case u'Æ': // LATIN CAPITAL AE 00c6 + case u'Ç': // LATIN CAPITAL C W/ CEDILLA 00c7 + case u'È': // LATIN CAPITAL E W/ GRAVE 00c8 + case u'É': // LATIN CAPITAL E W/ ACUTE 00c9 + case u'Ê': // LATIN CAPITAL E W/ CIRCUMFLEX 00ca + case u'Ë': // LATIN CAPITAL E W/ DIAERESIS 00cb + case u'Ì': // LATIN CAPITAL I W/ GRAVE 00cc + case u'Í': // LATIN CAPITAL I W/ ACUTE 00cd + case u'Î': // LATIN CAPITAL I W/ CIRCUMFLEX 00ce + case u'Ï': // LATIN CAPITAL I W/ DIAERESIS 00cf + case u'Ð': // LATIN CAPITAL ETH 00d0 + case u'Ñ': // LATIN CAPITAL N W/ TILDE 00d1 + case u'Ò': // LATIN CAPITAL O W/ GRAVE 00d2 + case u'Ó': // LATIN CAPITAL O W/ ACUTE 00d3 + case u'Ô': // LATIN CAPITAL O W/ CIRCUMFLEX 00d4 + case u'Õ': // LATIN CAPITAL O W/ TILDE 00d5 + case u'Ö': // LATIN CAPITAL O W/ DIAERESIS 00d6 + case u'Ø': // LATIN CAPITAL O W/ STROKE 00d8 + case u'Ù': // LATIN CAPITAL U W/ GRAVE 00d9 + case u'Ú': // LATIN CAPITAL U W/ ACUTE 00da + case u'Û': // LATIN CAPITAL U W/ CIRCUMFLEX 00db + case u'Ü': // LATIN CAPITAL U W/ DIAERESIS 00dc + case u'Ý': // LATIN CAPITAL Y W/ ACUTE 00dd + case u'Þ': // LATIN CAPITAL THORN 00de + case u'Ā': // LATIN CAPITAL A W/ MACRON 0100 + case u'Ă': // LATIN CAPITAL A W/ BREVE 0102 + case u'Ą': // LATIN CAPITAL A W/ OGONEK 0104 + case u'Ć': // LATIN CAPITAL C W/ ACUTE 0106 + case u'Ĉ': // LATIN CAPITAL C W/ CIRCUMFLEX 0108 + case u'Ċ': // LATIN CAPITAL C W/ DOT ABOVE 010a + case u'Č': // LATIN CAPITAL C W/ CARON 010c + case u'Ď': // LATIN CAPITAL D W/ CARON 010e + case u'Đ': // LATIN CAPITAL D W/ STROKE 0110 + case u'Ē': // LATIN CAPITAL E W/ MACRON 0112 + case u'Ĕ': // LATIN CAPITAL E W/ BREVE 0114 + case u'Ė': // LATIN CAPITAL E W/ DOT ABOVE 0116 + case u'Ę': // LATIN CAPITAL E W/ OGONEK 0118 + case u'Ě': // LATIN CAPITAL E W/ CARON 011a + case u'Ĝ': // LATIN CAPITAL G W/ CIRCUMFLEX 011c + case u'Ğ': // LATIN CAPITAL G W/ BREVE 011e + case u'Ġ': // LATIN CAPITAL G W/ DOT ABOVE 0120 + case u'Ģ': // LATIN CAPITAL G W/ CEDILLA 0122 + case u'Ĥ': // LATIN CAPITAL H W/ CIRCUMFLEX 0124 + case u'Ħ': // LATIN CAPITAL H W/ STROKE 0126 + case u'Ĩ': // LATIN CAPITAL I W/ TILDE 0128 + case u'Ī': // LATIN CAPITAL I W/ MACRON 012a + case u'Ĭ': // LATIN CAPITAL I W/ BREVE 012c + case u'Į': // LATIN CAPITAL I W/ OGONEK 012e + case u'İ': // LATIN CAPITAL I W/ DOT ABOVE 0130 + case u'IJ': // LATIN CAPITAL LIGATURE IJ 0132 + case u'Ĵ': // LATIN CAPITAL J W/ CIRCUMFLEX 0134 + case u'Ķ': // LATIN CAPITAL K W/ CEDILLA 0136 + case u'Ĺ': // LATIN CAPITAL L W/ ACUTE 0139 + case u'Ļ': // LATIN CAPITAL L W/ CEDILLA 013b + case u'Ľ': // LATIN CAPITAL L W/ CARON 013d + case u'Ŀ': // LATIN CAPITAL L W/ MIDDLE DOT 013f + case u'Ł': // LATIN CAPITAL L W/ STROKE 0141 + case u'Ń': // LATIN CAPITAL N W/ ACUTE 0143 + case u'Ņ': // LATIN CAPITAL N W/ CEDILLA 0145 + case u'Ň': // LATIN CAPITAL N W/ CARON 0147 + case u'Ŋ': // LATIN CAPITAL ENG 014a + case u'Ō': // LATIN CAPITAL O W/ MACRON 014c + case u'Ŏ': // LATIN CAPITAL O W/ BREVE 014e + case u'Ő': // LATIN CAPITAL O W/ DOUBLE ACUTE 0150 + case u'Œ': // LATIN CAPITAL LIGATURE OE 0152 + case u'Ŕ': // LATIN CAPITAL R W/ ACUTE 0154 + case u'Ŗ': // LATIN CAPITAL R W/ CEDILLA 0156 + case u'Ř': // LATIN CAPITAL R W/ CARON 0158 + case u'Ś': // LATIN CAPITAL S W/ ACUTE 015a + case u'Ŝ': // LATIN CAPITAL S W/ CIRCUMFLEX 015c + case u'Ş': // LATIN CAPITAL S W/ CEDILLA 015e + case u'Š': // LATIN CAPITAL S W/ CARON 0160 + case u'Ţ': // LATIN CAPITAL T W/ CEDILLA 0162 + case u'Ť': // LATIN CAPITAL T W/ CARON 0164 + case u'Ŧ': // LATIN CAPITAL T W/ STROKE 0166 + case u'Ũ': // LATIN CAPITAL U W/ TILDE 0168 + case u'Ū': // LATIN CAPITAL U W/ MACRON 016a + case u'Ŭ': // LATIN CAPITAL U W/ BREVE 016c + case u'Ů': // LATIN CAPITAL U W/ RING ABOVE 016e + case u'Ű': // LATIN CAPITAL U W/ DOUBLE ACUTE 0170 + case u'Ų': // LATIN CAPITAL U W/ OGONEK 0172 + case u'Ŵ': // LATIN CAPITAL W W/ CIRCUMFLEX 0174 + case u'Ŷ': // LATIN CAPITAL Y W/ CIRCUMFLEX 0176 + case u'Ÿ': // LATIN CAPITAL Y W/ DIAERESIS 0178 + case u'Ź': // LATIN CAPITAL Z W/ ACUTE 0179 + case u'Ż': // LATIN CAPITAL Z W/ DOT ABOVE 017b + case u'Ž': // LATIN CAPITAL Z W/ CARON 017d + case u'Ɓ': // LATIN CAPITAL B W/ HOOK 0181 + case u'Ƃ': // LATIN CAPITAL B W/ TOPBAR 0182 + case u'Ƅ': // LATIN CAPITAL TONE SIX 0184 + case u'Ɔ': // LATIN CAPITAL OPEN O 0186 + case u'Ƈ': // LATIN CAPITAL C W/ HOOK 0187 + case u'Ɖ': // LATIN CAPITAL AFRICAN D 0189 + case u'Ɗ': // LATIN CAPITAL D W/ HOOK 018a + case u'Ƌ': // LATIN CAPITAL D W/ TOPBAR 018b + case u'Ǝ': // LATIN CAPITAL REVERSED E 018e + case u'Ə': // LATIN CAPITAL SCHWA 018f + case u'Ɛ': // LATIN CAPITAL OPEN E 0190 + case u'Ƒ': // LATIN CAPITAL F W/ HOOK 0191 + case u'Ɠ': // LATIN CAPITAL G W/ HOOK 0193 + case u'Ɣ': // LATIN CAPITAL GAMMA 0194 + case u'Ɩ': // LATIN CAPITAL IOTA 0196 + case u'Ɨ': // LATIN CAPITAL I W/ STROKE 0197 + case u'Ƙ': // LATIN CAPITAL K W/ HOOK 0198 + case u'Ɯ': // LATIN CAPITAL TURNED M 019c + case u'Ɲ': // LATIN CAPITAL N W/ LEFT HOOK 019d + case u'Ɵ': // LATIN CAPITAL O W/ MIDDLE TILDE 019f + case u'Ơ': // LATIN CAPITAL O W/ HORN 01a0 + case u'Ƣ': // LATIN CAPITAL OI 01a2 + case u'Ƥ': // LATIN CAPITAL P W/ HOOK 01a4 + case u'Ʀ': // LATIN LETTER YR 01a6 + case u'Ƨ': // LATIN CAPITAL TONE TWO 01a7 + case u'Ʃ': // LATIN CAPITAL ESH 01a9 + case u'Ƭ': // LATIN CAPITAL T W/ HOOK 01ac + case u'Ʈ': // LATIN CAPITAL T W/ RETROFLEX HOOK 01ae + case u'Ư': // LATIN CAPITAL U W/ HORN 01af + case u'Ʊ': // LATIN CAPITAL UPSILON 01b1 + case u'Ʋ': // LATIN CAPITAL V W/ HOOK 01b2 + case u'Ƴ': // LATIN CAPITAL Y W/ HOOK 01b3 + case u'Ƶ': // LATIN CAPITAL Z W/ STROKE 01b5 + case u'Ʒ': // LATIN CAPITAL EZH 01b7 + case u'Ƹ': // LATIN CAPITAL EZH REVERSED 01b8 + case u'Ƽ': // LATIN CAPITAL TONE FIVE 01bc + case u'DŽ': // LATIN CAPITAL DZ W/ CARON 01c4 + case u'LJ': // LATIN CAPITAL LJ 01c7 + case u'NJ': // LATIN CAPITAL NJ 01ca + case u'Ǎ': // LATIN CAPITAL A W/ CARON 01cd + case u'Ǐ': // LATIN CAPITAL I W/ CARON 01cf + case u'Ǒ': // LATIN CAPITAL O W/ CARON 01d1 + case u'Ǔ': // LATIN CAPITAL U W/ CARON 01d3 + case u'Ǖ': // LATIN CAPITAL U W/ DIAERESIS AND MACRON 01d5 + case u'Ǘ': // LATIN CAPITAL U W/ DIAERESIS AND ACUTE 01d7 + case u'Ǚ': // LATIN CAPITAL U W/ DIAERESIS AND CARON 01d9 + case u'Ǜ': // LATIN CAPITAL U W/ DIAERESIS AND GRAVE 01db + case u'Ǟ': // LATIN CAPITAL A W/ DIAERESIS AND MACRON 01de + case u'Ǡ': // LATIN CAPITAL A W/ DOT ABOVE AND MACRON 01e0 + case u'Ǣ': // LATIN CAPITAL AE W/ MACRON 01e2 + case u'Ǥ': // LATIN CAPITAL G W/ STROKE 01e4 + case u'Ǧ': // LATIN CAPITAL G W/ CARON 01e6 + case u'Ǩ': // LATIN CAPITAL K W/ CARON 01e8 + case u'Ǫ': // LATIN CAPITAL O W/ OGONEK 01ea + case u'Ǭ': // LATIN CAPITAL O W/ OGONEK AND MACRON 01ec + case u'Ǯ': // LATIN CAPITAL EZH W/ CARON 01ee + case u'DZ': // LATIN CAPITAL DZ 01f1 + case u'Ǵ': // LATIN CAPITAL G W/ ACUTE 01f4 + case u'Ƕ': // LATIN CAPITAL HWAIR 01f6 + case u'Ƿ': // LATIN CAPITAL WYNN 01f7 + case u'Ǹ': // LATIN CAPITAL N W/ GRAVE 01f8 + case u'Ǻ': // LATIN CAPITAL A W/ RING ABOVE AND ACUTE 01fa + case u'Ǽ': // LATIN CAPITAL AE W/ ACUTE 01fc + case u'Ǿ': // LATIN CAPITAL O W/ STROKE AND ACUTE 01fe + case u'Ȁ': // LATIN CAPITAL A W/ DOUBLE GRAVE 0200 + case u'Ȃ': // LATIN CAPITAL A W/ INVERTED BREVE 0202 + case u'Ȅ': // LATIN CAPITAL E W/ DOUBLE GRAVE 0204 + case u'Ȇ': // LATIN CAPITAL E W/ INVERTED BREVE 0206 + case u'Ȉ': // LATIN CAPITAL I W/ DOUBLE GRAVE 0208 + case u'Ȋ': // LATIN CAPITAL I W/ INVERTED BREVE 020a + case u'Ȍ': // LATIN CAPITAL O W/ DOUBLE GRAVE 020c + case u'Ȏ': // LATIN CAPITAL O W/ INVERTED BREVE 020e + case u'Ȑ': // LATIN CAPITAL R W/ DOUBLE GRAVE 0210 + case u'Ȓ': // LATIN CAPITAL R W/ INVERTED BREVE 0212 + case u'Ȕ': // LATIN CAPITAL U W/ DOUBLE GRAVE 0214 + case u'Ȗ': // LATIN CAPITAL U W/ INVERTED BREVE 0216 + case u'Ș': // LATIN CAPITAL S W/ COMMA BELOW 0218 + case u'Ț': // LATIN CAPITAL T W/ COMMA BELOW 021a + case u'Ȝ': // LATIN CAPITAL YOGH 021c + case u'Ȟ': // LATIN CAPITAL H W/ CARON 021e + case u'Ƞ': // LATIN CAPITAL N W/ LONG RIGHT LEG 0220 + case u'Ȣ': // LATIN CAPITAL OU 0222 + case u'Ȥ': // LATIN CAPITAL Z W/ HOOK 0224 + case u'Ȧ': // LATIN CAPITAL A W/ DOT ABOVE 0226 + case u'Ȩ': // LATIN CAPITAL E W/ CEDILLA 0228 + case u'Ȫ': // LATIN CAPITAL O W/ DIAERESIS AND MACRON 022a + case u'Ȭ': // LATIN CAPITAL O W/ TILDE AND MACRON 022c + case u'Ȯ': // LATIN CAPITAL O W/ DOT ABOVE 022e + case u'Ȱ': // LATIN CAPITAL O W/ DOT ABOVE AND MACRON 0230 + case u'Ȳ': // LATIN CAPITAL Y W/ MACRON 0232 + case u'Ⱥ': // LATIN CAPITAL A W/ STROKE 023a + case u'Ȼ': // LATIN CAPITAL C W/ STROKE 023b + case u'Ƚ': // LATIN CAPITAL L W/ BAR 023d + case u'Ⱦ': // LATIN CAPITAL T W/ DIAGONAL STROKE 023e + case u'Ɂ': // LATIN CAPITAL GLOTTAL STOP 0241 + case u'Ƀ': // LATIN CAPITAL B W/ STROKE 0243 + case u'Ʉ': // LATIN CAPITAL U BAR 0244 + case u'Ʌ': // LATIN CAPITAL TURNED V 0245 + case u'Ɇ': // LATIN CAPITAL E W/ STROKE 0246 + case u'Ɉ': // LATIN CAPITAL J W/ STROKE 0248 + case u'Ɋ': // LATIN CAPITAL SMALL Q W/ HOOK TAIL 024a + case u'Ɍ': // LATIN CAPITAL R W/ STROKE 024c + case u'Ɏ': // LATIN CAPITAL Y W/ STROKE 024e + case u'Ͱ': // GREEK CAPITAL HETA 0370 + case u'Ͳ': // GREEK CAPITAL ARCHAIC SAMPI 0372 + case u'Ͷ': // GREEK CAPITAL PAMPHYLIAN DIGAMMA 0376 + case u'Ϳ': // GREEK CAPITAL YOT 037f + case u'Ά': // GREEK CAPITAL ALPHA W/ TONOS 0386 + case u'Έ': // GREEK CAPITAL EPSILON W/ TONOS 0388 + case u'Ή': // GREEK CAPITAL ETA W/ TONOS 0389 + case u'Ί': // GREEK CAPITAL IOTA W/ TONOS 038a + case u'Ό': // GREEK CAPITAL OMICRON W/ TONOS 038c + case u'Ύ': // GREEK CAPITAL UPSILON W/ TONOS 038e + case u'Ώ': // GREEK CAPITAL OMEGA W/ TONOS 038f + case u'Α': // GREEK CAPITAL ALPHA 0391 + case u'Β': // GREEK CAPITAL BETA 0392 + case u'Γ': // GREEK CAPITAL GAMMA 0393 + case u'Δ': // GREEK CAPITAL DELTA 0394 + case u'Ε': // GREEK CAPITAL EPSILON 0395 + case u'Ζ': // GREEK CAPITAL ZETA 0396 + case u'Η': // GREEK CAPITAL ETA 0397 + case u'Θ': // GREEK CAPITAL THETA 0398 + case u'Ι': // GREEK CAPITAL IOTA 0399 + case u'Κ': // GREEK CAPITAL KAPPA 039a + case u'Λ': // GREEK CAPITAL LAMDA 039b + case u'Μ': // GREEK CAPITAL MU 039c + case u'Ν': // GREEK CAPITAL NU 039d + case u'Ξ': // GREEK CAPITAL XI 039e + case u'Ο': // GREEK CAPITAL OMICRON 039f + case u'Π': // GREEK CAPITAL PI 03a0 + case u'Ρ': // GREEK CAPITAL RHO 03a1 + case u'Σ': // GREEK CAPITAL SIGMA 03a3 + case u'Τ': // GREEK CAPITAL TAU 03a4 + case u'Υ': // GREEK CAPITAL UPSILON 03a5 + case u'Φ': // GREEK CAPITAL PHI 03a6 + case u'Χ': // GREEK CAPITAL CHI 03a7 + case u'Ψ': // GREEK CAPITAL PSI 03a8 + case u'Ω': // GREEK CAPITAL OMEGA 03a9 + case u'Ϊ': // GREEK CAPITAL IOTA W/ DIALYTIKA 03aa + case u'Ϋ': // GREEK CAPITAL UPSILON W/ DIALYTIKA 03ab + case u'Ϗ': // GREEK CAPITAL KAI SYMBOL 03cf + case u'ϒ': // GREEK UPSILON W/ HOOK SYMBOL 03d2 + case u'ϓ': // GREEK UPSILON W/ ACUTE AND HOOK SYMBOL 03d3 + case u'ϔ': // GREEK UPSILON W/ DIAERESIS AND HOOK SYMBOL 03d4 + case u'Ϙ': // GREEK LETTER ARCHAIC KOPPA 03d8 + case u'Ϛ': // GREEK LETTER STIGMA 03da + case u'Ϝ': // GREEK LETTER DIGAMMA 03dc + case u'Ϟ': // GREEK LETTER KOPPA 03de + case u'Ϡ': // GREEK LETTER SAMPI 03e0 + case u'Ϣ': // COPTIC CAPITAL SHEI 03e2 + case u'Ϥ': // COPTIC CAPITAL FEI 03e4 + case u'Ϧ': // COPTIC CAPITAL KHEI 03e6 + case u'Ϩ': // COPTIC CAPITAL HORI 03e8 + case u'Ϫ': // COPTIC CAPITAL GANGIA 03ea + case u'Ϭ': // COPTIC CAPITAL SHIMA 03ec + case u'Ϯ': // COPTIC CAPITAL DEI 03ee + case u'ϴ': // GREEK CAPITAL THETA SYMBOL 03f4 + case u'Ϸ': // GREEK CAPITAL SHO 03f7 + case u'Ϲ': // GREEK CAPITAL LUNATE SIGMA SYMBOL 03f9 + case u'Ϻ': // GREEK CAPITAL SAN 03fa + case u'Ͻ': // GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL 03fd + case u'Ͼ': // GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL 03fe + case u'Ͽ': // GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL 03ff + case u'Ѐ': // CYRILLIC CAPITAL IE W/ GRAVE 0400 + case u'Ё': // CYRILLIC CAPITAL IO 0401 + case u'Ђ': // CYRILLIC CAPITAL DJE 0402 + case u'Ѓ': // CYRILLIC CAPITAL GJE 0403 + case u'Є': // CYRILLIC CAPITAL UKRAINIAN IE 0404 + case u'Ѕ': // CYRILLIC CAPITAL DZE 0405 + case u'І': // CYRILLIC CAPITAL BYELORUSSIAN-UKRAINIAN I 0406 + case u'Ї': // CYRILLIC CAPITAL YI 0407 + case u'Ј': // CYRILLIC CAPITAL JE 0408 + case u'Љ': // CYRILLIC CAPITAL LJE 0409 + case u'Њ': // CYRILLIC CAPITAL NJE 040a + case u'Ћ': // CYRILLIC CAPITAL TSHE 040b + case u'Ќ': // CYRILLIC CAPITAL KJE 040c + case u'Ѝ': // CYRILLIC CAPITAL I W/ GRAVE 040d + case u'Ў': // CYRILLIC CAPITAL SHORT U 040e + case u'Џ': // CYRILLIC CAPITAL DZHE 040f + case u'А': // CYRILLIC CAPITAL A 0410 + case u'Б': // CYRILLIC CAPITAL BE 0411 + case u'В': // CYRILLIC CAPITAL VE 0412 + case u'Г': // CYRILLIC CAPITAL GHE 0413 + case u'Д': // CYRILLIC CAPITAL DE 0414 + case u'Е': // CYRILLIC CAPITAL IE 0415 + case u'Ж': // CYRILLIC CAPITAL ZHE 0416 + case u'З': // CYRILLIC CAPITAL ZE 0417 + case u'И': // CYRILLIC CAPITAL I 0418 + case u'Й': // CYRILLIC CAPITAL SHORT I 0419 + case u'К': // CYRILLIC CAPITAL KA 041a + case u'Л': // CYRILLIC CAPITAL EL 041b + case u'М': // CYRILLIC CAPITAL EM 041c + case u'Н': // CYRILLIC CAPITAL EN 041d + case u'О': // CYRILLIC CAPITAL O 041e + case u'П': // CYRILLIC CAPITAL PE 041f + case u'Р': // CYRILLIC CAPITAL ER 0420 + case u'С': // CYRILLIC CAPITAL ES 0421 + case u'Т': // CYRILLIC CAPITAL TE 0422 + case u'У': // CYRILLIC CAPITAL U 0423 + case u'Ф': // CYRILLIC CAPITAL EF 0424 + case u'Х': // CYRILLIC CAPITAL HA 0425 + case u'Ц': // CYRILLIC CAPITAL TSE 0426 + case u'Ч': // CYRILLIC CAPITAL CHE 0427 + case u'Ш': // CYRILLIC CAPITAL SHA 0428 + case u'Щ': // CYRILLIC CAPITAL SHCHA 0429 + case u'Ъ': // CYRILLIC CAPITAL HARD SIGN 042a + case u'Ы': // CYRILLIC CAPITAL YERU 042b + case u'Ь': // CYRILLIC CAPITAL SOFT SIGN 042c + case u'Э': // CYRILLIC CAPITAL E 042d + case u'Ю': // CYRILLIC CAPITAL YU 042e + case u'Я': // CYRILLIC CAPITAL YA 042f + case u'Ѡ': // CYRILLIC CAPITAL OMEGA 0460 + case u'Ѣ': // CYRILLIC CAPITAL YAT 0462 + case u'Ѥ': // CYRILLIC CAPITAL IOTIFIED E 0464 + case u'Ѧ': // CYRILLIC CAPITAL LITTLE YUS 0466 + case u'Ѩ': // CYRILLIC CAPITAL IOTIFIED LITTLE YUS 0468 + case u'Ѫ': // CYRILLIC CAPITAL BIG YUS 046a + case u'Ѭ': // CYRILLIC CAPITAL IOTIFIED BIG YUS 046c + case u'Ѯ': // CYRILLIC CAPITAL KSI 046e + case u'Ѱ': // CYRILLIC CAPITAL PSI 0470 + case u'Ѳ': // CYRILLIC CAPITAL FITA 0472 + case u'Ѵ': // CYRILLIC CAPITAL IZHITSA 0474 + case u'Ѷ': // CYRILLIC CAPITAL IZHITSA W/ DOUBLE GRAVE ACCENT 0476 + case u'Ѹ': // CYRILLIC CAPITAL UK 0478 + case u'Ѻ': // CYRILLIC CAPITAL ROUND OMEGA 047a + case u'Ѽ': // CYRILLIC CAPITAL OMEGA W/ TITLO 047c + case u'Ѿ': // CYRILLIC CAPITAL OT 047e + case u'Ҁ': // CYRILLIC CAPITAL KOPPA 0480 + case u'Ҋ': // CYRILLIC CAPITAL SHORT I W/ TAIL 048a + case u'Ҍ': // CYRILLIC CAPITAL SEMISOFT SIGN 048c + case u'Ҏ': // CYRILLIC CAPITAL ER W/ TICK 048e + case u'Ґ': // CYRILLIC CAPITAL GHE W/ UPTURN 0490 + case u'Ғ': // CYRILLIC CAPITAL GHE W/ STROKE 0492 + case u'Ҕ': // CYRILLIC CAPITAL GHE W/ MIDDLE HOOK 0494 + case u'Җ': // CYRILLIC CAPITAL ZHE W/ DESCENDER 0496 + case u'Ҙ': // CYRILLIC CAPITAL ZE W/ DESCENDER 0498 + case u'Қ': // CYRILLIC CAPITAL KA W/ DESCENDER 049a + case u'Ҝ': // CYRILLIC CAPITAL KA W/ VERTICAL STROKE 049c + case u'Ҟ': // CYRILLIC CAPITAL KA W/ STROKE 049e + case u'Ҡ': // CYRILLIC CAPITAL BASHKIR KA 04a0 + case u'Ң': // CYRILLIC CAPITAL EN W/ DESCENDER 04a2 + case u'Ҥ': // CYRILLIC CAPITAL LIGATURE EN GHE 04a4 + case u'Ҧ': // CYRILLIC CAPITAL PE W/ MIDDLE HOOK 04a6 + case u'Ҩ': // CYRILLIC CAPITAL ABKHASIAN HA 04a8 + case u'Ҫ': // CYRILLIC CAPITAL ES W/ DESCENDER 04aa + case u'Ҭ': // CYRILLIC CAPITAL TE W/ DESCENDER 04ac + case u'Ү': // CYRILLIC CAPITAL STRAIGHT U 04ae + case u'Ұ': // CYRILLIC CAPITAL STRAIGHT U W/ STROKE 04b0 + case u'Ҳ': // CYRILLIC CAPITAL HA W/ DESCENDER 04b2 + case u'Ҵ': // CYRILLIC CAPITAL LIGATURE TE TSE 04b4 + case u'Ҷ': // CYRILLIC CAPITAL CHE W/ DESCENDER 04b6 + case u'Ҹ': // CYRILLIC CAPITAL CHE W/ VERTICAL STROKE 04b8 + case u'Һ': // CYRILLIC CAPITAL SHHA 04ba + case u'Ҽ': // CYRILLIC CAPITAL ABKHASIAN CHE 04bc + case u'Ҿ': // CYRILLIC CAPITAL ABKHASIAN CHE W/ DESCENDER 04be + case u'Ӏ': // CYRILLIC LETTER PALOCHKA 04c0 + case u'Ӂ': // CYRILLIC CAPITAL ZHE W/ BREVE 04c1 + case u'Ӄ': // CYRILLIC CAPITAL KA W/ HOOK 04c3 + case u'Ӆ': // CYRILLIC CAPITAL EL W/ TAIL 04c5 + case u'Ӈ': // CYRILLIC CAPITAL EN W/ HOOK 04c7 + case u'Ӊ': // CYRILLIC CAPITAL EN W/ TAIL 04c9 + case u'Ӌ': // CYRILLIC CAPITAL KHAKASSIAN CHE 04cb + case u'Ӎ': // CYRILLIC CAPITAL EM W/ TAIL 04cd + case u'Ӑ': // CYRILLIC CAPITAL A W/ BREVE 04d0 + case u'Ӓ': // CYRILLIC CAPITAL A W/ DIAERESIS 04d2 + case u'Ӕ': // CYRILLIC CAPITAL LIGATURE A IE 04d4 + case u'Ӗ': // CYRILLIC CAPITAL IE W/ BREVE 04d6 + case u'Ә': // CYRILLIC CAPITAL SCHWA 04d8 + case u'Ӛ': // CYRILLIC CAPITAL SCHWA W/ DIAERESIS 04da + case u'Ӝ': // CYRILLIC CAPITAL ZHE W/ DIAERESIS 04dc + case u'Ӟ': // CYRILLIC CAPITAL ZE W/ DIAERESIS 04de + case u'Ӡ': // CYRILLIC CAPITAL ABKHASIAN DZE 04e0 + case u'Ӣ': // CYRILLIC CAPITAL I W/ MACRON 04e2 + case u'Ӥ': // CYRILLIC CAPITAL I W/ DIAERESIS 04e4 + case u'Ӧ': // CYRILLIC CAPITAL O W/ DIAERESIS 04e6 + case u'Ө': // CYRILLIC CAPITAL BARRED O 04e8 + case u'Ӫ': // CYRILLIC CAPITAL BARRED O W/ DIAERESIS 04ea + case u'Ӭ': // CYRILLIC CAPITAL E W/ DIAERESIS 04ec + case u'Ӯ': // CYRILLIC CAPITAL U W/ MACRON 04ee + case u'Ӱ': // CYRILLIC CAPITAL U W/ DIAERESIS 04f0 + case u'Ӳ': // CYRILLIC CAPITAL U W/ DOUBLE ACUTE 04f2 + case u'Ӵ': // CYRILLIC CAPITAL CHE W/ DIAERESIS 04f4 + case u'Ӷ': // CYRILLIC CAPITAL GHE W/ DESCENDER 04f6 + case u'Ӹ': // CYRILLIC CAPITAL YERU W/ DIAERESIS 04f8 + case u'Ӻ': // CYRILLIC CAPITAL GHE W/ STROKE AND HOOK 04fa + case u'Ӽ': // CYRILLIC CAPITAL HA W/ HOOK 04fc + case u'Ӿ': // CYRILLIC CAPITAL HA W/ STROKE 04fe + case u'Ԁ': // CYRILLIC CAPITAL KOMI DE 0500 + case u'Ԃ': // CYRILLIC CAPITAL KOMI DJE 0502 + case u'Ԅ': // CYRILLIC CAPITAL KOMI ZJE 0504 + case u'Ԇ': // CYRILLIC CAPITAL KOMI DZJE 0506 + case u'Ԉ': // CYRILLIC CAPITAL KOMI LJE 0508 + case u'Ԋ': // CYRILLIC CAPITAL KOMI NJE 050a + case u'Ԍ': // CYRILLIC CAPITAL KOMI SJE 050c + case u'Ԏ': // CYRILLIC CAPITAL KOMI TJE 050e + case u'Ԑ': // CYRILLIC CAPITAL REVERSED ZE 0510 + case u'Ԓ': // CYRILLIC CAPITAL EL W/ HOOK 0512 + case u'Ԕ': // CYRILLIC CAPITAL LHA 0514 + case u'Ԗ': // CYRILLIC CAPITAL RHA 0516 + case u'Ԙ': // CYRILLIC CAPITAL YAE 0518 + case u'Ԛ': // CYRILLIC CAPITAL QA 051a + case u'Ԝ': // CYRILLIC CAPITAL WE 051c + case u'Ԟ': // CYRILLIC CAPITAL ALEUT KA 051e + case u'Ԡ': // CYRILLIC CAPITAL EL W/ MIDDLE HOOK 0520 + case u'Ԣ': // CYRILLIC CAPITAL EN W/ MIDDLE HOOK 0522 + case u'Ԥ': // CYRILLIC CAPITAL PE W/ DESCENDER 0524 + case u'Ԧ': // CYRILLIC CAPITAL SHHA W/ DESCENDER 0526 + case u'Ԩ': // CYRILLIC CAPITAL EN W/ LEFT HOOK 0528 + case u'Ԫ': // CYRILLIC CAPITAL DZZHE 052a + case u'Ԭ': // CYRILLIC CAPITAL DCHE 052c + case u'Ԯ': // CYRILLIC CAPITAL EL W/ DESCENDER 052e + case u'Ա': // ARMENIAN CAPITAL AYB 0531 + case u'Բ': // ARMENIAN CAPITAL BEN 0532 + case u'Գ': // ARMENIAN CAPITAL GIM 0533 + case u'Դ': // ARMENIAN CAPITAL DA 0534 + case u'Ե': // ARMENIAN CAPITAL ECH 0535 + case u'Զ': // ARMENIAN CAPITAL ZA 0536 + case u'Է': // ARMENIAN CAPITAL EH 0537 + case u'Ը': // ARMENIAN CAPITAL ET 0538 + case u'Թ': // ARMENIAN CAPITAL TO 0539 + case u'Ժ': // ARMENIAN CAPITAL ZHE 053a + case u'Ի': // ARMENIAN CAPITAL INI 053b + case u'Լ': // ARMENIAN CAPITAL LIWN 053c + case u'Խ': // ARMENIAN CAPITAL XEH 053d + case u'Ծ': // ARMENIAN CAPITAL CA 053e + case u'Կ': // ARMENIAN CAPITAL KEN 053f + case u'Հ': // ARMENIAN CAPITAL HO 0540 + case u'Ձ': // ARMENIAN CAPITAL JA 0541 + case u'Ղ': // ARMENIAN CAPITAL GHAD 0542 + case u'Ճ': // ARMENIAN CAPITAL CHEH 0543 + case u'Մ': // ARMENIAN CAPITAL MEN 0544 + case u'Յ': // ARMENIAN CAPITAL YI 0545 + case u'Ն': // ARMENIAN CAPITAL NOW 0546 + case u'Շ': // ARMENIAN CAPITAL SHA 0547 + case u'Ո': // ARMENIAN CAPITAL VO 0548 + case u'Չ': // ARMENIAN CAPITAL CHA 0549 + case u'Պ': // ARMENIAN CAPITAL PEH 054a + case u'Ջ': // ARMENIAN CAPITAL JHEH 054b + case u'Ռ': // ARMENIAN CAPITAL RA 054c + case u'Ս': // ARMENIAN CAPITAL SEH 054d + case u'Վ': // ARMENIAN CAPITAL VEW 054e + case u'Տ': // ARMENIAN CAPITAL TIWN 054f + case u'Ր': // ARMENIAN CAPITAL REH 0550 + case u'Ց': // ARMENIAN CAPITAL CO 0551 + case u'Ւ': // ARMENIAN CAPITAL YIWN 0552 + case u'Փ': // ARMENIAN CAPITAL PIWR 0553 + case u'Ք': // ARMENIAN CAPITAL KEH 0554 + case u'Օ': // ARMENIAN CAPITAL OH 0555 + case u'Ֆ': // ARMENIAN CAPITAL FEH 0556 + case u'Ⴀ': // GEORGIAN CAPITAL AN 10a0 + case u'Ⴁ': // GEORGIAN CAPITAL BAN 10a1 + case u'Ⴂ': // GEORGIAN CAPITAL GAN 10a2 + case u'Ⴃ': // GEORGIAN CAPITAL DON 10a3 + case u'Ⴄ': // GEORGIAN CAPITAL EN 10a4 + case u'Ⴅ': // GEORGIAN CAPITAL VIN 10a5 + case u'Ⴆ': // GEORGIAN CAPITAL ZEN 10a6 + case u'Ⴇ': // GEORGIAN CAPITAL TAN 10a7 + case u'Ⴈ': // GEORGIAN CAPITAL IN 10a8 + case u'Ⴉ': // GEORGIAN CAPITAL KAN 10a9 + case u'Ⴊ': // GEORGIAN CAPITAL LAS 10aa + case u'Ⴋ': // GEORGIAN CAPITAL MAN 10ab + case u'Ⴌ': // GEORGIAN CAPITAL NAR 10ac + case u'Ⴍ': // GEORGIAN CAPITAL ON 10ad + case u'Ⴎ': // GEORGIAN CAPITAL PAR 10ae + case u'Ⴏ': // GEORGIAN CAPITAL ZHAR 10af + case u'Ⴐ': // GEORGIAN CAPITAL RAE 10b0 + case u'Ⴑ': // GEORGIAN CAPITAL SAN 10b1 + case u'Ⴒ': // GEORGIAN CAPITAL TAR 10b2 + case u'Ⴓ': // GEORGIAN CAPITAL UN 10b3 + case u'Ⴔ': // GEORGIAN CAPITAL PHAR 10b4 + case u'Ⴕ': // GEORGIAN CAPITAL KHAR 10b5 + case u'Ⴖ': // GEORGIAN CAPITAL GHAN 10b6 + case u'Ⴗ': // GEORGIAN CAPITAL QAR 10b7 + case u'Ⴘ': // GEORGIAN CAPITAL SHIN 10b8 + case u'Ⴙ': // GEORGIAN CAPITAL CHIN 10b9 + case u'Ⴚ': // GEORGIAN CAPITAL CAN 10ba + case u'Ⴛ': // GEORGIAN CAPITAL JIL 10bb + case u'Ⴜ': // GEORGIAN CAPITAL CIL 10bc + case u'Ⴝ': // GEORGIAN CAPITAL CHAR 10bd + case u'Ⴞ': // GEORGIAN CAPITAL XAN 10be + case u'Ⴟ': // GEORGIAN CAPITAL JHAN 10bf + case u'Ⴠ': // GEORGIAN CAPITAL HAE 10c0 + case u'Ⴡ': // GEORGIAN CAPITAL HE 10c1 + case u'Ⴢ': // GEORGIAN CAPITAL HIE 10c2 + case u'Ⴣ': // GEORGIAN CAPITAL WE 10c3 + case u'Ⴤ': // GEORGIAN CAPITAL HAR 10c4 + case u'Ⴥ': // GEORGIAN CAPITAL HOE 10c5 + case u'Ⴧ': // GEORGIAN CAPITAL YN 10c7 + case u'Ⴭ': // GEORGIAN CAPITAL AEN 10cd + case u'Ꭰ': // CHEROKEE LETTER A 13a0 + case u'Ꭱ': // CHEROKEE LETTER E 13a1 + case u'Ꭲ': // CHEROKEE LETTER I 13a2 + case u'Ꭳ': // CHEROKEE LETTER O 13a3 + case u'Ꭴ': // CHEROKEE LETTER U 13a4 + case u'Ꭵ': // CHEROKEE LETTER V 13a5 + case u'Ꭶ': // CHEROKEE LETTER GA 13a6 + case u'Ꭷ': // CHEROKEE LETTER KA 13a7 + case u'Ꭸ': // CHEROKEE LETTER GE 13a8 + case u'Ꭹ': // CHEROKEE LETTER GI 13a9 + case u'Ꭺ': // CHEROKEE LETTER GO 13aa + case u'Ꭻ': // CHEROKEE LETTER GU 13ab + case u'Ꭼ': // CHEROKEE LETTER GV 13ac + case u'Ꭽ': // CHEROKEE LETTER HA 13ad + case u'Ꭾ': // CHEROKEE LETTER HE 13ae + case u'Ꭿ': // CHEROKEE LETTER HI 13af + case u'Ꮀ': // CHEROKEE LETTER HO 13b0 + case u'Ꮁ': // CHEROKEE LETTER HU 13b1 + case u'Ꮂ': // CHEROKEE LETTER HV 13b2 + case u'Ꮃ': // CHEROKEE LETTER LA 13b3 + case u'Ꮄ': // CHEROKEE LETTER LE 13b4 + case u'Ꮅ': // CHEROKEE LETTER LI 13b5 + case u'Ꮆ': // CHEROKEE LETTER LO 13b6 + case u'Ꮇ': // CHEROKEE LETTER LU 13b7 + case u'Ꮈ': // CHEROKEE LETTER LV 13b8 + case u'Ꮉ': // CHEROKEE LETTER MA 13b9 + case u'Ꮊ': // CHEROKEE LETTER ME 13ba + case u'Ꮋ': // CHEROKEE LETTER MI 13bb + case u'Ꮌ': // CHEROKEE LETTER MO 13bc + case u'Ꮍ': // CHEROKEE LETTER MU 13bd + case u'Ꮎ': // CHEROKEE LETTER NA 13be + case u'Ꮏ': // CHEROKEE LETTER HNA 13bf + case u'Ꮐ': // CHEROKEE LETTER NAH 13c0 + case u'Ꮑ': // CHEROKEE LETTER NE 13c1 + case u'Ꮒ': // CHEROKEE LETTER NI 13c2 + case u'Ꮓ': // CHEROKEE LETTER NO 13c3 + case u'Ꮔ': // CHEROKEE LETTER NU 13c4 + case u'Ꮕ': // CHEROKEE LETTER NV 13c5 + case u'Ꮖ': // CHEROKEE LETTER QUA 13c6 + case u'Ꮗ': // CHEROKEE LETTER QUE 13c7 + case u'Ꮘ': // CHEROKEE LETTER QUI 13c8 + case u'Ꮙ': // CHEROKEE LETTER QUO 13c9 + case u'Ꮚ': // CHEROKEE LETTER QUU 13ca + case u'Ꮛ': // CHEROKEE LETTER QUV 13cb + case u'Ꮜ': // CHEROKEE LETTER SA 13cc + case u'Ꮝ': // CHEROKEE LETTER S 13cd + case u'Ꮞ': // CHEROKEE LETTER SE 13ce + case u'Ꮟ': // CHEROKEE LETTER SI 13cf + case u'Ꮠ': // CHEROKEE LETTER SO 13d0 + case u'Ꮡ': // CHEROKEE LETTER SU 13d1 + case u'Ꮢ': // CHEROKEE LETTER SV 13d2 + case u'Ꮣ': // CHEROKEE LETTER DA 13d3 + case u'Ꮤ': // CHEROKEE LETTER TA 13d4 + case u'Ꮥ': // CHEROKEE LETTER DE 13d5 + case u'Ꮦ': // CHEROKEE LETTER TE 13d6 + case u'Ꮧ': // CHEROKEE LETTER DI 13d7 + case u'Ꮨ': // CHEROKEE LETTER TI 13d8 + case u'Ꮩ': // CHEROKEE LETTER DO 13d9 + case u'Ꮪ': // CHEROKEE LETTER DU 13da + case u'Ꮫ': // CHEROKEE LETTER DV 13db + case u'Ꮬ': // CHEROKEE LETTER DLA 13dc + case u'Ꮭ': // CHEROKEE LETTER TLA 13dd + case u'Ꮮ': // CHEROKEE LETTER TLE 13de + case u'Ꮯ': // CHEROKEE LETTER TLI 13df + case u'Ꮰ': // CHEROKEE LETTER TLO 13e0 + case u'Ꮱ': // CHEROKEE LETTER TLU 13e1 + case u'Ꮲ': // CHEROKEE LETTER TLV 13e2 + case u'Ꮳ': // CHEROKEE LETTER TSA 13e3 + case u'Ꮴ': // CHEROKEE LETTER TSE 13e4 + case u'Ꮵ': // CHEROKEE LETTER TSI 13e5 + case u'Ꮶ': // CHEROKEE LETTER TSO 13e6 + case u'Ꮷ': // CHEROKEE LETTER TSU 13e7 + case u'Ꮸ': // CHEROKEE LETTER TSV 13e8 + case u'Ꮹ': // CHEROKEE LETTER WA 13e9 + case u'Ꮺ': // CHEROKEE LETTER WE 13ea + case u'Ꮻ': // CHEROKEE LETTER WI 13eb + case u'Ꮼ': // CHEROKEE LETTER WO 13ec + case u'Ꮽ': // CHEROKEE LETTER WU 13ed + case u'Ꮾ': // CHEROKEE LETTER WV 13ee + case u'Ꮿ': // CHEROKEE LETTER YA 13ef + case u'Ᏸ': // CHEROKEE LETTER YE 13f0 + case u'Ᏹ': // CHEROKEE LETTER YI 13f1 + case u'Ᏺ': // CHEROKEE LETTER YO 13f2 + case u'Ᏻ': // CHEROKEE LETTER YU 13f3 + case u'Ᏼ': // CHEROKEE LETTER YV 13f4 + case u'Ᏽ': // CHEROKEE LETTER MV 13f5 + case u'Ა': // GEORGIAN MTAVRULI CAPITAL AN 1c90 + case u'Ბ': // GEORGIAN MTAVRULI CAPITAL BAN 1c91 + case u'Გ': // GEORGIAN MTAVRULI CAPITAL GAN 1c92 + case u'Დ': // GEORGIAN MTAVRULI CAPITAL DON 1c93 + case u'Ე': // GEORGIAN MTAVRULI CAPITAL EN 1c94 + case u'Ვ': // GEORGIAN MTAVRULI CAPITAL VIN 1c95 + case u'Ზ': // GEORGIAN MTAVRULI CAPITAL ZEN 1c96 + case u'Თ': // GEORGIAN MTAVRULI CAPITAL TAN 1c97 + case u'Ი': // GEORGIAN MTAVRULI CAPITAL IN 1c98 + case u'Კ': // GEORGIAN MTAVRULI CAPITAL KAN 1c99 + case u'Ლ': // GEORGIAN MTAVRULI CAPITAL LAS 1c9a + case u'Მ': // GEORGIAN MTAVRULI CAPITAL MAN 1c9b + case u'Ნ': // GEORGIAN MTAVRULI CAPITAL NAR 1c9c + case u'Ო': // GEORGIAN MTAVRULI CAPITAL ON 1c9d + case u'Პ': // GEORGIAN MTAVRULI CAPITAL PAR 1c9e + case u'Ჟ': // GEORGIAN MTAVRULI CAPITAL ZHAR 1c9f + case u'Რ': // GEORGIAN MTAVRULI CAPITAL RAE 1ca0 + case u'Ს': // GEORGIAN MTAVRULI CAPITAL SAN 1ca1 + case u'Ტ': // GEORGIAN MTAVRULI CAPITAL TAR 1ca2 + case u'Უ': // GEORGIAN MTAVRULI CAPITAL UN 1ca3 + case u'Ფ': // GEORGIAN MTAVRULI CAPITAL PHAR 1ca4 + case u'Ქ': // GEORGIAN MTAVRULI CAPITAL KHAR 1ca5 + case u'Ღ': // GEORGIAN MTAVRULI CAPITAL GHAN 1ca6 + case u'Ყ': // GEORGIAN MTAVRULI CAPITAL QAR 1ca7 + case u'Შ': // GEORGIAN MTAVRULI CAPITAL SHIN 1ca8 + case u'Ჩ': // GEORGIAN MTAVRULI CAPITAL CHIN 1ca9 + case u'Ც': // GEORGIAN MTAVRULI CAPITAL CAN 1caa + case u'Ძ': // GEORGIAN MTAVRULI CAPITAL JIL 1cab + case u'Წ': // GEORGIAN MTAVRULI CAPITAL CIL 1cac + case u'Ჭ': // GEORGIAN MTAVRULI CAPITAL CHAR 1cad + case u'Ხ': // GEORGIAN MTAVRULI CAPITAL XAN 1cae + case u'Ჯ': // GEORGIAN MTAVRULI CAPITAL JHAN 1caf + case u'Ჰ': // GEORGIAN MTAVRULI CAPITAL HAE 1cb0 + case u'Ჱ': // GEORGIAN MTAVRULI CAPITAL HE 1cb1 + case u'Ჲ': // GEORGIAN MTAVRULI CAPITAL HIE 1cb2 + case u'Ჳ': // GEORGIAN MTAVRULI CAPITAL WE 1cb3 + case u'Ჴ': // GEORGIAN MTAVRULI CAPITAL HAR 1cb4 + case u'Ჵ': // GEORGIAN MTAVRULI CAPITAL HOE 1cb5 + case u'Ჶ': // GEORGIAN MTAVRULI CAPITAL FI 1cb6 + case u'Ჷ': // GEORGIAN MTAVRULI CAPITAL YN 1cb7 + case u'Ჸ': // GEORGIAN MTAVRULI CAPITAL ELIFI 1cb8 + case u'Ჹ': // GEORGIAN MTAVRULI CAPITAL TURNED GAN 1cb9 + case u'Ჺ': // GEORGIAN MTAVRULI CAPITAL AIN 1cba + case u'Ჽ': // GEORGIAN MTAVRULI CAPITAL AEN 1cbd + case u'Ჾ': // GEORGIAN MTAVRULI CAPITAL HARD SIGN 1cbe + case u'Ჿ': // GEORGIAN MTAVRULI CAPITAL LABIAL SIGN 1cbf + case u'Ḁ': // LATIN CAPITAL A W/ RING BELOW 1e00 + case u'Ḃ': // LATIN CAPITAL B W/ DOT ABOVE 1e02 + case u'Ḅ': // LATIN CAPITAL B W/ DOT BELOW 1e04 + case u'Ḇ': // LATIN CAPITAL B W/ LINE BELOW 1e06 + case u'Ḉ': // LATIN CAPITAL C W/ CEDILLA AND ACUTE 1e08 + case u'Ḋ': // LATIN CAPITAL D W/ DOT ABOVE 1e0a + case u'Ḍ': // LATIN CAPITAL D W/ DOT BELOW 1e0c + case u'Ḏ': // LATIN CAPITAL D W/ LINE BELOW 1e0e + case u'Ḑ': // LATIN CAPITAL D W/ CEDILLA 1e10 + case u'Ḓ': // LATIN CAPITAL D W/ CIRCUMFLEX BELOW 1e12 + case u'Ḕ': // LATIN CAPITAL E W/ MACRON AND GRAVE 1e14 + case u'Ḗ': // LATIN CAPITAL E W/ MACRON AND ACUTE 1e16 + case u'Ḙ': // LATIN CAPITAL E W/ CIRCUMFLEX BELOW 1e18 + case u'Ḛ': // LATIN CAPITAL E W/ TILDE BELOW 1e1a + case u'Ḝ': // LATIN CAPITAL E W/ CEDILLA AND BREVE 1e1c + case u'Ḟ': // LATIN CAPITAL F W/ DOT ABOVE 1e1e + case u'Ḡ': // LATIN CAPITAL G W/ MACRON 1e20 + case u'Ḣ': // LATIN CAPITAL H W/ DOT ABOVE 1e22 + case u'Ḥ': // LATIN CAPITAL H W/ DOT BELOW 1e24 + case u'Ḧ': // LATIN CAPITAL H W/ DIAERESIS 1e26 + case u'Ḩ': // LATIN CAPITAL H W/ CEDILLA 1e28 + case u'Ḫ': // LATIN CAPITAL H W/ BREVE BELOW 1e2a + case u'Ḭ': // LATIN CAPITAL I W/ TILDE BELOW 1e2c + case u'Ḯ': // LATIN CAPITAL I W/ DIAERESIS AND ACUTE 1e2e + case u'Ḱ': // LATIN CAPITAL K W/ ACUTE 1e30 + case u'Ḳ': // LATIN CAPITAL K W/ DOT BELOW 1e32 + case u'Ḵ': // LATIN CAPITAL K W/ LINE BELOW 1e34 + case u'Ḷ': // LATIN CAPITAL L W/ DOT BELOW 1e36 + case u'Ḹ': // LATIN CAPITAL L W/ DOT BELOW AND MACRON 1e38 + case u'Ḻ': // LATIN CAPITAL L W/ LINE BELOW 1e3a + case u'Ḽ': // LATIN CAPITAL L W/ CIRCUMFLEX BELOW 1e3c + case u'Ḿ': // LATIN CAPITAL M W/ ACUTE 1e3e + case u'Ṁ': // LATIN CAPITAL M W/ DOT ABOVE 1e40 + case u'Ṃ': // LATIN CAPITAL M W/ DOT BELOW 1e42 + case u'Ṅ': // LATIN CAPITAL N W/ DOT ABOVE 1e44 + case u'Ṇ': // LATIN CAPITAL N W/ DOT BELOW 1e46 + case u'Ṉ': // LATIN CAPITAL N W/ LINE BELOW 1e48 + case u'Ṋ': // LATIN CAPITAL N W/ CIRCUMFLEX BELOW 1e4a + case u'Ṍ': // LATIN CAPITAL O W/ TILDE AND ACUTE 1e4c + case u'Ṏ': // LATIN CAPITAL O W/ TILDE AND DIAERESIS 1e4e + case u'Ṑ': // LATIN CAPITAL O W/ MACRON AND GRAVE 1e50 + case u'Ṓ': // LATIN CAPITAL O W/ MACRON AND ACUTE 1e52 + case u'Ṕ': // LATIN CAPITAL P W/ ACUTE 1e54 + case u'Ṗ': // LATIN CAPITAL P W/ DOT ABOVE 1e56 + case u'Ṙ': // LATIN CAPITAL R W/ DOT ABOVE 1e58 + case u'Ṛ': // LATIN CAPITAL R W/ DOT BELOW 1e5a + case u'Ṝ': // LATIN CAPITAL R W/ DOT BELOW AND MACRON 1e5c + case u'Ṟ': // LATIN CAPITAL R W/ LINE BELOW 1e5e + case u'Ṡ': // LATIN CAPITAL S W/ DOT ABOVE 1e60 + case u'Ṣ': // LATIN CAPITAL S W/ DOT BELOW 1e62 + case u'Ṥ': // LATIN CAPITAL S W/ ACUTE AND DOT ABOVE 1e64 + case u'Ṧ': // LATIN CAPITAL S W/ CARON AND DOT ABOVE 1e66 + case u'Ṩ': // LATIN CAPITAL S W/ DOT BELOW AND DOT ABOVE 1e68 + case u'Ṫ': // LATIN CAPITAL T W/ DOT ABOVE 1e6a + case u'Ṭ': // LATIN CAPITAL T W/ DOT BELOW 1e6c + case u'Ṯ': // LATIN CAPITAL T W/ LINE BELOW 1e6e + case u'Ṱ': // LATIN CAPITAL T W/ CIRCUMFLEX BELOW 1e70 + case u'Ṳ': // LATIN CAPITAL U W/ DIAERESIS BELOW 1e72 + case u'Ṵ': // LATIN CAPITAL U W/ TILDE BELOW 1e74 + case u'Ṷ': // LATIN CAPITAL U W/ CIRCUMFLEX BELOW 1e76 + case u'Ṹ': // LATIN CAPITAL U W/ TILDE AND ACUTE 1e78 + case u'Ṻ': // LATIN CAPITAL U W/ MACRON AND DIAERESIS 1e7a + case u'Ṽ': // LATIN CAPITAL V W/ TILDE 1e7c + case u'Ṿ': // LATIN CAPITAL V W/ DOT BELOW 1e7e + case u'Ẁ': // LATIN CAPITAL W W/ GRAVE 1e80 + case u'Ẃ': // LATIN CAPITAL W W/ ACUTE 1e82 + case u'Ẅ': // LATIN CAPITAL W W/ DIAERESIS 1e84 + case u'Ẇ': // LATIN CAPITAL W W/ DOT ABOVE 1e86 + case u'Ẉ': // LATIN CAPITAL W W/ DOT BELOW 1e88 + case u'Ẋ': // LATIN CAPITAL X W/ DOT ABOVE 1e8a + case u'Ẍ': // LATIN CAPITAL X W/ DIAERESIS 1e8c + case u'Ẏ': // LATIN CAPITAL Y W/ DOT ABOVE 1e8e + case u'Ẑ': // LATIN CAPITAL Z W/ CIRCUMFLEX 1e90 + case u'Ẓ': // LATIN CAPITAL Z W/ DOT BELOW 1e92 + case u'Ẕ': // LATIN CAPITAL Z W/ LINE BELOW 1e94 + case u'ẞ': // LATIN CAPITAL SHARP S 1e9e + case u'Ạ': // LATIN CAPITAL A W/ DOT BELOW 1ea0 + case u'Ả': // LATIN CAPITAL A W/ HOOK ABOVE 1ea2 + case u'Ấ': // LATIN CAPITAL A W/ CIRCUMFLEX AND ACUTE 1ea4 + case u'Ầ': // LATIN CAPITAL A W/ CIRCUMFLEX AND GRAVE 1ea6 + case u'Ẩ': // LATIN CAPITAL A W/ CIRCUMFLEX AND HOOK ABOVE 1ea8 + case u'Ẫ': // LATIN CAPITAL A W/ CIRCUMFLEX AND TILDE 1eaa + case u'Ậ': // LATIN CAPITAL A W/ CIRCUMFLEX AND DOT BELOW 1eac + case u'Ắ': // LATIN CAPITAL A W/ BREVE AND ACUTE 1eae + case u'Ằ': // LATIN CAPITAL A W/ BREVE AND GRAVE 1eb0 + case u'Ẳ': // LATIN CAPITAL A W/ BREVE AND HOOK ABOVE 1eb2 + case u'Ẵ': // LATIN CAPITAL A W/ BREVE AND TILDE 1eb4 + case u'Ặ': // LATIN CAPITAL A W/ BREVE AND DOT BELOW 1eb6 + case u'Ẹ': // LATIN CAPITAL E W/ DOT BELOW 1eb8 + case u'Ẻ': // LATIN CAPITAL E W/ HOOK ABOVE 1eba + case u'Ẽ': // LATIN CAPITAL E W/ TILDE 1ebc + case u'Ế': // LATIN CAPITAL E W/ CIRCUMFLEX AND ACUTE 1ebe + case u'Ề': // LATIN CAPITAL E W/ CIRCUMFLEX AND GRAVE 1ec0 + case u'Ể': // LATIN CAPITAL E W/ CIRCUMFLEX AND HOOK ABOVE 1ec2 + case u'Ễ': // LATIN CAPITAL E W/ CIRCUMFLEX AND TILDE 1ec4 + case u'Ệ': // LATIN CAPITAL E W/ CIRCUMFLEX AND DOT BELOW 1ec6 + case u'Ỉ': // LATIN CAPITAL I W/ HOOK ABOVE 1ec8 + case u'Ị': // LATIN CAPITAL I W/ DOT BELOW 1eca + case u'Ọ': // LATIN CAPITAL O W/ DOT BELOW 1ecc + case u'Ỏ': // LATIN CAPITAL O W/ HOOK ABOVE 1ece + case u'Ố': // LATIN CAPITAL O W/ CIRCUMFLEX AND ACUTE 1ed0 + case u'Ồ': // LATIN CAPITAL O W/ CIRCUMFLEX AND GRAVE 1ed2 + case u'Ổ': // LATIN CAPITAL O W/ CIRCUMFLEX AND HOOK ABOVE 1ed4 + case u'Ỗ': // LATIN CAPITAL O W/ CIRCUMFLEX AND TILDE 1ed6 + case u'Ộ': // LATIN CAPITAL O W/ CIRCUMFLEX AND DOT BELOW 1ed8 + case u'Ớ': // LATIN CAPITAL O W/ HORN AND ACUTE 1eda + case u'Ờ': // LATIN CAPITAL O W/ HORN AND GRAVE 1edc + case u'Ở': // LATIN CAPITAL O W/ HORN AND HOOK ABOVE 1ede + case u'Ỡ': // LATIN CAPITAL O W/ HORN AND TILDE 1ee0 + case u'Ợ': // LATIN CAPITAL O W/ HORN AND DOT BELOW 1ee2 + case u'Ụ': // LATIN CAPITAL U W/ DOT BELOW 1ee4 + case u'Ủ': // LATIN CAPITAL U W/ HOOK ABOVE 1ee6 + case u'Ứ': // LATIN CAPITAL U W/ HORN AND ACUTE 1ee8 + case u'Ừ': // LATIN CAPITAL U W/ HORN AND GRAVE 1eea + case u'Ử': // LATIN CAPITAL U W/ HORN AND HOOK ABOVE 1eec + case u'Ữ': // LATIN CAPITAL U W/ HORN AND TILDE 1eee + case u'Ự': // LATIN CAPITAL U W/ HORN AND DOT BELOW 1ef0 + case u'Ỳ': // LATIN CAPITAL Y W/ GRAVE 1ef2 + case u'Ỵ': // LATIN CAPITAL Y W/ DOT BELOW 1ef4 + case u'Ỷ': // LATIN CAPITAL Y W/ HOOK ABOVE 1ef6 + case u'Ỹ': // LATIN CAPITAL Y W/ TILDE 1ef8 + case u'Ỻ': // LATIN CAPITAL MIDDLE-WELSH LL 1efa + case u'Ỽ': // LATIN CAPITAL MIDDLE-WELSH V 1efc + case u'Ỿ': // LATIN CAPITAL Y W/ LOOP 1efe + case u'Ἀ': // GREEK CAPITAL ALPHA W/ PSILI 1f08 + case u'Ἁ': // GREEK CAPITAL ALPHA W/ DASIA 1f09 + case u'Ἂ': // GREEK CAPITAL ALPHA W/ PSILI AND VARIA 1f0a + case u'Ἃ': // GREEK CAPITAL ALPHA W/ DASIA AND VARIA 1f0b + case u'Ἄ': // GREEK CAPITAL ALPHA W/ PSILI AND OXIA 1f0c + case u'Ἅ': // GREEK CAPITAL ALPHA W/ DASIA AND OXIA 1f0d + case u'Ἆ': // GREEK CAPITAL ALPHA W/ PSILI AND PERISPOMENI 1f0e + case u'Ἇ': // GREEK CAPITAL ALPHA W/ DASIA AND PERISPOMENI 1f0f + case u'Ἐ': // GREEK CAPITAL EPSILON W/ PSILI 1f18 + case u'Ἑ': // GREEK CAPITAL EPSILON W/ DASIA 1f19 + case u'Ἒ': // GREEK CAPITAL EPSILON W/ PSILI AND VARIA 1f1a + case u'Ἓ': // GREEK CAPITAL EPSILON W/ DASIA AND VARIA 1f1b + case u'Ἔ': // GREEK CAPITAL EPSILON W/ PSILI AND OXIA 1f1c + case u'Ἕ': // GREEK CAPITAL EPSILON W/ DASIA AND OXIA 1f1d + case u'Ἠ': // GREEK CAPITAL ETA W/ PSILI 1f28 + case u'Ἡ': // GREEK CAPITAL ETA W/ DASIA 1f29 + case u'Ἢ': // GREEK CAPITAL ETA W/ PSILI AND VARIA 1f2a + case u'Ἣ': // GREEK CAPITAL ETA W/ DASIA AND VARIA 1f2b + case u'Ἤ': // GREEK CAPITAL ETA W/ PSILI AND OXIA 1f2c + case u'Ἥ': // GREEK CAPITAL ETA W/ DASIA AND OXIA 1f2d + case u'Ἦ': // GREEK CAPITAL ETA W/ PSILI AND PERISPOMENI 1f2e + case u'Ἧ': // GREEK CAPITAL ETA W/ DASIA AND PERISPOMENI 1f2f + case u'Ἰ': // GREEK CAPITAL IOTA W/ PSILI 1f38 + case u'Ἱ': // GREEK CAPITAL IOTA W/ DASIA 1f39 + case u'Ἲ': // GREEK CAPITAL IOTA W/ PSILI AND VARIA 1f3a + case u'Ἳ': // GREEK CAPITAL IOTA W/ DASIA AND VARIA 1f3b + case u'Ἴ': // GREEK CAPITAL IOTA W/ PSILI AND OXIA 1f3c + case u'Ἵ': // GREEK CAPITAL IOTA W/ DASIA AND OXIA 1f3d + case u'Ἶ': // GREEK CAPITAL IOTA W/ PSILI AND PERISPOMENI 1f3e + case u'Ἷ': // GREEK CAPITAL IOTA W/ DASIA AND PERISPOMENI 1f3f + case u'Ὀ': // GREEK CAPITAL OMICRON W/ PSILI 1f48 + case u'Ὁ': // GREEK CAPITAL OMICRON W/ DASIA 1f49 + case u'Ὂ': // GREEK CAPITAL OMICRON W/ PSILI AND VARIA 1f4a + case u'Ὃ': // GREEK CAPITAL OMICRON W/ DASIA AND VARIA 1f4b + case u'Ὄ': // GREEK CAPITAL OMICRON W/ PSILI AND OXIA 1f4c + case u'Ὅ': // GREEK CAPITAL OMICRON W/ DASIA AND OXIA 1f4d + case u'Ὑ': // GREEK CAPITAL UPSILON W/ DASIA 1f59 + case u'Ὓ': // GREEK CAPITAL UPSILON W/ DASIA AND VARIA 1f5b + case u'Ὕ': // GREEK CAPITAL UPSILON W/ DASIA AND OXIA 1f5d + case u'Ὗ': // GREEK CAPITAL UPSILON W/ DASIA AND PERISPOMENI 1f5f + case u'Ὠ': // GREEK CAPITAL OMEGA W/ PSILI 1f68 + case u'Ὡ': // GREEK CAPITAL OMEGA W/ DASIA 1f69 + case u'Ὢ': // GREEK CAPITAL OMEGA W/ PSILI AND VARIA 1f6a + case u'Ὣ': // GREEK CAPITAL OMEGA W/ DASIA AND VARIA 1f6b + case u'Ὤ': // GREEK CAPITAL OMEGA W/ PSILI AND OXIA 1f6c + case u'Ὥ': // GREEK CAPITAL OMEGA W/ DASIA AND OXIA 1f6d + case u'Ὦ': // GREEK CAPITAL OMEGA W/ PSILI AND PERISPOMENI 1f6e + case u'Ὧ': // GREEK CAPITAL OMEGA W/ DASIA AND PERISPOMENI 1f6f + case u'Ᾰ': // GREEK CAPITAL ALPHA W/ VRACHY 1fb8 + case u'Ᾱ': // GREEK CAPITAL ALPHA W/ MACRON 1fb9 + case u'Ὰ': // GREEK CAPITAL ALPHA W/ VARIA 1fba + case u'Ά': // GREEK CAPITAL ALPHA W/ OXIA 1fbb + case u'Ὲ': // GREEK CAPITAL EPSILON W/ VARIA 1fc8 + case u'Έ': // GREEK CAPITAL EPSILON W/ OXIA 1fc9 + case u'Ὴ': // GREEK CAPITAL ETA W/ VARIA 1fca + case u'Ή': // GREEK CAPITAL ETA W/ OXIA 1fcb + case u'Ῐ': // GREEK CAPITAL IOTA W/ VRACHY 1fd8 + case u'Ῑ': // GREEK CAPITAL IOTA W/ MACRON 1fd9 + case u'Ὶ': // GREEK CAPITAL IOTA W/ VARIA 1fda + case u'Ί': // GREEK CAPITAL IOTA W/ OXIA 1fdb + case u'Ῠ': // GREEK CAPITAL UPSILON W/ VRACHY 1fe8 + case u'Ῡ': // GREEK CAPITAL UPSILON W/ MACRON 1fe9 + case u'Ὺ': // GREEK CAPITAL UPSILON W/ VARIA 1fea + case u'Ύ': // GREEK CAPITAL UPSILON W/ OXIA 1feb + case u'Ῥ': // GREEK CAPITAL RHO W/ DASIA 1fec + case u'Ὸ': // GREEK CAPITAL OMICRON W/ VARIA 1ff8 + case u'Ό': // GREEK CAPITAL OMICRON W/ OXIA 1ff9 + case u'Ὼ': // GREEK CAPITAL OMEGA W/ VARIA 1ffa + case u'Ώ': // GREEK CAPITAL OMEGA W/ OXIA 1ffb + case u'ℂ': // DOUBLE-STRUCK CAPITAL C 2102 + case u'ℇ': // EULER CONSTANT 2107 + case u'ℋ': // SCRIPT CAPITAL H 210b + case u'ℌ': // BLACK-LETTER CAPITAL H 210c + case u'ℍ': // DOUBLE-STRUCK CAPITAL H 210d + case u'ℐ': // SCRIPT CAPITAL I 2110 + case u'ℑ': // BLACK-LETTER CAPITAL I 2111 + case u'ℒ': // SCRIPT CAPITAL L 2112 + case u'ℕ': // DOUBLE-STRUCK CAPITAL N 2115 + case u'ℙ': // DOUBLE-STRUCK CAPITAL P 2119 + case u'ℚ': // DOUBLE-STRUCK CAPITAL Q 211a + case u'ℛ': // SCRIPT CAPITAL R 211b + case u'ℜ': // BLACK-LETTER CAPITAL R 211c + case u'ℝ': // DOUBLE-STRUCK CAPITAL R 211d + case u'ℤ': // DOUBLE-STRUCK CAPITAL Z 2124 + case u'Ω': // OHM SIGN 2126 + case u'ℨ': // BLACK-LETTER CAPITAL Z 2128 + case u'K': // KELVIN SIGN 212a + case u'Å': // ANGSTROM SIGN 212b + case u'ℬ': // SCRIPT CAPITAL B 212c + case u'ℭ': // BLACK-LETTER CAPITAL C 212d + case u'ℰ': // SCRIPT CAPITAL E 2130 + case u'ℱ': // SCRIPT CAPITAL F 2131 + case u'Ⅎ': // TURNED CAPITAL F 2132 + case u'ℳ': // SCRIPT CAPITAL M 2133 + case u'ℾ': // DOUBLE-STRUCK CAPITAL GAMMA 213e + case u'ℿ': // DOUBLE-STRUCK CAPITAL PI 213f + case u'ⅅ': // DOUBLE-STRUCK ITALIC CAPITAL D 2145 + case u'Ↄ': // ROMAN NUMERAL REVERSED ONE HUNDRED 2183 + case u'Ⰰ': // GLAGOLITIC CAPITAL AZU 2c00 + case u'Ⰱ': // GLAGOLITIC CAPITAL BUKY 2c01 + case u'Ⰲ': // GLAGOLITIC CAPITAL VEDE 2c02 + case u'Ⰳ': // GLAGOLITIC CAPITAL GLAGOLI 2c03 + case u'Ⰴ': // GLAGOLITIC CAPITAL DOBRO 2c04 + case u'Ⰵ': // GLAGOLITIC CAPITAL YESTU 2c05 + case u'Ⰶ': // GLAGOLITIC CAPITAL ZHIVETE 2c06 + case u'Ⰷ': // GLAGOLITIC CAPITAL DZELO 2c07 + case u'Ⰸ': // GLAGOLITIC CAPITAL ZEMLJA 2c08 + case u'Ⰹ': // GLAGOLITIC CAPITAL IZHE 2c09 + case u'Ⰺ': // GLAGOLITIC CAPITAL INITIAL IZHE 2c0a + case u'Ⰻ': // GLAGOLITIC CAPITAL I 2c0b + case u'Ⰼ': // GLAGOLITIC CAPITAL DJERVI 2c0c + case u'Ⰽ': // GLAGOLITIC CAPITAL KAKO 2c0d + case u'Ⰾ': // GLAGOLITIC CAPITAL LJUDIJE 2c0e + case u'Ⰿ': // GLAGOLITIC CAPITAL MYSLITE 2c0f + case u'Ⱀ': // GLAGOLITIC CAPITAL NASHI 2c10 + case u'Ⱁ': // GLAGOLITIC CAPITAL ONU 2c11 + case u'Ⱂ': // GLAGOLITIC CAPITAL POKOJI 2c12 + case u'Ⱃ': // GLAGOLITIC CAPITAL RITSI 2c13 + case u'Ⱄ': // GLAGOLITIC CAPITAL SLOVO 2c14 + case u'Ⱅ': // GLAGOLITIC CAPITAL TVRIDO 2c15 + case u'Ⱆ': // GLAGOLITIC CAPITAL UKU 2c16 + case u'Ⱇ': // GLAGOLITIC CAPITAL FRITU 2c17 + case u'Ⱈ': // GLAGOLITIC CAPITAL HERU 2c18 + case u'Ⱉ': // GLAGOLITIC CAPITAL OTU 2c19 + case u'Ⱊ': // GLAGOLITIC CAPITAL PE 2c1a + case u'Ⱋ': // GLAGOLITIC CAPITAL SHTA 2c1b + case u'Ⱌ': // GLAGOLITIC CAPITAL TSI 2c1c + case u'Ⱍ': // GLAGOLITIC CAPITAL CHRIVI 2c1d + case u'Ⱎ': // GLAGOLITIC CAPITAL SHA 2c1e + case u'Ⱏ': // GLAGOLITIC CAPITAL YERU 2c1f + case u'Ⱐ': // GLAGOLITIC CAPITAL YERI 2c20 + case u'Ⱑ': // GLAGOLITIC CAPITAL YATI 2c21 + case u'Ⱒ': // GLAGOLITIC CAPITAL SPIDERY HA 2c22 + case u'Ⱓ': // GLAGOLITIC CAPITAL YU 2c23 + case u'Ⱔ': // GLAGOLITIC CAPITAL SMALL YUS 2c24 + case u'Ⱕ': // GLAGOLITIC CAPITAL SMALL YUS W/ TAIL 2c25 + case u'Ⱖ': // GLAGOLITIC CAPITAL YO 2c26 + case u'Ⱗ': // GLAGOLITIC CAPITAL IOTATED SMALL YUS 2c27 + case u'Ⱘ': // GLAGOLITIC CAPITAL BIG YUS 2c28 + case u'Ⱙ': // GLAGOLITIC CAPITAL IOTATED BIG YUS 2c29 + case u'Ⱚ': // GLAGOLITIC CAPITAL FITA 2c2a + case u'Ⱛ': // GLAGOLITIC CAPITAL IZHITSA 2c2b + case u'Ⱜ': // GLAGOLITIC CAPITAL SHTAPIC 2c2c + case u'Ⱝ': // GLAGOLITIC CAPITAL TROKUTASTI A 2c2d + case u'Ⱞ': // GLAGOLITIC CAPITAL LATINATE MYSLITE 2c2e + case u'Ⱡ': // LATIN CAPITAL L W/ DOUBLE BAR 2c60 + case u'Ɫ': // LATIN CAPITAL L W/ MIDDLE TILDE 2c62 + case u'Ᵽ': // LATIN CAPITAL P W/ STROKE 2c63 + case u'Ɽ': // LATIN CAPITAL R W/ TAIL 2c64 + case u'Ⱨ': // LATIN CAPITAL H W/ DESCENDER 2c67 + case u'Ⱪ': // LATIN CAPITAL K W/ DESCENDER 2c69 + case u'Ⱬ': // LATIN CAPITAL Z W/ DESCENDER 2c6b + case u'Ɑ': // LATIN CAPITAL ALPHA 2c6d + case u'Ɱ': // LATIN CAPITAL M W/ HOOK 2c6e + case u'Ɐ': // LATIN CAPITAL TURNED A 2c6f + case u'Ɒ': // LATIN CAPITAL TURNED ALPHA 2c70 + case u'Ⱳ': // LATIN CAPITAL W W/ HOOK 2c72 + case u'Ⱶ': // LATIN CAPITAL HALF H 2c75 + case u'Ȿ': // LATIN CAPITAL S W/ SWASH TAIL 2c7e + case u'Ɀ': // LATIN CAPITAL Z W/ SWASH TAIL 2c7f + case u'Ⲁ': // COPTIC CAPITAL ALFA 2c80 + case u'Ⲃ': // COPTIC CAPITAL VIDA 2c82 + case u'Ⲅ': // COPTIC CAPITAL GAMMA 2c84 + case u'Ⲇ': // COPTIC CAPITAL DALDA 2c86 + case u'Ⲉ': // COPTIC CAPITAL EIE 2c88 + case u'Ⲋ': // COPTIC CAPITAL SOU 2c8a + case u'Ⲍ': // COPTIC CAPITAL ZATA 2c8c + case u'Ⲏ': // COPTIC CAPITAL HATE 2c8e + case u'Ⲑ': // COPTIC CAPITAL THETHE 2c90 + case u'Ⲓ': // COPTIC CAPITAL IAUDA 2c92 + case u'Ⲕ': // COPTIC CAPITAL KAPA 2c94 + case u'Ⲗ': // COPTIC CAPITAL LAULA 2c96 + case u'Ⲙ': // COPTIC CAPITAL MI 2c98 + case u'Ⲛ': // COPTIC CAPITAL NI 2c9a + case u'Ⲝ': // COPTIC CAPITAL KSI 2c9c + case u'Ⲟ': // COPTIC CAPITAL O 2c9e + case u'Ⲡ': // COPTIC CAPITAL PI 2ca0 + case u'Ⲣ': // COPTIC CAPITAL RO 2ca2 + case u'Ⲥ': // COPTIC CAPITAL SIMA 2ca4 + case u'Ⲧ': // COPTIC CAPITAL TAU 2ca6 + case u'Ⲩ': // COPTIC CAPITAL UA 2ca8 + case u'Ⲫ': // COPTIC CAPITAL FI 2caa + case u'Ⲭ': // COPTIC CAPITAL KHI 2cac + case u'Ⲯ': // COPTIC CAPITAL PSI 2cae + case u'Ⲱ': // COPTIC CAPITAL OOU 2cb0 + case u'Ⲳ': // COPTIC CAPITAL DIALECT-P ALEF 2cb2 + case u'Ⲵ': // COPTIC CAPITAL OLD COPTIC AIN 2cb4 + case u'Ⲷ': // COPTIC CAPITAL CRYPTOGRAMMIC EIE 2cb6 + case u'Ⲹ': // COPTIC CAPITAL DIALECT-P KAPA 2cb8 + case u'Ⲻ': // COPTIC CAPITAL DIALECT-P NI 2cba + case u'Ⲽ': // COPTIC CAPITAL CRYPTOGRAMMIC NI 2cbc + case u'Ⲿ': // COPTIC CAPITAL OLD COPTIC OOU 2cbe + case u'Ⳁ': // COPTIC CAPITAL SAMPI 2cc0 + case u'Ⳃ': // COPTIC CAPITAL CROSSED SHEI 2cc2 + case u'Ⳅ': // COPTIC CAPITAL OLD COPTIC SHEI 2cc4 + case u'Ⳇ': // COPTIC CAPITAL OLD COPTIC ESH 2cc6 + case u'Ⳉ': // COPTIC CAPITAL AKHMIMIC KHEI 2cc8 + case u'Ⳋ': // COPTIC CAPITAL DIALECT-P HORI 2cca + case u'Ⳍ': // COPTIC CAPITAL OLD COPTIC HORI 2ccc + case u'Ⳏ': // COPTIC CAPITAL OLD COPTIC HA 2cce + case u'Ⳑ': // COPTIC CAPITAL L-SHAPED HA 2cd0 + case u'Ⳓ': // COPTIC CAPITAL OLD COPTIC HEI 2cd2 + case u'Ⳕ': // COPTIC CAPITAL OLD COPTIC HAT 2cd4 + case u'Ⳗ': // COPTIC CAPITAL OLD COPTIC GANGIA 2cd6 + case u'Ⳙ': // COPTIC CAPITAL OLD COPTIC DJA 2cd8 + case u'Ⳛ': // COPTIC CAPITAL OLD COPTIC SHIMA 2cda + case u'Ⳝ': // COPTIC CAPITAL OLD NUBIAN SHIMA 2cdc + case u'Ⳟ': // COPTIC CAPITAL OLD NUBIAN NGI 2cde + case u'Ⳡ': // COPTIC CAPITAL OLD NUBIAN NYI 2ce0 + case u'Ⳣ': // COPTIC CAPITAL OLD NUBIAN WAU 2ce2 + case u'Ⳬ': // COPTIC CAPITAL CRYPTOGRAMMIC SHEI 2ceb + case u'Ⳮ': // COPTIC CAPITAL CRYPTOGRAMMIC GANGIA 2ced + case u'Ⳳ': // COPTIC CAPITAL BOHAIRIC KHEI 2cf2 + case u'Ꙁ': // CYRILLIC CAPITAL ZEMLYA a640 + case u'Ꙃ': // CYRILLIC CAPITAL DZELO a642 + case u'Ꙅ': // CYRILLIC CAPITAL REVERSED DZE a644 + case u'Ꙇ': // CYRILLIC CAPITAL IOTA a646 + case u'Ꙉ': // CYRILLIC CAPITAL DJERV a648 + case u'Ꙋ': // CYRILLIC CAPITAL MONOGRAPH UK a64a + case u'Ꙍ': // CYRILLIC CAPITAL BROAD OMEGA a64c + case u'Ꙏ': // CYRILLIC CAPITAL NEUTRAL YER a64e + case u'Ꙑ': // CYRILLIC CAPITAL YERU W/ BACK YER a650 + case u'Ꙓ': // CYRILLIC CAPITAL IOTIFIED YAT a652 + case u'Ꙕ': // CYRILLIC CAPITAL REVERSED YU a654 + case u'Ꙗ': // CYRILLIC CAPITAL IOTIFIED A a656 + case u'Ꙙ': // CYRILLIC CAPITAL CLOSED LITTLE YUS a658 + case u'Ꙛ': // CYRILLIC CAPITAL BLENDED YUS a65a + case u'Ꙝ': // CYRILLIC CAPITAL IOTIFIED CLOSED LITTLE YUS a65c + case u'Ꙟ': // CYRILLIC CAPITAL YN a65e + case u'Ꙡ': // CYRILLIC CAPITAL REVERSED TSE a660 + case u'Ꙣ': // CYRILLIC CAPITAL SOFT DE a662 + case u'Ꙥ': // CYRILLIC CAPITAL SOFT EL a664 + case u'Ꙧ': // CYRILLIC CAPITAL SOFT EM a666 + case u'Ꙩ': // CYRILLIC CAPITAL MONOCULAR O a668 + case u'Ꙫ': // CYRILLIC CAPITAL BINOCULAR O a66a + case u'Ꙭ': // CYRILLIC CAPITAL DOUBLE MONOCULAR O a66c + case 0xa680: // CYRILLIC CAPITAL DWE a680 + case 0xa682: // CYRILLIC CAPITAL DZWE a682 + case 0xa684: // CYRILLIC CAPITAL ZHWE a684 + case 0xa686: // CYRILLIC CAPITAL CCHE a686 + case 0xa688: // CYRILLIC CAPITAL DZZE a688 + case 0xa68a: // CYRILLIC CAPITAL TE W/ MIDDLE HOOK a68a + case 0xa68c: // CYRILLIC CAPITAL TWE a68c + case 0xa68e: // CYRILLIC CAPITAL TSWE a68e + case 0xa690: // CYRILLIC CAPITAL TSSE a690 + case 0xa692: // CYRILLIC CAPITAL TCHE a692 + case 0xa694: // CYRILLIC CAPITAL HWE a694 + case 0xa696: // CYRILLIC CAPITAL SHWE a696 + case 0xa698: // CYRILLIC CAPITAL DOUBLE O a698 + case 0xa69a: // CYRILLIC CAPITAL CROSSED O a69a + case u'Ꜣ': // LATIN CAPITAL EGYPTOLOGICAL ALEF a722 + case u'Ꜥ': // LATIN CAPITAL EGYPTOLOGICAL AIN a724 + case u'Ꜧ': // LATIN CAPITAL HENG a726 + case u'Ꜩ': // LATIN CAPITAL TZ a728 + case u'Ꜫ': // LATIN CAPITAL TRESILLO a72a + case u'Ꜭ': // LATIN CAPITAL CUATRILLO a72c + case u'Ꜯ': // LATIN CAPITAL CUATRILLO W/ COMMA a72e + case u'Ꜳ': // LATIN CAPITAL AA a732 + case u'Ꜵ': // LATIN CAPITAL AO a734 + case u'Ꜷ': // LATIN CAPITAL AU a736 + case u'Ꜹ': // LATIN CAPITAL AV a738 + case u'Ꜻ': // LATIN CAPITAL AV W/ HORIZONTAL BAR a73a + case u'Ꜽ': // LATIN CAPITAL AY a73c + case u'Ꜿ': // LATIN CAPITAL REVERSED C W/ DOT a73e + case u'Ꝁ': // LATIN CAPITAL K W/ STROKE a740 + case u'Ꝃ': // LATIN CAPITAL K W/ DIAGONAL STROKE a742 + case u'Ꝅ': // LATIN CAPITAL K W/ STROKE AND DIAGONAL STROKE a744 + case u'Ꝇ': // LATIN CAPITAL BROKEN L a746 + case u'Ꝉ': // LATIN CAPITAL L W/ HIGH STROKE a748 + case u'Ꝋ': // LATIN CAPITAL O W/ LONG STROKE OVERLAY a74a + case u'Ꝍ': // LATIN CAPITAL O W/ LOOP a74c + case u'Ꝏ': // LATIN CAPITAL OO a74e + case u'Ꝑ': // LATIN CAPITAL P W/ STROKE THROUGH DESCENDER a750 + case u'Ꝓ': // LATIN CAPITAL P W/ FLOURISH a752 + case u'Ꝕ': // LATIN CAPITAL P W/ SQUIRREL TAIL a754 + case u'Ꝗ': // LATIN CAPITAL Q W/ STROKE THROUGH DESCENDER a756 + case u'Ꝙ': // LATIN CAPITAL Q W/ DIAGONAL STROKE a758 + case u'Ꝛ': // LATIN CAPITAL R ROTUNDA a75a + case u'Ꝝ': // LATIN CAPITAL RUM ROTUNDA a75c + case u'Ꝟ': // LATIN CAPITAL V W/ DIAGONAL STROKE a75e + case u'Ꝡ': // LATIN CAPITAL VY a760 + case u'Ꝣ': // LATIN CAPITAL VISIGOTHIC Z a762 + case u'Ꝥ': // LATIN CAPITAL THORN W/ STROKE a764 + case u'Ꝧ': // LATIN CAPITAL THORN W/ STROKE THROUGH DESCENDER a766 + case u'Ꝩ': // LATIN CAPITAL VEND a768 + case u'Ꝫ': // LATIN CAPITAL ET a76a + case u'Ꝭ': // LATIN CAPITAL IS a76c + case u'Ꝯ': // LATIN CAPITAL CON a76e + case u'Ꝺ': // LATIN CAPITAL INSULAR D a779 + case u'Ꝼ': // LATIN CAPITAL INSULAR F a77b + case u'Ᵹ': // LATIN CAPITAL INSULAR G a77d + case u'Ꝿ': // LATIN CAPITAL TURNED INSULAR G a77e + case u'Ꞁ': // LATIN CAPITAL TURNED L a780 + case u'Ꞃ': // LATIN CAPITAL INSULAR R a782 + case u'Ꞅ': // LATIN CAPITAL INSULAR S a784 + case u'Ꞇ': // LATIN CAPITAL INSULAR T a786 + case u'Ꞌ': // LATIN CAPITAL SALTILLO a78b + case u'Ɥ': // LATIN CAPITAL TURNED H a78d + case u'Ꞑ': // LATIN CAPITAL N W/ DESCENDER a790 + case u'Ꞓ': // LATIN CAPITAL C W/ BAR a792 + case u'Ꞗ': // LATIN CAPITAL B W/ FLOURISH a796 + case u'Ꞙ': // LATIN CAPITAL F W/ STROKE a798 + case u'Ꞛ': // LATIN CAPITAL VOLAPUK AE a79a + case u'Ꞝ': // LATIN CAPITAL VOLAPUK OE a79c + case u'Ꞟ': // LATIN CAPITAL VOLAPUK UE a79e + case u'Ꞡ': // LATIN CAPITAL G W/ OBLIQUE STROKE a7a0 + case u'Ꞣ': // LATIN CAPITAL K W/ OBLIQUE STROKE a7a2 + case u'Ꞥ': // LATIN CAPITAL N W/ OBLIQUE STROKE a7a4 + case u'Ꞧ': // LATIN CAPITAL R W/ OBLIQUE STROKE a7a6 + case u'Ꞩ': // LATIN CAPITAL S W/ OBLIQUE STROKE a7a8 + case u'Ɦ': // LATIN CAPITAL H W/ HOOK a7aa + case u'Ɜ': // LATIN CAPITAL REVERSED OPEN E a7ab + case u'Ɡ': // LATIN CAPITAL SCRIPT G a7ac + case u'Ɬ': // LATIN CAPITAL L W/ BELT a7ad + case u'Ɪ': // LATIN CAPITAL SMALL CAPITAL I a7ae + case u'Ʞ': // LATIN CAPITAL TURNED K a7b0 + case u'Ʇ': // LATIN CAPITAL TURNED T a7b1 + case u'Ʝ': // LATIN CAPITAL J W/ CROSSED-TAIL a7b2 + case u'Ꭓ': // LATIN CAPITAL CHI a7b3 + case u'Ꞵ': // LATIN CAPITAL BETA a7b4 + case u'Ꞷ': // LATIN CAPITAL OMEGA a7b6 + case u'Ꞹ': // LATIN CAPITAL U W/ STROKE a7b8 + case u'Ꞻ': // LATIN CAPITAL GLOTTAL A a7ba + case u'Ꞽ': // LATIN CAPITAL GLOTTAL I a7bc + case u'Ꞿ': // LATIN CAPITAL GLOTTAL U a7be + case u'Ꟃ': // LATIN CAPITAL ANGLICANA W a7c2 + case u'Ꞔ': // LATIN CAPITAL C W/ PALATAL HOOK a7c4 + case u'Ʂ': // LATIN CAPITAL S W/ HOOK a7c5 + case u'Ᶎ': // LATIN CAPITAL Z W/ PALATAL HOOK a7c6 + case u'Ａ': // FULLWIDTH LATIN CAPITAL A ff21 + case u'Ｂ': // FULLWIDTH LATIN CAPITAL B ff22 + case u'Ｃ': // FULLWIDTH LATIN CAPITAL C ff23 + case u'Ｄ': // FULLWIDTH LATIN CAPITAL D ff24 + case u'Ｅ': // FULLWIDTH LATIN CAPITAL E ff25 + case u'Ｆ': // FULLWIDTH LATIN CAPITAL F ff26 + case u'Ｇ': // FULLWIDTH LATIN CAPITAL G ff27 + case u'Ｈ': // FULLWIDTH LATIN CAPITAL H ff28 + case u'Ｉ': // FULLWIDTH LATIN CAPITAL I ff29 + case u'Ｊ': // FULLWIDTH LATIN CAPITAL J ff2a + case u'Ｋ': // FULLWIDTH LATIN CAPITAL K ff2b + case u'Ｌ': // FULLWIDTH LATIN CAPITAL L ff2c + case u'Ｍ': // FULLWIDTH LATIN CAPITAL M ff2d + case u'Ｎ': // FULLWIDTH LATIN CAPITAL N ff2e + case u'Ｏ': // FULLWIDTH LATIN CAPITAL O ff2f + case u'Ｐ': // FULLWIDTH LATIN CAPITAL P ff30 + case u'Ｑ': // FULLWIDTH LATIN CAPITAL Q ff31 + case u'Ｒ': // FULLWIDTH LATIN CAPITAL R ff32 + case u'Ｓ': // FULLWIDTH LATIN CAPITAL S ff33 + case u'Ｔ': // FULLWIDTH LATIN CAPITAL T ff34 + case u'Ｕ': // FULLWIDTH LATIN CAPITAL U ff35 + case u'Ｖ': // FULLWIDTH LATIN CAPITAL V ff36 + case u'Ｗ': // FULLWIDTH LATIN CAPITAL W ff37 + case u'Ｘ': // FULLWIDTH LATIN CAPITAL X ff38 + case u'Ｙ': // FULLWIDTH LATIN CAPITAL Y ff39 + case u'Ｚ': // FULLWIDTH LATIN CAPITAL Z ff3a + case L'𐐀': // DESERET CAPITAL LONG I 10400 + case L'𐐁': // DESERET CAPITAL LONG E 10401 + case L'𐐂': // DESERET CAPITAL LONG A 10402 + case L'𐐃': // DESERET CAPITAL LONG AH 10403 + case L'𐐄': // DESERET CAPITAL LONG O 10404 + case L'𐐅': // DESERET CAPITAL LONG OO 10405 + case L'𐐆': // DESERET CAPITAL SHORT I 10406 + case L'𐐇': // DESERET CAPITAL SHORT E 10407 + case L'𐐈': // DESERET CAPITAL SHORT A 10408 + case L'𐐉': // DESERET CAPITAL SHORT AH 10409 + case L'𐐊': // DESERET CAPITAL SHORT O 1040a + case L'𐐋': // DESERET CAPITAL SHORT OO 1040b + case L'𐐌': // DESERET CAPITAL AY 1040c + case L'𐐍': // DESERET CAPITAL OW 1040d + case L'𐐎': // DESERET CAPITAL WU 1040e + case L'𐐏': // DESERET CAPITAL YEE 1040f + case L'𐐐': // DESERET CAPITAL H 10410 + case L'𐐑': // DESERET CAPITAL PEE 10411 + case L'𐐒': // DESERET CAPITAL BEE 10412 + case L'𐐓': // DESERET CAPITAL TEE 10413 + case L'𐐔': // DESERET CAPITAL DEE 10414 + case L'𐐕': // DESERET CAPITAL CHEE 10415 + case L'𐐖': // DESERET CAPITAL JEE 10416 + case L'𐐗': // DESERET CAPITAL KAY 10417 + case L'𐐘': // DESERET CAPITAL GAY 10418 + case L'𐐙': // DESERET CAPITAL EF 10419 + case L'𐐚': // DESERET CAPITAL VEE 1041a + case L'𐐛': // DESERET CAPITAL ETH 1041b + case L'𐐜': // DESERET CAPITAL THEE 1041c + case L'𐐝': // DESERET CAPITAL ES 1041d + case L'𐐞': // DESERET CAPITAL ZEE 1041e + case L'𐐟': // DESERET CAPITAL ESH 1041f + case L'𐐠': // DESERET CAPITAL ZHEE 10420 + case L'𐐡': // DESERET CAPITAL ER 10421 + case L'𐐢': // DESERET CAPITAL EL 10422 + case L'𐐣': // DESERET CAPITAL EM 10423 + case L'𐐤': // DESERET CAPITAL EN 10424 + case L'𐐥': // DESERET CAPITAL ENG 10425 + case L'𐐦': // DESERET CAPITAL OI 10426 + case L'𐐧': // DESERET CAPITAL EW 10427 + case L'𐒰': // OSAGE CAPITAL A 104b0 + case L'𐒱': // OSAGE CAPITAL AI 104b1 + case L'𐒲': // OSAGE CAPITAL AIN 104b2 + case L'𐒳': // OSAGE CAPITAL AH 104b3 + case L'𐒴': // OSAGE CAPITAL BRA 104b4 + case L'𐒵': // OSAGE CAPITAL CHA 104b5 + case L'𐒶': // OSAGE CAPITAL EHCHA 104b6 + case L'𐒷': // OSAGE CAPITAL E 104b7 + case L'𐒸': // OSAGE CAPITAL EIN 104b8 + case L'𐒹': // OSAGE CAPITAL HA 104b9 + case L'𐒺': // OSAGE CAPITAL HYA 104ba + case L'𐒻': // OSAGE CAPITAL I 104bb + case L'𐒼': // OSAGE CAPITAL KA 104bc + case L'𐒽': // OSAGE CAPITAL EHKA 104bd + case L'𐒾': // OSAGE CAPITAL KYA 104be + case L'𐒿': // OSAGE CAPITAL LA 104bf + case L'𐓀': // OSAGE CAPITAL MA 104c0 + case L'𐓁': // OSAGE CAPITAL NA 104c1 + case L'𐓂': // OSAGE CAPITAL O 104c2 + case L'𐓃': // OSAGE CAPITAL OIN 104c3 + case L'𐓄': // OSAGE CAPITAL PA 104c4 + case L'𐓅': // OSAGE CAPITAL EHPA 104c5 + case L'𐓆': // OSAGE CAPITAL SA 104c6 + case L'𐓇': // OSAGE CAPITAL SHA 104c7 + case L'𐓈': // OSAGE CAPITAL TA 104c8 + case L'𐓉': // OSAGE CAPITAL EHTA 104c9 + case L'𐓊': // OSAGE CAPITAL TSA 104ca + case L'𐓋': // OSAGE CAPITAL EHTSA 104cb + case L'𐓌': // OSAGE CAPITAL TSHA 104cc + case L'𐓍': // OSAGE CAPITAL DHA 104cd + case L'𐓎': // OSAGE CAPITAL U 104ce + case L'𐓏': // OSAGE CAPITAL WA 104cf + case L'𐓐': // OSAGE CAPITAL KHA 104d0 + case L'𐓑': // OSAGE CAPITAL GHA 104d1 + case L'𐓒': // OSAGE CAPITAL ZA 104d2 + case L'𐓓': // OSAGE CAPITAL ZHA 104d3 + case L'𐲀': // OLD HUNGARIAN CAPITAL A 10c80 + case L'𐲁': // OLD HUNGARIAN CAPITAL AA 10c81 + case L'𐲂': // OLD HUNGARIAN CAPITAL EB 10c82 + case L'𐲃': // OLD HUNGARIAN CAPITAL AMB 10c83 + case L'𐲄': // OLD HUNGARIAN CAPITAL EC 10c84 + case L'𐲅': // OLD HUNGARIAN CAPITAL ENC 10c85 + case L'𐲆': // OLD HUNGARIAN CAPITAL ECS 10c86 + case L'𐲇': // OLD HUNGARIAN CAPITAL ED 10c87 + case L'𐲈': // OLD HUNGARIAN CAPITAL AND 10c88 + case L'𐲉': // OLD HUNGARIAN CAPITAL E 10c89 + case L'𐲊': // OLD HUNGARIAN CAPITAL CLOSE E 10c8a + case L'𐲋': // OLD HUNGARIAN CAPITAL EE 10c8b + case L'𐲌': // OLD HUNGARIAN CAPITAL EF 10c8c + case L'𐲍': // OLD HUNGARIAN CAPITAL EG 10c8d + case L'𐲎': // OLD HUNGARIAN CAPITAL EGY 10c8e + case L'𐲏': // OLD HUNGARIAN CAPITAL EH 10c8f + case L'𐲐': // OLD HUNGARIAN CAPITAL I 10c90 + case L'𐲑': // OLD HUNGARIAN CAPITAL II 10c91 + case L'𐲒': // OLD HUNGARIAN CAPITAL EJ 10c92 + case L'𐲓': // OLD HUNGARIAN CAPITAL EK 10c93 + case L'𐲔': // OLD HUNGARIAN CAPITAL AK 10c94 + case L'𐲕': // OLD HUNGARIAN CAPITAL UNK 10c95 + case L'𐲖': // OLD HUNGARIAN CAPITAL EL 10c96 + case L'𐲗': // OLD HUNGARIAN CAPITAL ELY 10c97 + case L'𐲘': // OLD HUNGARIAN CAPITAL EM 10c98 + case L'𐲙': // OLD HUNGARIAN CAPITAL EN 10c99 + case L'𐲚': // OLD HUNGARIAN CAPITAL ENY 10c9a + case L'𐲛': // OLD HUNGARIAN CAPITAL O 10c9b + case L'𐲜': // OLD HUNGARIAN CAPITAL OO 10c9c + case L'𐲝': // OLD HUNGARIAN CAPITAL NIKOLSBURG OE 10c9d + case L'𐲞': // OLD HUNGARIAN CAPITAL RUDIMENTA OE 10c9e + case L'𐲟': // OLD HUNGARIAN CAPITAL OEE 10c9f + case L'𐲠': // OLD HUNGARIAN CAPITAL EP 10ca0 + case L'𐲡': // OLD HUNGARIAN CAPITAL EMP 10ca1 + case L'𐲢': // OLD HUNGARIAN CAPITAL ER 10ca2 + case L'𐲣': // OLD HUNGARIAN CAPITAL SHORT ER 10ca3 + case L'𐲤': // OLD HUNGARIAN CAPITAL ES 10ca4 + case L'𐲥': // OLD HUNGARIAN CAPITAL ESZ 10ca5 + case L'𐲦': // OLD HUNGARIAN CAPITAL ET 10ca6 + case L'𐲧': // OLD HUNGARIAN CAPITAL ENT 10ca7 + case L'𐲨': // OLD HUNGARIAN CAPITAL ETY 10ca8 + case L'𐲩': // OLD HUNGARIAN CAPITAL ECH 10ca9 + case L'𐲪': // OLD HUNGARIAN CAPITAL U 10caa + case L'𐲫': // OLD HUNGARIAN CAPITAL UU 10cab + case L'𐲬': // OLD HUNGARIAN CAPITAL NIKOLSBURG UE 10cac + case L'𐲭': // OLD HUNGARIAN CAPITAL RUDIMENTA UE 10cad + case L'𐲮': // OLD HUNGARIAN CAPITAL EV 10cae + case L'𐲯': // OLD HUNGARIAN CAPITAL EZ 10caf + case L'𐲰': // OLD HUNGARIAN CAPITAL EZS 10cb0 + case L'𐲱': // OLD HUNGARIAN CAPITAL ENT-SHAPED SIGN 10cb1 + case L'𐲲': // OLD HUNGARIAN CAPITAL US 10cb2 + case L'𑢠': // WARANG CITI CAPITAL NGAA 118a0 + case L'𑢡': // WARANG CITI CAPITAL A 118a1 + case L'𑢢': // WARANG CITI CAPITAL WI 118a2 + case L'𑢣': // WARANG CITI CAPITAL YU 118a3 + case L'𑢤': // WARANG CITI CAPITAL YA 118a4 + case L'𑢥': // WARANG CITI CAPITAL YO 118a5 + case L'𑢦': // WARANG CITI CAPITAL II 118a6 + case L'𑢧': // WARANG CITI CAPITAL UU 118a7 + case L'𑢨': // WARANG CITI CAPITAL E 118a8 + case L'𑢩': // WARANG CITI CAPITAL O 118a9 + case L'𑢪': // WARANG CITI CAPITAL ANG 118aa + case L'𑢫': // WARANG CITI CAPITAL GA 118ab + case L'𑢬': // WARANG CITI CAPITAL KO 118ac + case L'𑢭': // WARANG CITI CAPITAL ENY 118ad + case L'𑢮': // WARANG CITI CAPITAL YUJ 118ae + case L'𑢯': // WARANG CITI CAPITAL UC 118af + case L'𑢰': // WARANG CITI CAPITAL ENN 118b0 + case L'𑢱': // WARANG CITI CAPITAL ODD 118b1 + case L'𑢲': // WARANG CITI CAPITAL TTE 118b2 + case L'𑢳': // WARANG CITI CAPITAL NUNG 118b3 + case L'𑢴': // WARANG CITI CAPITAL DA 118b4 + case L'𑢵': // WARANG CITI CAPITAL AT 118b5 + case L'𑢶': // WARANG CITI CAPITAL AM 118b6 + case L'𑢷': // WARANG CITI CAPITAL BU 118b7 + case L'𑢸': // WARANG CITI CAPITAL PU 118b8 + case L'𑢹': // WARANG CITI CAPITAL HIYO 118b9 + case L'𑢺': // WARANG CITI CAPITAL HOLO 118ba + case L'𑢻': // WARANG CITI CAPITAL HORR 118bb + case L'𑢼': // WARANG CITI CAPITAL HAR 118bc + case L'𑢽': // WARANG CITI CAPITAL SSUU 118bd + case L'𑢾': // WARANG CITI CAPITAL SII 118be + case L'𑢿': // WARANG CITI CAPITAL VIYO 118bf + case L'𖹀': // MEDEFAIDRIN CAPITAL M 16e40 + case L'𖹁': // MEDEFAIDRIN CAPITAL S 16e41 + case L'𖹂': // MEDEFAIDRIN CAPITAL V 16e42 + case L'𖹃': // MEDEFAIDRIN CAPITAL W 16e43 + case L'𖹄': // MEDEFAIDRIN CAPITAL ATIU 16e44 + case L'𖹅': // MEDEFAIDRIN CAPITAL Z 16e45 + case L'𖹆': // MEDEFAIDRIN CAPITAL KP 16e46 + case L'𖹇': // MEDEFAIDRIN CAPITAL P 16e47 + case L'𖹈': // MEDEFAIDRIN CAPITAL T 16e48 + case L'𖹉': // MEDEFAIDRIN CAPITAL G 16e49 + case L'𖹊': // MEDEFAIDRIN CAPITAL F 16e4a + case L'𖹋': // MEDEFAIDRIN CAPITAL I 16e4b + case L'𖹌': // MEDEFAIDRIN CAPITAL K 16e4c + case L'𖹍': // MEDEFAIDRIN CAPITAL A 16e4d + case L'𖹎': // MEDEFAIDRIN CAPITAL J 16e4e + case L'𖹏': // MEDEFAIDRIN CAPITAL E 16e4f + case L'𖹐': // MEDEFAIDRIN CAPITAL B 16e50 + case L'𖹑': // MEDEFAIDRIN CAPITAL C 16e51 + case L'𖹒': // MEDEFAIDRIN CAPITAL U 16e52 + case L'𖹓': // MEDEFAIDRIN CAPITAL YU 16e53 + case L'𖹔': // MEDEFAIDRIN CAPITAL L 16e54 + case L'𖹕': // MEDEFAIDRIN CAPITAL Q 16e55 + case L'𖹖': // MEDEFAIDRIN CAPITAL HP 16e56 + case L'𖹗': // MEDEFAIDRIN CAPITAL NY 16e57 + case L'𖹘': // MEDEFAIDRIN CAPITAL X 16e58 + case L'𖹙': // MEDEFAIDRIN CAPITAL D 16e59 + case L'𖹚': // MEDEFAIDRIN CAPITAL OE 16e5a + case L'𖹛': // MEDEFAIDRIN CAPITAL N 16e5b + case L'𖹜': // MEDEFAIDRIN CAPITAL R 16e5c + case L'𖹝': // MEDEFAIDRIN CAPITAL O 16e5d + case L'𖹞': // MEDEFAIDRIN CAPITAL AI 16e5e + case L'𖹟': // MEDEFAIDRIN CAPITAL Y 16e5f + case L'𝐀': // MATHEMATICAL BOLD CAPITAL A 1d400 + case L'𝐁': // MATHEMATICAL BOLD CAPITAL B 1d401 + case L'𝐂': // MATHEMATICAL BOLD CAPITAL C 1d402 + case L'𝐃': // MATHEMATICAL BOLD CAPITAL D 1d403 + case L'𝐄': // MATHEMATICAL BOLD CAPITAL E 1d404 + case L'𝐅': // MATHEMATICAL BOLD CAPITAL F 1d405 + case L'𝐆': // MATHEMATICAL BOLD CAPITAL G 1d406 + case L'𝐇': // MATHEMATICAL BOLD CAPITAL H 1d407 + case L'𝐈': // MATHEMATICAL BOLD CAPITAL I 1d408 + case L'𝐉': // MATHEMATICAL BOLD CAPITAL J 1d409 + case L'𝐊': // MATHEMATICAL BOLD CAPITAL K 1d40a + case L'𝐋': // MATHEMATICAL BOLD CAPITAL L 1d40b + case L'𝐌': // MATHEMATICAL BOLD CAPITAL M 1d40c + case L'𝐍': // MATHEMATICAL BOLD CAPITAL N 1d40d + case L'𝐎': // MATHEMATICAL BOLD CAPITAL O 1d40e + case L'𝐏': // MATHEMATICAL BOLD CAPITAL P 1d40f + case L'𝐐': // MATHEMATICAL BOLD CAPITAL Q 1d410 + case L'𝐑': // MATHEMATICAL BOLD CAPITAL R 1d411 + case L'𝐒': // MATHEMATICAL BOLD CAPITAL S 1d412 + case L'𝐓': // MATHEMATICAL BOLD CAPITAL T 1d413 + case L'𝐔': // MATHEMATICAL BOLD CAPITAL U 1d414 + case L'𝐕': // MATHEMATICAL BOLD CAPITAL V 1d415 + case L'𝐖': // MATHEMATICAL BOLD CAPITAL W 1d416 + case L'𝐗': // MATHEMATICAL BOLD CAPITAL X 1d417 + case L'𝐘': // MATHEMATICAL BOLD CAPITAL Y 1d418 + case L'𝐙': // MATHEMATICAL BOLD CAPITAL Z 1d419 + case L'𝐴': // MATHEMATICAL ITALIC CAPITAL A 1d434 + case L'𝐵': // MATHEMATICAL ITALIC CAPITAL B 1d435 + case L'𝐶': // MATHEMATICAL ITALIC CAPITAL C 1d436 + case L'𝐷': // MATHEMATICAL ITALIC CAPITAL D 1d437 + case L'𝐸': // MATHEMATICAL ITALIC CAPITAL E 1d438 + case L'𝐹': // MATHEMATICAL ITALIC CAPITAL F 1d439 + case L'𝐺': // MATHEMATICAL ITALIC CAPITAL G 1d43a + case L'𝐻': // MATHEMATICAL ITALIC CAPITAL H 1d43b + case L'𝐼': // MATHEMATICAL ITALIC CAPITAL I 1d43c + case L'𝐽': // MATHEMATICAL ITALIC CAPITAL J 1d43d + case L'𝐾': // MATHEMATICAL ITALIC CAPITAL K 1d43e + case L'𝐿': // MATHEMATICAL ITALIC CAPITAL L 1d43f + case L'𝑀': // MATHEMATICAL ITALIC CAPITAL M 1d440 + case L'𝑁': // MATHEMATICAL ITALIC CAPITAL N 1d441 + case L'𝑂': // MATHEMATICAL ITALIC CAPITAL O 1d442 + case L'𝑃': // MATHEMATICAL ITALIC CAPITAL P 1d443 + case L'𝑄': // MATHEMATICAL ITALIC CAPITAL Q 1d444 + case L'𝑅': // MATHEMATICAL ITALIC CAPITAL R 1d445 + case L'𝑆': // MATHEMATICAL ITALIC CAPITAL S 1d446 + case L'𝑇': // MATHEMATICAL ITALIC CAPITAL T 1d447 + case L'𝑈': // MATHEMATICAL ITALIC CAPITAL U 1d448 + case L'𝑉': // MATHEMATICAL ITALIC CAPITAL V 1d449 + case L'𝑊': // MATHEMATICAL ITALIC CAPITAL W 1d44a + case L'𝑋': // MATHEMATICAL ITALIC CAPITAL X 1d44b + case L'𝑌': // MATHEMATICAL ITALIC CAPITAL Y 1d44c + case L'𝑍': // MATHEMATICAL ITALIC CAPITAL Z 1d44d + case L'𝑨': // MATHEMATICAL BOLD ITALIC CAPITAL A 1d468 + case L'𝑩': // MATHEMATICAL BOLD ITALIC CAPITAL B 1d469 + case L'𝑪': // MATHEMATICAL BOLD ITALIC CAPITAL C 1d46a + case L'𝑫': // MATHEMATICAL BOLD ITALIC CAPITAL D 1d46b + case L'𝑬': // MATHEMATICAL BOLD ITALIC CAPITAL E 1d46c + case L'𝑭': // MATHEMATICAL BOLD ITALIC CAPITAL F 1d46d + case L'𝑮': // MATHEMATICAL BOLD ITALIC CAPITAL G 1d46e + case L'𝑯': // MATHEMATICAL BOLD ITALIC CAPITAL H 1d46f + case L'𝑰': // MATHEMATICAL BOLD ITALIC CAPITAL I 1d470 + case L'𝑱': // MATHEMATICAL BOLD ITALIC CAPITAL J 1d471 + case L'𝑲': // MATHEMATICAL BOLD ITALIC CAPITAL K 1d472 + case L'𝑳': // MATHEMATICAL BOLD ITALIC CAPITAL L 1d473 + case L'𝑴': // MATHEMATICAL BOLD ITALIC CAPITAL M 1d474 + case L'𝑵': // MATHEMATICAL BOLD ITALIC CAPITAL N 1d475 + case L'𝑶': // MATHEMATICAL BOLD ITALIC CAPITAL O 1d476 + case L'𝑷': // MATHEMATICAL BOLD ITALIC CAPITAL P 1d477 + case L'𝑸': // MATHEMATICAL BOLD ITALIC CAPITAL Q 1d478 + case L'𝑹': // MATHEMATICAL BOLD ITALIC CAPITAL R 1d479 + case L'𝑺': // MATHEMATICAL BOLD ITALIC CAPITAL S 1d47a + case L'𝑻': // MATHEMATICAL BOLD ITALIC CAPITAL T 1d47b + case L'𝑼': // MATHEMATICAL BOLD ITALIC CAPITAL U 1d47c + case L'𝑽': // MATHEMATICAL BOLD ITALIC CAPITAL V 1d47d + case L'𝑾': // MATHEMATICAL BOLD ITALIC CAPITAL W 1d47e + case L'𝑿': // MATHEMATICAL BOLD ITALIC CAPITAL X 1d47f + case L'𝒀': // MATHEMATICAL BOLD ITALIC CAPITAL Y 1d480 + case L'𝒁': // MATHEMATICAL BOLD ITALIC CAPITAL Z 1d481 + case L'𝒜': // MATHEMATICAL SCRIPT CAPITAL A 1d49c + case L'𝒞': // MATHEMATICAL SCRIPT CAPITAL C 1d49e + case L'𝒟': // MATHEMATICAL SCRIPT CAPITAL D 1d49f + case L'𝒢': // MATHEMATICAL SCRIPT CAPITAL G 1d4a2 + case L'𝒥': // MATHEMATICAL SCRIPT CAPITAL J 1d4a5 + case L'𝒦': // MATHEMATICAL SCRIPT CAPITAL K 1d4a6 + case L'𝒩': // MATHEMATICAL SCRIPT CAPITAL N 1d4a9 + case L'𝒪': // MATHEMATICAL SCRIPT CAPITAL O 1d4aa + case L'𝒫': // MATHEMATICAL SCRIPT CAPITAL P 1d4ab + case L'𝒬': // MATHEMATICAL SCRIPT CAPITAL Q 1d4ac + case L'𝒮': // MATHEMATICAL SCRIPT CAPITAL S 1d4ae + case L'𝒯': // MATHEMATICAL SCRIPT CAPITAL T 1d4af + case L'𝒰': // MATHEMATICAL SCRIPT CAPITAL U 1d4b0 + case L'𝒱': // MATHEMATICAL SCRIPT CAPITAL V 1d4b1 + case L'𝒲': // MATHEMATICAL SCRIPT CAPITAL W 1d4b2 + case L'𝒳': // MATHEMATICAL SCRIPT CAPITAL X 1d4b3 + case L'𝒴': // MATHEMATICAL SCRIPT CAPITAL Y 1d4b4 + case L'𝒵': // MATHEMATICAL SCRIPT CAPITAL Z 1d4b5 + case L'𝓐': // MATHEMATICAL BOLD SCRIPT CAPITAL A 1d4d0 + case L'𝓑': // MATHEMATICAL BOLD SCRIPT CAPITAL B 1d4d1 + case L'𝓒': // MATHEMATICAL BOLD SCRIPT CAPITAL C 1d4d2 + case L'𝓓': // MATHEMATICAL BOLD SCRIPT CAPITAL D 1d4d3 + case L'𝓔': // MATHEMATICAL BOLD SCRIPT CAPITAL E 1d4d4 + case L'𝓕': // MATHEMATICAL BOLD SCRIPT CAPITAL F 1d4d5 + case L'𝓖': // MATHEMATICAL BOLD SCRIPT CAPITAL G 1d4d6 + case L'𝓗': // MATHEMATICAL BOLD SCRIPT CAPITAL H 1d4d7 + case L'𝓘': // MATHEMATICAL BOLD SCRIPT CAPITAL I 1d4d8 + case L'𝓙': // MATHEMATICAL BOLD SCRIPT CAPITAL J 1d4d9 + case L'𝓚': // MATHEMATICAL BOLD SCRIPT CAPITAL K 1d4da + case L'𝓛': // MATHEMATICAL BOLD SCRIPT CAPITAL L 1d4db + case L'𝓜': // MATHEMATICAL BOLD SCRIPT CAPITAL M 1d4dc + case L'𝓝': // MATHEMATICAL BOLD SCRIPT CAPITAL N 1d4dd + case L'𝓞': // MATHEMATICAL BOLD SCRIPT CAPITAL O 1d4de + case L'𝓟': // MATHEMATICAL BOLD SCRIPT CAPITAL P 1d4df + case L'𝓠': // MATHEMATICAL BOLD SCRIPT CAPITAL Q 1d4e0 + case L'𝓡': // MATHEMATICAL BOLD SCRIPT CAPITAL R 1d4e1 + case L'𝓢': // MATHEMATICAL BOLD SCRIPT CAPITAL S 1d4e2 + case L'𝓣': // MATHEMATICAL BOLD SCRIPT CAPITAL T 1d4e3 + case L'𝓤': // MATHEMATICAL BOLD SCRIPT CAPITAL U 1d4e4 + case L'𝓥': // MATHEMATICAL BOLD SCRIPT CAPITAL V 1d4e5 + case L'𝓦': // MATHEMATICAL BOLD SCRIPT CAPITAL W 1d4e6 + case L'𝓧': // MATHEMATICAL BOLD SCRIPT CAPITAL X 1d4e7 + case L'𝓨': // MATHEMATICAL BOLD SCRIPT CAPITAL Y 1d4e8 + case L'𝓩': // MATHEMATICAL BOLD SCRIPT CAPITAL Z 1d4e9 + case L'𝔄': // MATHEMATICAL FRAKTUR CAPITAL A 1d504 + case L'𝔅': // MATHEMATICAL FRAKTUR CAPITAL B 1d505 + case L'𝔇': // MATHEMATICAL FRAKTUR CAPITAL D 1d507 + case L'𝔈': // MATHEMATICAL FRAKTUR CAPITAL E 1d508 + case L'𝔉': // MATHEMATICAL FRAKTUR CAPITAL F 1d509 + case L'𝔊': // MATHEMATICAL FRAKTUR CAPITAL G 1d50a + case L'𝔍': // MATHEMATICAL FRAKTUR CAPITAL J 1d50d + case L'𝔎': // MATHEMATICAL FRAKTUR CAPITAL K 1d50e + case L'𝔏': // MATHEMATICAL FRAKTUR CAPITAL L 1d50f + case L'𝔐': // MATHEMATICAL FRAKTUR CAPITAL M 1d510 + case L'𝔑': // MATHEMATICAL FRAKTUR CAPITAL N 1d511 + case L'𝔒': // MATHEMATICAL FRAKTUR CAPITAL O 1d512 + case L'𝔓': // MATHEMATICAL FRAKTUR CAPITAL P 1d513 + case L'𝔔': // MATHEMATICAL FRAKTUR CAPITAL Q 1d514 + case L'𝔖': // MATHEMATICAL FRAKTUR CAPITAL S 1d516 + case L'𝔗': // MATHEMATICAL FRAKTUR CAPITAL T 1d517 + case L'𝔘': // MATHEMATICAL FRAKTUR CAPITAL U 1d518 + case L'𝔙': // MATHEMATICAL FRAKTUR CAPITAL V 1d519 + case L'𝔚': // MATHEMATICAL FRAKTUR CAPITAL W 1d51a + case L'𝔛': // MATHEMATICAL FRAKTUR CAPITAL X 1d51b + case L'𝔜': // MATHEMATICAL FRAKTUR CAPITAL Y 1d51c + case L'𝔸': // MATHEMATICAL DOUBLE-STRUCK CAPITAL A 1d538 + case L'𝔹': // MATHEMATICAL DOUBLE-STRUCK CAPITAL B 1d539 + case L'𝔻': // MATHEMATICAL DOUBLE-STRUCK CAPITAL D 1d53b + case L'𝔼': // MATHEMATICAL DOUBLE-STRUCK CAPITAL E 1d53c + case L'𝔽': // MATHEMATICAL DOUBLE-STRUCK CAPITAL F 1d53d + case L'𝔾': // MATHEMATICAL DOUBLE-STRUCK CAPITAL G 1d53e + case L'𝕀': // MATHEMATICAL DOUBLE-STRUCK CAPITAL I 1d540 + case L'𝕁': // MATHEMATICAL DOUBLE-STRUCK CAPITAL J 1d541 + case L'𝕂': // MATHEMATICAL DOUBLE-STRUCK CAPITAL K 1d542 + case L'𝕃': // MATHEMATICAL DOUBLE-STRUCK CAPITAL L 1d543 + case L'𝕄': // MATHEMATICAL DOUBLE-STRUCK CAPITAL M 1d544 + case L'𝕆': // MATHEMATICAL DOUBLE-STRUCK CAPITAL O 1d546 + case L'𝕊': // MATHEMATICAL DOUBLE-STRUCK CAPITAL S 1d54a + case L'𝕋': // MATHEMATICAL DOUBLE-STRUCK CAPITAL T 1d54b + case L'𝕌': // MATHEMATICAL DOUBLE-STRUCK CAPITAL U 1d54c + case L'𝕍': // MATHEMATICAL DOUBLE-STRUCK CAPITAL V 1d54d + case L'𝕎': // MATHEMATICAL DOUBLE-STRUCK CAPITAL W 1d54e + case L'𝕏': // MATHEMATICAL DOUBLE-STRUCK CAPITAL X 1d54f + case L'𝕐': // MATHEMATICAL DOUBLE-STRUCK CAPITAL Y 1d550 + case L'𝕬': // MATHEMATICAL BOLD FRAKTUR CAPITAL A 1d56c + case L'𝕭': // MATHEMATICAL BOLD FRAKTUR CAPITAL B 1d56d + case L'𝕮': // MATHEMATICAL BOLD FRAKTUR CAPITAL C 1d56e + case L'𝕯': // MATHEMATICAL BOLD FRAKTUR CAPITAL D 1d56f + case L'𝕰': // MATHEMATICAL BOLD FRAKTUR CAPITAL E 1d570 + case L'𝕱': // MATHEMATICAL BOLD FRAKTUR CAPITAL F 1d571 + case L'𝕲': // MATHEMATICAL BOLD FRAKTUR CAPITAL G 1d572 + case L'𝕳': // MATHEMATICAL BOLD FRAKTUR CAPITAL H 1d573 + case L'𝕴': // MATHEMATICAL BOLD FRAKTUR CAPITAL I 1d574 + case L'𝕵': // MATHEMATICAL BOLD FRAKTUR CAPITAL J 1d575 + case L'𝕶': // MATHEMATICAL BOLD FRAKTUR CAPITAL K 1d576 + case L'𝕷': // MATHEMATICAL BOLD FRAKTUR CAPITAL L 1d577 + case L'𝕸': // MATHEMATICAL BOLD FRAKTUR CAPITAL M 1d578 + case L'𝕹': // MATHEMATICAL BOLD FRAKTUR CAPITAL N 1d579 + case L'𝕺': // MATHEMATICAL BOLD FRAKTUR CAPITAL O 1d57a + case L'𝕻': // MATHEMATICAL BOLD FRAKTUR CAPITAL P 1d57b + case L'𝕼': // MATHEMATICAL BOLD FRAKTUR CAPITAL Q 1d57c + case L'𝕽': // MATHEMATICAL BOLD FRAKTUR CAPITAL R 1d57d + case L'𝕾': // MATHEMATICAL BOLD FRAKTUR CAPITAL S 1d57e + case L'𝕿': // MATHEMATICAL BOLD FRAKTUR CAPITAL T 1d57f + case L'𝖀': // MATHEMATICAL BOLD FRAKTUR CAPITAL U 1d580 + case L'𝖁': // MATHEMATICAL BOLD FRAKTUR CAPITAL V 1d581 + case L'𝖂': // MATHEMATICAL BOLD FRAKTUR CAPITAL W 1d582 + case L'𝖃': // MATHEMATICAL BOLD FRAKTUR CAPITAL X 1d583 + case L'𝖄': // MATHEMATICAL BOLD FRAKTUR CAPITAL Y 1d584 + case L'𝖅': // MATHEMATICAL BOLD FRAKTUR CAPITAL Z 1d585 + case L'𝖠': // MATHEMATICAL SANS CAPITAL A 1d5a0 + case L'𝖡': // MATHEMATICAL SANS CAPITAL B 1d5a1 + case L'𝖢': // MATHEMATICAL SANS CAPITAL C 1d5a2 + case L'𝖣': // MATHEMATICAL SANS CAPITAL D 1d5a3 + case L'𝖤': // MATHEMATICAL SANS CAPITAL E 1d5a4 + case L'𝖥': // MATHEMATICAL SANS CAPITAL F 1d5a5 + case L'𝖦': // MATHEMATICAL SANS CAPITAL G 1d5a6 + case L'𝖧': // MATHEMATICAL SANS CAPITAL H 1d5a7 + case L'𝖨': // MATHEMATICAL SANS CAPITAL I 1d5a8 + case L'𝖩': // MATHEMATICAL SANS CAPITAL J 1d5a9 + case L'𝖪': // MATHEMATICAL SANS CAPITAL K 1d5aa + case L'𝖫': // MATHEMATICAL SANS CAPITAL L 1d5ab + case L'𝖬': // MATHEMATICAL SANS CAPITAL M 1d5ac + case L'𝖭': // MATHEMATICAL SANS CAPITAL N 1d5ad + case L'𝖮': // MATHEMATICAL SANS CAPITAL O 1d5ae + case L'𝖯': // MATHEMATICAL SANS CAPITAL P 1d5af + case L'𝖰': // MATHEMATICAL SANS CAPITAL Q 1d5b0 + case L'𝖱': // MATHEMATICAL SANS CAPITAL R 1d5b1 + case L'𝖲': // MATHEMATICAL SANS CAPITAL S 1d5b2 + case L'𝖳': // MATHEMATICAL SANS CAPITAL T 1d5b3 + case L'𝖴': // MATHEMATICAL SANS CAPITAL U 1d5b4 + case L'𝖵': // MATHEMATICAL SANS CAPITAL V 1d5b5 + case L'𝖶': // MATHEMATICAL SANS CAPITAL W 1d5b6 + case L'𝖷': // MATHEMATICAL SANS CAPITAL X 1d5b7 + case L'𝖸': // MATHEMATICAL SANS CAPITAL Y 1d5b8 + case L'𝖹': // MATHEMATICAL SANS CAPITAL Z 1d5b9 + case L'𝗔': // MATHEMATICAL SANS BOLD CAPITAL A 1d5d4 + case L'𝗕': // MATHEMATICAL SANS BOLD CAPITAL B 1d5d5 + case L'𝗖': // MATHEMATICAL SANS BOLD CAPITAL C 1d5d6 + case L'𝗗': // MATHEMATICAL SANS BOLD CAPITAL D 1d5d7 + case L'𝗘': // MATHEMATICAL SANS BOLD CAPITAL E 1d5d8 + case L'𝗙': // MATHEMATICAL SANS BOLD CAPITAL F 1d5d9 + case L'𝗚': // MATHEMATICAL SANS BOLD CAPITAL G 1d5da + case L'𝗛': // MATHEMATICAL SANS BOLD CAPITAL H 1d5db + case L'𝗜': // MATHEMATICAL SANS BOLD CAPITAL I 1d5dc + case L'𝗝': // MATHEMATICAL SANS BOLD CAPITAL J 1d5dd + case L'𝗞': // MATHEMATICAL SANS BOLD CAPITAL K 1d5de + case L'𝗟': // MATHEMATICAL SANS BOLD CAPITAL L 1d5df + case L'𝗠': // MATHEMATICAL SANS BOLD CAPITAL M 1d5e0 + case L'𝗡': // MATHEMATICAL SANS BOLD CAPITAL N 1d5e1 + case L'𝗢': // MATHEMATICAL SANS BOLD CAPITAL O 1d5e2 + case L'𝗣': // MATHEMATICAL SANS BOLD CAPITAL P 1d5e3 + case L'𝗤': // MATHEMATICAL SANS BOLD CAPITAL Q 1d5e4 + case L'𝗥': // MATHEMATICAL SANS BOLD CAPITAL R 1d5e5 + case L'𝗦': // MATHEMATICAL SANS BOLD CAPITAL S 1d5e6 + case L'𝗧': // MATHEMATICAL SANS BOLD CAPITAL T 1d5e7 + case L'𝗨': // MATHEMATICAL SANS BOLD CAPITAL U 1d5e8 + case L'𝗩': // MATHEMATICAL SANS BOLD CAPITAL V 1d5e9 + case L'𝗪': // MATHEMATICAL SANS BOLD CAPITAL W 1d5ea + case L'𝗫': // MATHEMATICAL SANS BOLD CAPITAL X 1d5eb + case L'𝗬': // MATHEMATICAL SANS BOLD CAPITAL Y 1d5ec + case L'𝗭': // MATHEMATICAL SANS BOLD CAPITAL Z 1d5ed + case L'𝘈': // MATHEMATICAL SANS ITALIC CAPITAL A 1d608 + case L'𝘉': // MATHEMATICAL SANS ITALIC CAPITAL B 1d609 + case L'𝘊': // MATHEMATICAL SANS ITALIC CAPITAL C 1d60a + case L'𝘋': // MATHEMATICAL SANS ITALIC CAPITAL D 1d60b + case L'𝘌': // MATHEMATICAL SANS ITALIC CAPITAL E 1d60c + case L'𝘍': // MATHEMATICAL SANS ITALIC CAPITAL F 1d60d + case L'𝘎': // MATHEMATICAL SANS ITALIC CAPITAL G 1d60e + case L'𝘏': // MATHEMATICAL SANS ITALIC CAPITAL H 1d60f + case L'𝘐': // MATHEMATICAL SANS ITALIC CAPITAL I 1d610 + case L'𝘑': // MATHEMATICAL SANS ITALIC CAPITAL J 1d611 + case L'𝘒': // MATHEMATICAL SANS ITALIC CAPITAL K 1d612 + case L'𝘓': // MATHEMATICAL SANS ITALIC CAPITAL L 1d613 + case L'𝘔': // MATHEMATICAL SANS ITALIC CAPITAL M 1d614 + case L'𝘕': // MATHEMATICAL SANS ITALIC CAPITAL N 1d615 + case L'𝘖': // MATHEMATICAL SANS ITALIC CAPITAL O 1d616 + case L'𝘗': // MATHEMATICAL SANS ITALIC CAPITAL P 1d617 + case L'𝘘': // MATHEMATICAL SANS ITALIC CAPITAL Q 1d618 + case L'𝘙': // MATHEMATICAL SANS ITALIC CAPITAL R 1d619 + case L'𝘚': // MATHEMATICAL SANS ITALIC CAPITAL S 1d61a + case L'𝘛': // MATHEMATICAL SANS ITALIC CAPITAL T 1d61b + case L'𝘜': // MATHEMATICAL SANS ITALIC CAPITAL U 1d61c + case L'𝘝': // MATHEMATICAL SANS ITALIC CAPITAL V 1d61d + case L'𝘞': // MATHEMATICAL SANS ITALIC CAPITAL W 1d61e + case L'𝘟': // MATHEMATICAL SANS ITALIC CAPITAL X 1d61f + case L'𝘠': // MATHEMATICAL SANS ITALIC CAPITAL Y 1d620 + case L'𝘡': // MATHEMATICAL SANS ITALIC CAPITAL Z 1d621 + case L'𝘼': // MATHEMATICAL SANS BOLD ITALIC CAPITAL A 1d63c + case L'𝘽': // MATHEMATICAL SANS BOLD ITALIC CAPITAL B 1d63d + case L'𝘾': // MATHEMATICAL SANS BOLD ITALIC CAPITAL C 1d63e + case L'𝘿': // MATHEMATICAL SANS BOLD ITALIC CAPITAL D 1d63f + case L'𝙀': // MATHEMATICAL SANS BOLD ITALIC CAPITAL E 1d640 + case L'𝙁': // MATHEMATICAL SANS BOLD ITALIC CAPITAL F 1d641 + case L'𝙂': // MATHEMATICAL SANS BOLD ITALIC CAPITAL G 1d642 + case L'𝙃': // MATHEMATICAL SANS BOLD ITALIC CAPITAL H 1d643 + case L'𝙄': // MATHEMATICAL SANS BOLD ITALIC CAPITAL I 1d644 + case L'𝙅': // MATHEMATICAL SANS BOLD ITALIC CAPITAL J 1d645 + case L'𝙆': // MATHEMATICAL SANS BOLD ITALIC CAPITAL K 1d646 + case L'𝙇': // MATHEMATICAL SANS BOLD ITALIC CAPITAL L 1d647 + case L'𝙈': // MATHEMATICAL SANS BOLD ITALIC CAPITAL M 1d648 + case L'𝙉': // MATHEMATICAL SANS BOLD ITALIC CAPITAL N 1d649 + case L'𝙊': // MATHEMATICAL SANS BOLD ITALIC CAPITAL O 1d64a + case L'𝙋': // MATHEMATICAL SANS BOLD ITALIC CAPITAL P 1d64b + case L'𝙌': // MATHEMATICAL SANS BOLD ITALIC CAPITAL Q 1d64c + case L'𝙍': // MATHEMATICAL SANS BOLD ITALIC CAPITAL R 1d64d + case L'𝙎': // MATHEMATICAL SANS BOLD ITALIC CAPITAL S 1d64e + case L'𝙏': // MATHEMATICAL SANS BOLD ITALIC CAPITAL T 1d64f + case L'𝙐': // MATHEMATICAL SANS BOLD ITALIC CAPITAL U 1d650 + case L'𝙑': // MATHEMATICAL SANS BOLD ITALIC CAPITAL V 1d651 + case L'𝙒': // MATHEMATICAL SANS BOLD ITALIC CAPITAL W 1d652 + case L'𝙓': // MATHEMATICAL SANS BOLD ITALIC CAPITAL X 1d653 + case L'𝙔': // MATHEMATICAL SANS BOLD ITALIC CAPITAL Y 1d654 + case L'𝙕': // MATHEMATICAL SANS BOLD ITALIC CAPITAL Z 1d655 + case L'𝙰': // MATHEMATICAL MONOSPACE CAPITAL A 1d670 + case L'𝙱': // MATHEMATICAL MONOSPACE CAPITAL B 1d671 + case L'𝙲': // MATHEMATICAL MONOSPACE CAPITAL C 1d672 + case L'𝙳': // MATHEMATICAL MONOSPACE CAPITAL D 1d673 + case L'𝙴': // MATHEMATICAL MONOSPACE CAPITAL E 1d674 + case L'𝙵': // MATHEMATICAL MONOSPACE CAPITAL F 1d675 + case L'𝙶': // MATHEMATICAL MONOSPACE CAPITAL G 1d676 + case L'𝙷': // MATHEMATICAL MONOSPACE CAPITAL H 1d677 + case L'𝙸': // MATHEMATICAL MONOSPACE CAPITAL I 1d678 + case L'𝙹': // MATHEMATICAL MONOSPACE CAPITAL J 1d679 + case L'𝙺': // MATHEMATICAL MONOSPACE CAPITAL K 1d67a + case L'𝙻': // MATHEMATICAL MONOSPACE CAPITAL L 1d67b + case L'𝙼': // MATHEMATICAL MONOSPACE CAPITAL M 1d67c + case L'𝙽': // MATHEMATICAL MONOSPACE CAPITAL N 1d67d + case L'𝙾': // MATHEMATICAL MONOSPACE CAPITAL O 1d67e + case L'𝙿': // MATHEMATICAL MONOSPACE CAPITAL P 1d67f + case L'𝚀': // MATHEMATICAL MONOSPACE CAPITAL Q 1d680 + case L'𝚁': // MATHEMATICAL MONOSPACE CAPITAL R 1d681 + case L'𝚂': // MATHEMATICAL MONOSPACE CAPITAL S 1d682 + case L'𝚃': // MATHEMATICAL MONOSPACE CAPITAL T 1d683 + case L'𝚄': // MATHEMATICAL MONOSPACE CAPITAL U 1d684 + case L'𝚅': // MATHEMATICAL MONOSPACE CAPITAL V 1d685 + case L'𝚆': // MATHEMATICAL MONOSPACE CAPITAL W 1d686 + case L'𝚇': // MATHEMATICAL MONOSPACE CAPITAL X 1d687 + case L'𝚈': // MATHEMATICAL MONOSPACE CAPITAL Y 1d688 + case L'𝚉': // MATHEMATICAL MONOSPACE CAPITAL Z 1d689 + case L'𝚨': // MATHEMATICAL BOLD CAPITAL ALPHA 1d6a8 + case L'𝚩': // MATHEMATICAL BOLD CAPITAL BETA 1d6a9 + case L'𝚪': // MATHEMATICAL BOLD CAPITAL GAMMA 1d6aa + case L'𝚫': // MATHEMATICAL BOLD CAPITAL DELTA 1d6ab + case L'𝚬': // MATHEMATICAL BOLD CAPITAL EPSILON 1d6ac + case L'𝚭': // MATHEMATICAL BOLD CAPITAL ZETA 1d6ad + case L'𝚮': // MATHEMATICAL BOLD CAPITAL ETA 1d6ae + case L'𝚯': // MATHEMATICAL BOLD CAPITAL THETA 1d6af + case L'𝚰': // MATHEMATICAL BOLD CAPITAL IOTA 1d6b0 + case L'𝚱': // MATHEMATICAL BOLD CAPITAL KAPPA 1d6b1 + case L'𝚲': // MATHEMATICAL BOLD CAPITAL LAMDA 1d6b2 + case L'𝚳': // MATHEMATICAL BOLD CAPITAL MU 1d6b3 + case L'𝚴': // MATHEMATICAL BOLD CAPITAL NU 1d6b4 + case L'𝚵': // MATHEMATICAL BOLD CAPITAL XI 1d6b5 + case L'𝚶': // MATHEMATICAL BOLD CAPITAL OMICRON 1d6b6 + case L'𝚷': // MATHEMATICAL BOLD CAPITAL PI 1d6b7 + case L'𝚸': // MATHEMATICAL BOLD CAPITAL RHO 1d6b8 + case L'𝚹': // MATHEMATICAL BOLD CAPITAL THETA SYMBOL 1d6b9 + case L'𝚺': // MATHEMATICAL BOLD CAPITAL SIGMA 1d6ba + case L'𝚻': // MATHEMATICAL BOLD CAPITAL TAU 1d6bb + case L'𝚼': // MATHEMATICAL BOLD CAPITAL UPSILON 1d6bc + case L'𝚽': // MATHEMATICAL BOLD CAPITAL PHI 1d6bd + case L'𝚾': // MATHEMATICAL BOLD CAPITAL CHI 1d6be + case L'𝚿': // MATHEMATICAL BOLD CAPITAL PSI 1d6bf + case L'𝛀': // MATHEMATICAL BOLD CAPITAL OMEGA 1d6c0 + case L'𝛢': // MATHEMATICAL ITALIC CAPITAL ALPHA 1d6e2 + case L'𝛣': // MATHEMATICAL ITALIC CAPITAL BETA 1d6e3 + case L'𝛤': // MATHEMATICAL ITALIC CAPITAL GAMMA 1d6e4 + case L'𝛥': // MATHEMATICAL ITALIC CAPITAL DELTA 1d6e5 + case L'𝛦': // MATHEMATICAL ITALIC CAPITAL EPSILON 1d6e6 + case L'𝛧': // MATHEMATICAL ITALIC CAPITAL ZETA 1d6e7 + case L'𝛨': // MATHEMATICAL ITALIC CAPITAL ETA 1d6e8 + case L'𝛩': // MATHEMATICAL ITALIC CAPITAL THETA 1d6e9 + case L'𝛪': // MATHEMATICAL ITALIC CAPITAL IOTA 1d6ea + case L'𝛫': // MATHEMATICAL ITALIC CAPITAL KAPPA 1d6eb + case L'𝛬': // MATHEMATICAL ITALIC CAPITAL LAMDA 1d6ec + case L'𝛭': // MATHEMATICAL ITALIC CAPITAL MU 1d6ed + case L'𝛮': // MATHEMATICAL ITALIC CAPITAL NU 1d6ee + case L'𝛯': // MATHEMATICAL ITALIC CAPITAL XI 1d6ef + case L'𝛰': // MATHEMATICAL ITALIC CAPITAL OMICRON 1d6f0 + case L'𝛱': // MATHEMATICAL ITALIC CAPITAL PI 1d6f1 + case L'𝛲': // MATHEMATICAL ITALIC CAPITAL RHO 1d6f2 + case L'𝛳': // MATHEMATICAL ITALIC CAPITAL THETA SYMBOL 1d6f3 + case L'𝛴': // MATHEMATICAL ITALIC CAPITAL SIGMA 1d6f4 + case L'𝛵': // MATHEMATICAL ITALIC CAPITAL TAU 1d6f5 + case L'𝛶': // MATHEMATICAL ITALIC CAPITAL UPSILON 1d6f6 + case L'𝛷': // MATHEMATICAL ITALIC CAPITAL PHI 1d6f7 + case L'𝛸': // MATHEMATICAL ITALIC CAPITAL CHI 1d6f8 + case L'𝛹': // MATHEMATICAL ITALIC CAPITAL PSI 1d6f9 + case L'𝛺': // MATHEMATICAL ITALIC CAPITAL OMEGA 1d6fa + case L'𝜜': // MATHEMATICAL BOLD ITALIC CAPITAL ALPHA 1d71c + case L'𝜝': // MATHEMATICAL BOLD ITALIC CAPITAL BETA 1d71d + case L'𝜞': // MATHEMATICAL BOLD ITALIC CAPITAL GAMMA 1d71e + case L'𝜟': // MATHEMATICAL BOLD ITALIC CAPITAL DELTA 1d71f + case L'𝜠': // MATHEMATICAL BOLD ITALIC CAPITAL EPSILON 1d720 + case L'𝜡': // MATHEMATICAL BOLD ITALIC CAPITAL ZETA 1d721 + case L'𝜢': // MATHEMATICAL BOLD ITALIC CAPITAL ETA 1d722 + case L'𝜣': // MATHEMATICAL BOLD ITALIC CAPITAL THETA 1d723 + case L'𝜤': // MATHEMATICAL BOLD ITALIC CAPITAL IOTA 1d724 + case L'𝜥': // MATHEMATICAL BOLD ITALIC CAPITAL KAPPA 1d725 + case L'𝜦': // MATHEMATICAL BOLD ITALIC CAPITAL LAMDA 1d726 + case L'𝜧': // MATHEMATICAL BOLD ITALIC CAPITAL MU 1d727 + case L'𝜨': // MATHEMATICAL BOLD ITALIC CAPITAL NU 1d728 + case L'𝜩': // MATHEMATICAL BOLD ITALIC CAPITAL XI 1d729 + case L'𝜪': // MATHEMATICAL BOLD ITALIC CAPITAL OMICRON 1d72a + case L'𝜫': // MATHEMATICAL BOLD ITALIC CAPITAL PI 1d72b + case L'𝜬': // MATHEMATICAL BOLD ITALIC CAPITAL RHO 1d72c + case L'𝜭': // MATHEMATICAL BOLD ITALIC CAPITAL THETA SYMBOL 1d72d + case L'𝜮': // MATHEMATICAL BOLD ITALIC CAPITAL SIGMA 1d72e + case L'𝜯': // MATHEMATICAL BOLD ITALIC CAPITAL TAU 1d72f + case L'𝜰': // MATHEMATICAL BOLD ITALIC CAPITAL UPSILON 1d730 + case L'𝜱': // MATHEMATICAL BOLD ITALIC CAPITAL PHI 1d731 + case L'𝜲': // MATHEMATICAL BOLD ITALIC CAPITAL CHI 1d732 + case L'𝜳': // MATHEMATICAL BOLD ITALIC CAPITAL PSI 1d733 + case L'𝜴': // MATHEMATICAL BOLD ITALIC CAPITAL OMEGA 1d734 + case L'𝝖': // MATHEMATICAL SANS BOLD CAPITAL ALPHA 1d756 + case L'𝝗': // MATHEMATICAL SANS BOLD CAPITAL BETA 1d757 + case L'𝝘': // MATHEMATICAL SANS BOLD CAPITAL GAMMA 1d758 + case L'𝝙': // MATHEMATICAL SANS BOLD CAPITAL DELTA 1d759 + case L'𝝚': // MATHEMATICAL SANS BOLD CAPITAL EPSILON 1d75a + case L'𝝛': // MATHEMATICAL SANS BOLD CAPITAL ZETA 1d75b + case L'𝝜': // MATHEMATICAL SANS BOLD CAPITAL ETA 1d75c + case L'𝝝': // MATHEMATICAL SANS BOLD CAPITAL THETA 1d75d + case L'𝝞': // MATHEMATICAL SANS BOLD CAPITAL IOTA 1d75e + case L'𝝟': // MATHEMATICAL SANS BOLD CAPITAL KAPPA 1d75f + case L'𝝠': // MATHEMATICAL SANS BOLD CAPITAL LAMDA 1d760 + case L'𝝡': // MATHEMATICAL SANS BOLD CAPITAL MU 1d761 + case L'𝝢': // MATHEMATICAL SANS BOLD CAPITAL NU 1d762 + case L'𝝣': // MATHEMATICAL SANS BOLD CAPITAL XI 1d763 + case L'𝝤': // MATHEMATICAL SANS BOLD CAPITAL OMICRON 1d764 + case L'𝝥': // MATHEMATICAL SANS BOLD CAPITAL PI 1d765 + case L'𝝦': // MATHEMATICAL SANS BOLD CAPITAL RHO 1d766 + case L'𝝧': // MATHEMATICAL SANS BOLD CAPITAL THETA SYMBOL 1d767 + case L'𝝨': // MATHEMATICAL SANS BOLD CAPITAL SIGMA 1d768 + case L'𝝩': // MATHEMATICAL SANS BOLD CAPITAL TAU 1d769 + case L'𝝪': // MATHEMATICAL SANS BOLD CAPITAL UPSILON 1d76a + case L'𝝫': // MATHEMATICAL SANS BOLD CAPITAL PHI 1d76b + case L'𝝬': // MATHEMATICAL SANS BOLD CAPITAL CHI 1d76c + case L'𝝭': // MATHEMATICAL SANS BOLD CAPITAL PSI 1d76d + case L'𝝮': // MATHEMATICAL SANS BOLD CAPITAL OMEGA 1d76e + case L'𝞐': // MATHEMATICAL SANS BOLD ITALIC CAPITAL ALPHA 1d790 + case L'𝞑': // MATHEMATICAL SANS BOLD ITALIC CAPITAL BETA 1d791 + case L'𝞒': // MATHEMATICAL SANS BOLD ITALIC CAPITAL GAMMA 1d792 + case L'𝞓': // MATHEMATICAL SANS BOLD ITALIC CAPITAL DELTA 1d793 + case L'𝞔': // MATHEMATICAL SANS BOLD ITALIC CAPITAL EPSILON 1d794 + case L'𝞕': // MATHEMATICAL SANS BOLD ITALIC CAPITAL ZETA 1d795 + case L'𝞖': // MATHEMATICAL SANS BOLD ITALIC CAPITAL ETA 1d796 + case L'𝞗': // MATHEMATICAL SANS BOLD ITALIC CAPITAL THETA 1d797 + case L'𝞘': // MATHEMATICAL SANS BOLD ITALIC CAPITAL IOTA 1d798 + case L'𝞙': // MATHEMATICAL SANS BOLD ITALIC CAPITAL KAPPA 1d799 + case L'𝞚': // MATHEMATICAL SANS BOLD ITALIC CAPITAL LAMDA 1d79a + case L'𝞛': // MATHEMATICAL SANS BOLD ITALIC CAPITAL MU 1d79b + case L'𝞜': // MATHEMATICAL SANS BOLD ITALIC CAPITAL NU 1d79c + case L'𝞝': // MATHEMATICAL SANS BOLD ITALIC CAPITAL XI 1d79d + case L'𝞞': // MATHEMATICAL SANS BOLD ITALIC CAPITAL OMICRON 1d79e + case L'𝞟': // MATHEMATICAL SANS BOLD ITALIC CAPITAL PI 1d79f + case L'𝞠': // MATHEMATICAL SANS BOLD ITALIC CAPITAL RHO 1d7a0 + case L'𝞡': // MATHEMATICAL SANS BOLD ITALIC CAPITAL THETA SYMBOL 1d7a1 + case L'𝞢': // MATHEMATICAL SANS BOLD ITALIC CAPITAL SIGMA 1d7a2 + case L'𝞣': // MATHEMATICAL SANS BOLD ITALIC CAPITAL TAU 1d7a3 + case L'𝞤': // MATHEMATICAL SANS BOLD ITALIC CAPITAL UPSILON 1d7a4 + case L'𝞥': // MATHEMATICAL SANS BOLD ITALIC CAPITAL PHI 1d7a5 + case L'𝞦': // MATHEMATICAL SANS BOLD ITALIC CAPITAL CHI 1d7a6 + case L'𝞧': // MATHEMATICAL SANS BOLD ITALIC CAPITAL PSI 1d7a7 + case L'𝞨': // MATHEMATICAL SANS BOLD ITALIC CAPITAL OMEGA 1d7a8 + case L'𝟊': // MATHEMATICAL BOLD CAPITAL DIGAMMA 1d7ca + case L'𞤀': // ADLAM CAPITAL ALIF 1e900 + case L'𞤁': // ADLAM CAPITAL DAALI 1e901 + case L'𞤂': // ADLAM CAPITAL LAAM 1e902 + case L'𞤃': // ADLAM CAPITAL MIIM 1e903 + case L'𞤄': // ADLAM CAPITAL BA 1e904 + case L'𞤅': // ADLAM CAPITAL SINNYIIYHE 1e905 + case L'𞤆': // ADLAM CAPITAL PE 1e906 + case L'𞤇': // ADLAM CAPITAL BHE 1e907 + case L'𞤈': // ADLAM CAPITAL RA 1e908 + case L'𞤉': // ADLAM CAPITAL E 1e909 + case L'𞤊': // ADLAM CAPITAL FA 1e90a + case L'𞤋': // ADLAM CAPITAL I 1e90b + case L'𞤌': // ADLAM CAPITAL O 1e90c + case L'𞤍': // ADLAM CAPITAL DHA 1e90d + case L'𞤎': // ADLAM CAPITAL YHE 1e90e + case L'𞤏': // ADLAM CAPITAL WAW 1e90f + case L'𞤐': // ADLAM CAPITAL NUN 1e910 + case L'𞤑': // ADLAM CAPITAL KAF 1e911 + case L'𞤒': // ADLAM CAPITAL YA 1e912 + case L'𞤓': // ADLAM CAPITAL U 1e913 + case L'𞤔': // ADLAM CAPITAL JIIM 1e914 + case L'𞤕': // ADLAM CAPITAL CHI 1e915 + case L'𞤖': // ADLAM CAPITAL HA 1e916 + case L'𞤗': // ADLAM CAPITAL QAAF 1e917 + case L'𞤘': // ADLAM CAPITAL GA 1e918 + case L'𞤙': // ADLAM CAPITAL NYA 1e919 + case L'𞤚': // ADLAM CAPITAL TU 1e91a + case L'𞤛': // ADLAM CAPITAL NHA 1e91b + case L'𞤜': // ADLAM CAPITAL VA 1e91c + case L'𞤝': // ADLAM CAPITAL KHA 1e91d + case L'𞤞': // ADLAM CAPITAL GBE 1e91e + case L'𞤟': // ADLAM CAPITAL ZAL 1e91f + case L'𞤠': // ADLAM CAPITAL KPO 1e920 + case L'𞤡': // ADLAM CAPITAL SHA 1e921 + return 1; + default: + return 0; + } } diff --git a/libc/str/iswxdigit.c b/libc/str/iswxdigit.c index 765b1be3f..5abc42c37 100644 --- a/libc/str/iswxdigit.c +++ b/libc/str/iswxdigit.c @@ -18,6 +18,10 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" -int iswxdigit(wint_t wc) { - return isxdigit(wc); +/** + * Returns nonzero if c is ascii hex digit. + */ +int iswxdigit(wint_t c) { + return ('0' <= c && c <= '9') || ('A' <= c && c <= 'F') || + ('a' <= c && c <= 'f'); } diff --git a/libc/str/isxdigit.c b/libc/str/isxdigit.c index 64f0c23ad..2aae1c040 100644 --- a/libc/str/isxdigit.c +++ b/libc/str/isxdigit.c @@ -16,6 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/limits.h" #include "libc/str/str.h" /** diff --git a/libc/str/iszipcdir64.c b/libc/str/iszipcdir64.c index ca18dcee2..044442336 100644 --- a/libc/str/iszipcdir64.c +++ b/libc/str/iszipcdir64.c @@ -23,11 +23,16 @@ * Returns true if zip64 end of central directory header seems legit. */ bool IsZipCdir64(const uint8_t *p, size_t n, size_t i) { - if (i > n || n - i < kZipCdir64HdrMinSize) return false; + if (i + kZipCdir64HdrMinSize > n) return false; if (READ32LE(p + i) != kZipCdir64HdrMagic) return false; - if (i + ZIP_CDIR64_HDRSIZE(p + i) > n) return false; - if (ZIP_CDIR64_DISK(p + i) != ZIP_CDIR64_STARTINGDISK(p + i)) return false; - if (ZIP_CDIR64_RECORDSONDISK(p + i) != ZIP_CDIR64_RECORDS(p + i)) { + if (i + ZIP_CDIR64_HDRSIZE(p + i) + kZipCdir64LocatorSize > n) { + return false; + } + if (ZIP_LOCATE64_MAGIC(p + i + ZIP_CDIR64_HDRSIZE(p + i)) != + kZipCdir64LocatorMagic) { + return false; + } + if (ZIP_LOCATE64_OFFSET(p + i + ZIP_CDIR64_HDRSIZE(p + i)) != i) { return false; } if (ZIP_CDIR64_RECORDS(p + i) * kZipCfileHdrMinSize > diff --git a/libc/str/memccpy.c b/libc/str/memccpy.c index 708a85fe5..f809cd0ff 100644 --- a/libc/str/memccpy.c +++ b/libc/str/memccpy.c @@ -62,7 +62,7 @@ void *memccpy(void *dst, const void *src, int c, size_t n) { d = dst; s = src; c &= 255; - v = 0x0101010101010101 * c; + v = 0x0101010101010101ul * c; for (; (uintptr_t)(s + i) & 7; ++i) { if (i == n) return NULL; if ((d[i] = s[i]) == c) return d + i + 1; diff --git a/libc/str/memchr.c b/libc/str/memchr.c index 1c917fb70..c44f48380 100644 --- a/libc/str/memchr.c +++ b/libc/str/memchr.c @@ -29,19 +29,22 @@ */ void *memchr(const void *m, int c, size_t n) { uint64_t v, w; - const unsigned char *p, *pe; + const char *p, *pe; c &= 255; - v = 0x0101010101010101 * c; - for (p = (const unsigned char *)m, pe = p + n; p + 8 <= pe; p += 8) { - w = (uint64_t)p[7] << 070 | (uint64_t)p[6] << 060 | (uint64_t)p[5] << 050 | - (uint64_t)p[4] << 040 | (uint64_t)p[3] << 030 | (uint64_t)p[2] << 020 | - (uint64_t)p[1] << 010 | (uint64_t)p[0] << 000; + v = 0x0101010101010101ul * c; + for (p = m, pe = p + n; p + 8 <= pe; p += 8) { + w = (uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 | + (uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 | + (uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 | + (uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000; if ((w = ~(w ^ v) & ((w ^ v) - 0x0101010101010101) & 0x8080808080808080)) { return p + ((unsigned)__builtin_ctzll(w) >> 3); } } for (; p < pe; ++p) { - if (*p == c) return p; + if ((*p & 255) == c) { + return p; + } } return NULL; } diff --git a/libc/str/memmem.c b/libc/str/memmem.c index fd307ab21..071d5ea3f 100644 --- a/libc/str/memmem.c +++ b/libc/str/memmem.c @@ -30,12 +30,22 @@ void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { size_t i, j; + const char *p; if (!needlelen) return haystack; - for (i = 0; i < haystacklen; ++i) { - for (j = 0;; ++j) { - if (j == needlelen) return (/*unconst*/ char *)haystack + i; - if (i + j == haystacklen) break; - if (((char *)needle)[j] != ((char *)haystack)[i + j]) break; + if (needlelen <= haystacklen) { + p = memchr(haystack, *(const char *)needle, haystacklen); + if (needlelen == 1) return p; + if (p) { + haystacklen -= p - (const char *)haystack; + haystack = p; + } + /* TODO: make not quadratic */ + for (i = 0; i < haystacklen; ++i) { + for (j = 0;; ++j) { + if (j == needlelen) return (/*unconst*/ char *)haystack + i; + if (i + j == haystacklen) break; + if (((char *)needle)[j] != ((char *)haystack)[i + j]) break; + } } } return NULL; diff --git a/libc/str/memset-pure.c b/libc/str/memset-pure.c index 951629f7e..cf1fe8e7d 100644 --- a/libc/str/memset-pure.c +++ b/libc/str/memset-pure.c @@ -31,7 +31,7 @@ void *memset_pure(void *p, int c, size_t n) { char *b; uint64_t x; b = p; - x = 0x0101010101010101 * (c & 0xff); + x = 0x0101010101010101ul * (c & 0xff); switch (n) { case 0: return p; diff --git a/libc/str/rawmemchr.c b/libc/str/rawmemchr.c index 4ffd31621..8538fd611 100644 --- a/libc/str/rawmemchr.c +++ b/libc/str/rawmemchr.c @@ -37,7 +37,7 @@ void *rawmemchr(const void *m, int c) { const unsigned char *s; s = m; c &= 255; - v = 0x0101010101010101 * c; + v = 0x0101010101010101ul * c; for (; (uintptr_t)s & 7; ++s) { if (*s == c) return s; } diff --git a/libc/str/str.h b/libc/str/str.h index a5fc31167..a245c330e 100644 --- a/libc/str/str.h +++ b/libc/str/str.h @@ -165,6 +165,7 @@ char *strncpy(char *, const char *, size_t) memcpyesque; char *strtok(char *, const char *) paramsnonnull((2)) libcesque; char *strtok_r(char *, const char *, char **) paramsnonnull((2, 3)); uint16_t *strcpyzbw(uint16_t *, const char *) memcpyesque; +wchar_t *wcstok(wchar_t *, const wchar_t *, wchar_t **) paramsnonnull((2, 3)); char *wstrtrunc(uint16_t *) memcpyesque; char *wstrntrunc(uint16_t *, size_t) memcpyesque; bool startswith(const char *, const char *) strlenesque; diff --git a/libc/str/str.mk b/libc/str/str.mk index 7dfdd48cc..83eaeb721 100644 --- a/libc/str/str.mk +++ b/libc/str/str.mk @@ -62,6 +62,12 @@ o/$(MODE)/libc/str/getziplfileuncompressedsize.o: \ OVERRIDE_CFLAGS += \ -Os +o/$(MODE)/libc/str/iswpunct.o \ +o/$(MODE)/libc/str/iswupper.o \ +o/$(MODE)/libc/str/iswlower.o: \ + OVERRIDE_CFLAGS += \ + -fno-jump-tables + LIBC_STR_LIBS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x))) LIBC_STR_SRCS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x)_SRCS)) LIBC_STR_HDRS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/str/strchr.c b/libc/str/strchr.c index 885a4ad8e..77ee980ab 100644 --- a/libc/str/strchr.c +++ b/libc/str/strchr.c @@ -17,14 +17,16 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/bits/bits.h" #include "libc/str/str.h" -noasan static const char *strchr_x64(const char *p, uint64_t c) { +static noasan inline const char *strchr_x64(const char *p, uint64_t c) { unsigned a, b; uint64_t w, x, y; for (c *= 0x0101010101010101;; p += 8) { - w = READ64LE(p); + w = (uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 | + (uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 | + (uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 | + (uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000; if ((x = ~(w ^ c) & ((w ^ c) - 0x0101010101010101) & 0x8080808080808080) | (y = ~w & (w - 0x0101010101010101) & 0x8080808080808080)) { if (x) { @@ -57,8 +59,8 @@ noasan static const char *strchr_x64(const char *p, uint64_t c) { */ char *strchr(const char *s, int c) { char *r; - for (c &= 0xff; (uintptr_t)s & 7; ++s) { - if ((*s & 0xff) == c) return s; + for (c &= 255; (uintptr_t)s & 7; ++s) { + if ((*s & 255) == c) return s; if (!*s) return NULL; } r = strchr_x64(s, c); diff --git a/libc/str/strncasecmp16.c b/libc/str/strncasecmp16.c index 842a2eb19..0754228e3 100644 --- a/libc/str/strncasecmp16.c +++ b/libc/str/strncasecmp16.c @@ -30,6 +30,6 @@ int strncasecmp16(const char16_t *a, const char16_t *b, size_t n) { int x, y; size_t i = 0; if (!n-- || a == b) return 0; - while ((x = tolower(a[i])) == (y = tolower(b[i])) && b[i] && i < n) ++i; + while ((x = towlower(a[i])) == (y = towlower(b[i])) && b[i] && i < n) ++i; return x - y; } diff --git a/libc/str/strrchr.c b/libc/str/strrchr.c index e5596d58a..a375aa1ea 100644 --- a/libc/str/strrchr.c +++ b/libc/str/strrchr.c @@ -26,4 +26,6 @@ * @return address of last c in s, or NULL if not found * @asyncsignalsafe */ -char *(strrchr)(const char *s, int c) { return memrchr(s, c, strlen(s)); } +char *strrchr(const char *s, int c) { + return memrchr(s, c, strlen(s)); +} diff --git a/libc/str/strstr.c b/libc/str/strstr.c index af79548c3..76fd72315 100644 --- a/libc/str/strstr.c +++ b/libc/str/strstr.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/bits/safemacros.internal.h" #include "libc/str/str.h" /** @@ -30,8 +29,13 @@ */ char *strstr(const char *haystack, const char *needle) { size_t i; - if (!*needle) return haystack; - haystack = firstnonnull(strchr(haystack, *needle), haystack); + const char *p; + if (!needle[0]) return haystack; + if (haystack == needle) return haystack; + p = strchr(haystack, needle[0]); + if (!needle[1]) return p; + if (p) haystack = p; + /* TODO: make not quadratic */ for (;;) { for (i = 0;;) { if (!needle[i]) return (/*unconst*/ char *)haystack; diff --git a/libc/str/strxfrm.c b/libc/str/strxfrm.c index 57cda307e..1f71ac3ff 100644 --- a/libc/str/strxfrm.c +++ b/libc/str/strxfrm.c @@ -25,7 +25,6 @@ │ OTHER DEALINGS IN THE SOFTWARE. │ │ │ ╚─────────────────────────────────────────────────────────────────────────────*/ - #include "libc/assert.h" #include "libc/str/str.h" diff --git a/libc/str/tolower.c b/libc/str/tolower.c index a3ad8fc48..7b0d7b8ae 100644 --- a/libc/str/tolower.c +++ b/libc/str/tolower.c @@ -19,7 +19,7 @@ #include "libc/str/str.h" /** - * Converts character to lower case. + * Converts character to ascii lower case. */ int tolower(int c) { return 'A' <= c && c <= 'Z' ? c + ('a' - 'A') : c; diff --git a/libc/str/towlower.c b/libc/str/towlower.c index 9c1c37816..848f3b4c7 100644 --- a/libc/str/towlower.c +++ b/libc/str/towlower.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,11 +16,1649 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/macros.internal.h" #include "libc/str/str.h" +#include "libc/dce.h" + +static const char16_t kUpperToLower[][2] = { + {u'À', u'à'}, // 00c0 → 00e0 + {u'Á', u'á'}, // 00c1 → 00e1 + {u'Â', u'â'}, // 00c2 → 00e2 + {u'Ã', u'ã'}, // 00c3 → 00e3 + {u'Ä', u'ä'}, // 00c4 → 00e4 + {u'Å', u'å'}, // 00c5 → 00e5 + {u'Æ', u'æ'}, // 00c6 → 00e6 + {u'Ç', u'ç'}, // 00c7 → 00e7 + {u'È', u'è'}, // 00c8 → 00e8 + {u'É', u'é'}, // 00c9 → 00e9 + {u'Ê', u'ê'}, // 00ca → 00ea + {u'Ë', u'ë'}, // 00cb → 00eb + {u'Ì', u'ì'}, // 00cc → 00ec + {u'Í', u'í'}, // 00cd → 00ed + {u'Î', u'î'}, // 00ce → 00ee + {u'Ï', u'ï'}, // 00cf → 00ef + {u'Ð', u'ð'}, // 00d0 → 00f0 + {u'Ñ', u'ñ'}, // 00d1 → 00f1 + {u'Ò', u'ò'}, // 00d2 → 00f2 + {u'Ó', u'ó'}, // 00d3 → 00f3 + {u'Ô', u'ô'}, // 00d4 → 00f4 + {u'Õ', u'õ'}, // 00d5 → 00f5 + {u'Ö', u'ö'}, // 00d6 → 00f6 + {u'Ø', u'ø'}, // 00d8 → 00f8 + {u'Ù', u'ù'}, // 00d9 → 00f9 + {u'Ú', u'ú'}, // 00da → 00fa + {u'Û', u'û'}, // 00db → 00fb + {u'Ü', u'ü'}, // 00dc → 00fc + {u'Ý', u'ý'}, // 00dd → 00fd + {u'Þ', u'þ'}, // 00de → 00fe +#ifndef TINY + {u'Ā', u'ā'}, // 0100 → 0101 + {u'Ă', u'ă'}, // 0102 → 0103 + {u'Ą', u'ą'}, // 0104 → 0105 + {u'Ć', u'ć'}, // 0106 → 0107 + {u'Ĉ', u'ĉ'}, // 0108 → 0109 + {u'Ċ', u'ċ'}, // 010a → 010b + {u'Č', u'č'}, // 010c → 010d + {u'Ď', u'ď'}, // 010e → 010f + {u'Đ', u'đ'}, // 0110 → 0111 + {u'Ē', u'ē'}, // 0112 → 0113 + {u'Ĕ', u'ĕ'}, // 0114 → 0115 + {u'Ė', u'ė'}, // 0116 → 0117 + {u'Ę', u'ę'}, // 0118 → 0119 + {u'Ě', u'ě'}, // 011a → 011b + {u'Ĝ', u'ĝ'}, // 011c → 011d + {u'Ğ', u'ğ'}, // 011e → 011f + {u'Ġ', u'ġ'}, // 0120 → 0121 + {u'Ģ', u'ģ'}, // 0122 → 0123 + {u'Ĥ', u'ĥ'}, // 0124 → 0125 + {u'Ħ', u'ħ'}, // 0126 → 0127 + {u'Ĩ', u'ĩ'}, // 0128 → 0129 + {u'Ī', u'ī'}, // 012a → 012b + {u'Ĭ', u'ĭ'}, // 012c → 012d + {u'Į', u'į'}, // 012e → 012f + {u'IJ', u'ij'}, // 0132 → 0133 + {u'Ĵ', u'ĵ'}, // 0134 → 0135 + {u'Ķ', u'ķ'}, // 0136 → 0137 + {u'Ĺ', u'ĺ'}, // 0139 → 013a + {u'Ļ', u'ļ'}, // 013b → 013c + {u'Ľ', u'ľ'}, // 013d → 013e + {u'Ŀ', u'ŀ'}, // 013f → 0140 + {u'Ł', u'ł'}, // 0141 → 0142 + {u'Ń', u'ń'}, // 0143 → 0144 + {u'Ņ', u'ņ'}, // 0145 → 0146 + {u'Ň', u'ň'}, // 0147 → 0148 + {u'Ŋ', u'ŋ'}, // 014a → 014b + {u'Ō', u'ō'}, // 014c → 014d + {u'Ŏ', u'ŏ'}, // 014e → 014f + {u'Ő', u'ő'}, // 0150 → 0151 + {u'Œ', u'œ'}, // 0152 → 0153 + {u'Ŕ', u'ŕ'}, // 0154 → 0155 + {u'Ŗ', u'ŗ'}, // 0156 → 0157 + {u'Ř', u'ř'}, // 0158 → 0159 + {u'Ś', u'ś'}, // 015a → 015b + {u'Ŝ', u'ŝ'}, // 015c → 015d + {u'Ş', u'ş'}, // 015e → 015f + {u'Š', u'š'}, // 0160 → 0161 + {u'Ţ', u'ţ'}, // 0162 → 0163 + {u'Ť', u'ť'}, // 0164 → 0165 + {u'Ŧ', u'ŧ'}, // 0166 → 0167 + {u'Ũ', u'ũ'}, // 0168 → 0169 + {u'Ū', u'ū'}, // 016a → 016b + {u'Ŭ', u'ŭ'}, // 016c → 016d + {u'Ů', u'ů'}, // 016e → 016f + {u'Ű', u'ű'}, // 0170 → 0171 + {u'Ų', u'ų'}, // 0172 → 0173 + {u'Ŵ', u'ŵ'}, // 0174 → 0175 + {u'Ŷ', u'ŷ'}, // 0176 → 0177 + {u'Ÿ', u'ÿ'}, // 0178 → 00ff + {u'Ź', u'ź'}, // 0179 → 017a + {u'Ż', u'ż'}, // 017b → 017c + {u'Ž', u'ž'}, // 017d → 017e + {u'Ɓ', u'ɓ'}, // 0181 → 0253 + {u'Ƃ', u'ƃ'}, // 0182 → 0183 + {u'Ƅ', u'ƅ'}, // 0184 → 0185 + {u'Ɔ', u'ɔ'}, // 0186 → 0254 + {u'Ƈ', u'ƈ'}, // 0187 → 0188 + {u'Ɗ', u'ɗ'}, // 018a → 0257 + {u'Ƌ', u'ƌ'}, // 018b → 018c + {u'Ǝ', u'ɘ'}, // 018e → 0258 + {u'Ə', u'ə'}, // 018f → 0259 + {u'Ɛ', u'ɛ'}, // 0190 → 025b + {u'Ƒ', u'ƒ'}, // 0191 → 0192 + {u'Ɠ', u'ɠ'}, // 0193 → 0260 + {u'Ɣ', u'ɣ'}, // 0194 → 0263 + {u'Ɩ', u'ɩ'}, // 0196 → 0269 + {u'Ɨ', u'ɨ'}, // 0197 → 0268 + {u'Ƙ', u'ƙ'}, // 0198 → 0199 + {u'Ɯ', u'ɯ'}, // 019c → 026f + {u'Ɲ', u'ɲ'}, // 019d → 0272 + {u'Ơ', u'ơ'}, // 01a0 → 01a1 + {u'Ƣ', u'ƣ'}, // 01a2 → 01a3 + {u'Ƥ', u'ƥ'}, // 01a4 → 01a5 + {u'Ƨ', u'ƨ'}, // 01a7 → 01a8 + {u'Ʃ', u'ʃ'}, // 01a9 → 0283 + {u'Ƭ', u'ƭ'}, // 01ac → 01ad + {u'Ʈ', u'ʈ'}, // 01ae → 0288 + {u'Ư', u'ư'}, // 01af → 01b0 + {u'Ʊ', u'ʊ'}, // 01b1 → 028a + {u'Ʋ', u'ʋ'}, // 01b2 → 028b + {u'Ƴ', u'ƴ'}, // 01b3 → 01b4 + {u'Ƶ', u'ƶ'}, // 01b5 → 01b6 + {u'Ʒ', u'ƹ'}, // 01b7 → 01b9 + {u'Ƹ', u'ƹ'}, // 01b8 → 01b9 + {u'Ƽ', u'ƽ'}, // 01bc → 01bd + {u'DŽ', u'dž'}, // 01c4 → 01c6 + {u'LJ', u'lj'}, // 01c7 → 01c9 + {u'NJ', u'nj'}, // 01ca → 01cc + {u'Ǎ', u'ǎ'}, // 01cd → 01ce + {u'Ǐ', u'ǐ'}, // 01cf → 01d0 + {u'Ǒ', u'ǒ'}, // 01d1 → 01d2 + {u'Ǔ', u'ǔ'}, // 01d3 → 01d4 + {u'Ǖ', u'ǖ'}, // 01d5 → 01d6 + {u'Ǘ', u'ǘ'}, // 01d7 → 01d8 + {u'Ǚ', u'ǚ'}, // 01d9 → 01da + {u'Ǜ', u'ǜ'}, // 01db → 01dc + {u'Ǟ', u'ǟ'}, // 01de → 01df + {u'Ǡ', u'ǡ'}, // 01e0 → 01e1 + {u'Ǣ', u'ǣ'}, // 01e2 → 01e3 + {u'Ǥ', u'ǥ'}, // 01e4 → 01e5 + {u'Ǧ', u'ǧ'}, // 01e6 → 01e7 + {u'Ǩ', u'ǩ'}, // 01e8 → 01e9 + {u'Ǫ', u'ǫ'}, // 01ea → 01eb + {u'Ǭ', u'ǭ'}, // 01ec → 01ed + {u'Ǯ', u'ǯ'}, // 01ee → 01ef + {u'DZ', u'dž'}, // 01f1 → 01c6 + {u'Ǵ', u'ǵ'}, // 01f4 → 01f5 + {u'Ǹ', u'ǹ'}, // 01f8 → 01f9 + {u'Ǻ', u'ǻ'}, // 01fa → 01fb + {u'Ǽ', u'ǽ'}, // 01fc → 01fd + {u'Ǿ', u'ǿ'}, // 01fe → 01ff + {u'Ȁ', u'ȁ'}, // 0200 → 0201 + {u'Ȃ', u'ȃ'}, // 0202 → 0203 + {u'Ȅ', u'ȅ'}, // 0204 → 0205 + {u'Ȇ', u'ȇ'}, // 0206 → 0207 + {u'Ȉ', u'ȉ'}, // 0208 → 0209 + {u'Ȋ', u'ȋ'}, // 020a → 020b + {u'Ȍ', u'ȍ'}, // 020c → 020d + {u'Ȏ', u'ȏ'}, // 020e → 020f + {u'Ȑ', u'ȑ'}, // 0210 → 0211 + {u'Ȓ', u'ȓ'}, // 0212 → 0213 + {u'Ȕ', u'ȕ'}, // 0214 → 0215 + {u'Ȗ', u'ȗ'}, // 0216 → 0217 + {u'Ș', u'ș'}, // 0218 → 0219 + {u'Ț', u'ț'}, // 021a → 021b + {u'Ȝ', u'ȝ'}, // 021c → 021d + {u'Ȟ', u'ȟ'}, // 021e → 021f + {u'Ƞ', u'ƞ'}, // 0220 → 019e + {u'Ȣ', u'ȣ'}, // 0222 → 0223 + {u'Ȥ', u'ȥ'}, // 0224 → 0225 + {u'Ȧ', u'ǡ'}, // 0226 → 01e1 + {u'Ȩ', u'ȩ'}, // 0228 → 0229 + {u'Ȫ', u'ȫ'}, // 022a → 022b + {u'Ȭ', u'ȭ'}, // 022c → 022d + {u'Ȯ', u'ȯ'}, // 022e → 022f + {u'Ȱ', u'ȱ'}, // 0230 → 0231 + {u'Ȳ', u'ȳ'}, // 0232 → 0233 + {u'Ⱥ', u'ⱥ'}, // 023a → 2c65 + {u'Ȼ', u'ȼ'}, // 023b → 023c + {u'Ƚ', u'ƚ'}, // 023d → 019a + {u'Ⱦ', u'ⱦ'}, // 023e → 2c66 + {u'Ɂ', u'ɂ'}, // 0241 → 0242 + {u'Ƀ', u'ƀ'}, // 0243 → 0180 + {u'Ʉ', u'ʉ'}, // 0244 → 0289 + {u'Ʌ', u'ʌ'}, // 0245 → 028c + {u'Ɇ', u'ɇ'}, // 0246 → 0247 + {u'Ɉ', u'ɉ'}, // 0248 → 0249 + {u'Ɍ', u'ɍ'}, // 024c → 024d + {u'Ɏ', u'ɏ'}, // 024e → 024f + {u'Ͱ', u'ͱ'}, // 0370 → 0371 + {u'Ͳ', u'ͳ'}, // 0372 → 0373 + {u'Ͷ', u'ͷ'}, // 0376 → 0377 + {u'Ά', u'ά'}, // 0386 → 03ac + {u'Έ', u'έ'}, // 0388 → 03ad + {u'Ή', u'ή'}, // 0389 → 03ae + {u'Ί', u'ί'}, // 038a → 03af + {u'Ό', u'ό'}, // 038c → 03cc + {u'Ύ', u'ύ'}, // 038e → 03cd + {u'Ώ', u'ώ'}, // 038f → 03ce + {u'Α', u'ά'}, // 0391 → 03ac + {u'Β', u'β'}, // 0392 → 03b2 + {u'Γ', u'γ'}, // 0393 → 03b3 + {u'Δ', u'δ'}, // 0394 → 03b4 + {u'Ε', u'έ'}, // 0395 → 03ad + {u'Ζ', u'ζ'}, // 0396 → 03b6 + {u'Η', u'ή'}, // 0397 → 03ae + {u'Θ', u'θ'}, // 0398 → 03b8 + {u'Ι', u'ΐ'}, // 0399 → 0390 + {u'Κ', u'κ'}, // 039a → 03ba + {u'Λ', u'λ'}, // 039b → 03bb + {u'Μ', u'μ'}, // 039c → 03bc + {u'Ν', u'ν'}, // 039d → 03bd + {u'Ξ', u'ξ'}, // 039e → 03be + {u'Ο', u'ο'}, // 039f → 03bf + {u'Π', u'π'}, // 03a0 → 03c0 + {u'Ρ', u'ρ'}, // 03a1 → 03c1 + {u'Σ', u'σ'}, // 03a3 → 03c3 + {u'Τ', u'τ'}, // 03a4 → 03c4 + {u'Υ', u'ΰ'}, // 03a5 → 03b0 + {u'Φ', u'φ'}, // 03a6 → 03c6 + {u'Χ', u'χ'}, // 03a7 → 03c7 + {u'Ψ', u'ψ'}, // 03a8 → 03c8 + {u'Ω', u'ω'}, // 03a9 → 03c9 + {u'Ϊ', u'ΐ'}, // 03aa → 0390 + {u'Ϋ', u'ΰ'}, // 03ab → 03b0 + {u'Ϣ', u'ϣ'}, // 03e2 → 03e3 + {u'Ϥ', u'ϥ'}, // 03e4 → 03e5 + {u'Ϧ', u'ϧ'}, // 03e6 → 03e7 + {u'Ϩ', u'ϩ'}, // 03e8 → 03e9 + {u'Ϫ', u'ϫ'}, // 03ea → 03eb + {u'Ϭ', u'ϭ'}, // 03ec → 03ed + {u'Ϯ', u'ϯ'}, // 03ee → 03ef + {u'Ϸ', u'ϸ'}, // 03f7 → 03f8 + {u'Ϻ', u'ϻ'}, // 03fa → 03fb + {u'Ͻ', u'ͻ'}, // 03fd → 037b + {u'Ͼ', u'ͼ'}, // 03fe → 037c + {u'Ͽ', u'ͽ'}, // 03ff → 037d + {u'Ѐ', u'ѐ'}, // 0400 → 0450 + {u'Ё', u'ё'}, // 0401 → 0451 + {u'Ђ', u'ђ'}, // 0402 → 0452 + {u'Ѓ', u'ѓ'}, // 0403 → 0453 + {u'Є', u'є'}, // 0404 → 0454 + {u'Ѕ', u'ѕ'}, // 0405 → 0455 + {u'І', u'і'}, // 0406 → 0456 + {u'Ї', u'ї'}, // 0407 → 0457 + {u'Ј', u'ј'}, // 0408 → 0458 + {u'Љ', u'љ'}, // 0409 → 0459 + {u'Њ', u'њ'}, // 040a → 045a + {u'Ћ', u'ћ'}, // 040b → 045b + {u'Ќ', u'ќ'}, // 040c → 045c + {u'Ѝ', u'ѝ'}, // 040d → 045d + {u'Ў', u'ў'}, // 040e → 045e + {u'Џ', u'џ'}, // 040f → 045f + {u'А', u'а'}, // 0410 → 0430 + {u'Б', u'б'}, // 0411 → 0431 + {u'В', u'в'}, // 0412 → 0432 + {u'Г', u'г'}, // 0413 → 0433 + {u'Д', u'д'}, // 0414 → 0434 + {u'Е', u'е'}, // 0415 → 0435 + {u'Ж', u'ж'}, // 0416 → 0436 + {u'З', u'з'}, // 0417 → 0437 + {u'И', u'е'}, // 0418 → 0435 + {u'Й', u'й'}, // 0419 → 0439 + {u'К', u'к'}, // 041a → 043a + {u'Л', u'л'}, // 041b → 043b + {u'М', u'м'}, // 041c → 043c + {u'Н', u'н'}, // 041d → 043d + {u'О', u'о'}, // 041e → 043e + {u'П', u'п'}, // 041f → 043f + {u'Р', u'р'}, // 0420 → 0440 + {u'С', u'с'}, // 0421 → 0441 + {u'Т', u'т'}, // 0422 → 0442 + {u'У', u'у'}, // 0423 → 0443 + {u'Ф', u'ф'}, // 0424 → 0444 + {u'Х', u'х'}, // 0425 → 0445 + {u'Ц', u'ц'}, // 0426 → 0446 + {u'Ч', u'ч'}, // 0427 → 0447 + {u'Ш', u'ш'}, // 0428 → 0448 + {u'Щ', u'щ'}, // 0429 → 0449 + {u'Ъ', u'ъ'}, // 042a → 044a + {u'Ы', u'ы'}, // 042b → 044b + {u'Ь', u'ь'}, // 042c → 044c + {u'Э', u'л'}, // 042d → 043b + {u'Ю', u'ю'}, // 042e → 044e + {u'Я', u'я'}, // 042f → 044f + {u'Ѡ', u'ѡ'}, // 0460 → 0461 + {u'Ѣ', u'ѣ'}, // 0462 → 0463 + {u'Ѥ', u'ѥ'}, // 0464 → 0465 + {u'Ѧ', u'ѧ'}, // 0466 → 0467 + {u'Ѩ', u'ѩ'}, // 0468 → 0469 + {u'Ѫ', u'ѫ'}, // 046a → 046b + {u'Ѭ', u'ѭ'}, // 046c → 046d + {u'Ѯ', u'ѯ'}, // 046e → 046f + {u'Ѱ', u'ѱ'}, // 0470 → 0471 + {u'Ѳ', u'ѳ'}, // 0472 → 0473 + {u'Ѵ', u'ѵ'}, // 0474 → 0475 + {u'Ѷ', u'ѷ'}, // 0476 → 0477 + {u'Ѹ', u'є'}, // 0478 → 0454 + {u'Ѻ', u'ѻ'}, // 047a → 047b + {u'Ѽ', u'ѽ'}, // 047c → 047d + {u'Ѿ', u'ѿ'}, // 047e → 047f + {u'Ҁ', u'ҁ'}, // 0480 → 0481 + {u'Ҋ', u'ҋ'}, // 048a → 048b + {u'Ҍ', u'ҍ'}, // 048c → 048d + {u'Ҏ', u'ҏ'}, // 048e → 048f + {u'Ґ', u'ґ'}, // 0490 → 0491 + {u'Ғ', u'ғ'}, // 0492 → 0493 + {u'Ҕ', u'ҕ'}, // 0494 → 0495 + {u'Җ', u'җ'}, // 0496 → 0497 + {u'Ҙ', u'ҙ'}, // 0498 → 0499 + {u'Қ', u'қ'}, // 049a → 049b + {u'Ҝ', u'ҝ'}, // 049c → 049d + {u'Ҟ', u'ҟ'}, // 049e → 049f + {u'Ҡ', u'ҡ'}, // 04a0 → 04a1 + {u'Ң', u'ң'}, // 04a2 → 04a3 + {u'Ҥ', u'ҥ'}, // 04a4 → 04a5 + {u'Ҧ', u'ҧ'}, // 04a6 → 04a7 + {u'Ҩ', u'ҩ'}, // 04a8 → 04a9 + {u'Ҫ', u'ҫ'}, // 04aa → 04ab + {u'Ҭ', u'ҭ'}, // 04ac → 04ad + {u'Ү', u'ү'}, // 04ae → 04af + {u'Ұ', u'ұ'}, // 04b0 → 04b1 + {u'Ҳ', u'ҳ'}, // 04b2 → 04b3 + {u'Ҵ', u'ҵ'}, // 04b4 → 04b5 + {u'Ҷ', u'ҷ'}, // 04b6 → 04b7 + {u'Ҹ', u'ҹ'}, // 04b8 → 04b9 + {u'Һ', u'һ'}, // 04ba → 04bb + {u'Ҽ', u'ҽ'}, // 04bc → 04bd + {u'Ҿ', u'ҿ'}, // 04be → 04bf + {u'Ӂ', u'ӂ'}, // 04c1 → 04c2 + {u'Ӄ', u'ӄ'}, // 04c3 → 04c4 + {u'Ӆ', u'ӆ'}, // 04c5 → 04c6 + {u'Ӈ', u'ӈ'}, // 04c7 → 04c8 + {u'Ӊ', u'ӊ'}, // 04c9 → 04ca + {u'Ӌ', u'ӌ'}, // 04cb → 04cc + {u'Ӎ', u'ӎ'}, // 04cd → 04ce + {u'Ӑ', u'ӑ'}, // 04d0 → 04d1 + {u'Ӓ', u'ӓ'}, // 04d2 → 04d3 + {u'Ӕ', u'ӕ'}, // 04d4 → 04d5 + {u'Ӗ', u'ӗ'}, // 04d6 → 04d7 + {u'Ә', u'ә'}, // 04d8 → 04d9 + {u'Ӛ', u'ӛ'}, // 04da → 04db + {u'Ӝ', u'ӝ'}, // 04dc → 04dd + {u'Ӟ', u'ӟ'}, // 04de → 04df + {u'Ӡ', u'ӡ'}, // 04e0 → 04e1 + {u'Ӣ', u'ӣ'}, // 04e2 → 04e3 + {u'Ӥ', u'ӥ'}, // 04e4 → 04e5 + {u'Ӧ', u'ӧ'}, // 04e6 → 04e7 + {u'Ө', u'ө'}, // 04e8 → 04e9 + {u'Ӫ', u'ӫ'}, // 04ea → 04eb + {u'Ӭ', u'ӭ'}, // 04ec → 04ed + {u'Ӯ', u'ӯ'}, // 04ee → 04ef + {u'Ӱ', u'ӱ'}, // 04f0 → 04f1 + {u'Ӳ', u'ӳ'}, // 04f2 → 04f3 + {u'Ӵ', u'ӵ'}, // 04f4 → 04f5 + {u'Ӷ', u'ӷ'}, // 04f6 → 04f7 + {u'Ӹ', u'ӹ'}, // 04f8 → 04f9 + {u'Ӻ', u'ӻ'}, // 04fa → 04fb + {u'Ӽ', u'ӽ'}, // 04fc → 04fd + {u'Ӿ', u'ӿ'}, // 04fe → 04ff + {u'Ԁ', u'ԁ'}, // 0500 → 0501 + {u'Ԃ', u'ԃ'}, // 0502 → 0503 + {u'Ԅ', u'ԅ'}, // 0504 → 0505 + {u'Ԇ', u'ԇ'}, // 0506 → 0507 + {u'Ԉ', u'ԉ'}, // 0508 → 0509 + {u'Ԋ', u'ԋ'}, // 050a → 050b + {u'Ԍ', u'ԍ'}, // 050c → 050d + {u'Ԏ', u'ԏ'}, // 050e → 050f + {u'Ԑ', u'ԑ'}, // 0510 → 0511 + {u'Ԓ', u'ԓ'}, // 0512 → 0513 + {u'Ԕ', u'ԕ'}, // 0514 → 0515 + {u'Ԗ', u'ԗ'}, // 0516 → 0517 + {u'Ԙ', u'ԙ'}, // 0518 → 0519 + {u'Ԛ', u'ԛ'}, // 051a → 051b + {u'Ԝ', u'ԝ'}, // 051c → 051d + {u'Ԟ', u'ԟ'}, // 051e → 051f + {u'Ԡ', u'ԡ'}, // 0520 → 0521 + {u'Ԣ', u'ԣ'}, // 0522 → 0523 + {u'Ԥ', u'ԥ'}, // 0524 → 0525 + {u'Ԧ', u'ԧ'}, // 0526 → 0527 + {u'Ԩ', u'ԩ'}, // 0528 → 0529 + {u'Ԫ', u'ԫ'}, // 052a → 052b + {u'Ԭ', u'ԭ'}, // 052c → 052d + {u'Ԯ', u'ԯ'}, // 052e → 052f + {u'Ա', u'ա'}, // 0531 → 0561 + {u'Բ', u'բ'}, // 0532 → 0562 + {u'Գ', u'գ'}, // 0533 → 0563 + {u'Դ', u'դ'}, // 0534 → 0564 + {u'Ե', u'ե'}, // 0535 → 0565 + {u'Զ', u'զ'}, // 0536 → 0566 + {u'Է', u'է'}, // 0537 → 0567 + {u'Ը', u'ը'}, // 0538 → 0568 + {u'Թ', u'թ'}, // 0539 → 0569 + {u'Ժ', u'ժ'}, // 053a → 056a + {u'Ի', u'ի'}, // 053b → 056b + {u'Լ', u'լ'}, // 053c → 056c + {u'Խ', u'խ'}, // 053d → 056d + {u'Ծ', u'ծ'}, // 053e → 056e + {u'Կ', u'կ'}, // 053f → 056f + {u'Հ', u'հ'}, // 0540 → 0570 + {u'Ձ', u'ձ'}, // 0541 → 0571 + {u'Ղ', u'ղ'}, // 0542 → 0572 + {u'Ճ', u'ճ'}, // 0543 → 0573 + {u'Մ', u'մ'}, // 0544 → 0574 + {u'Յ', u'յ'}, // 0545 → 0575 + {u'Ն', u'ն'}, // 0546 → 0576 + {u'Շ', u'շ'}, // 0547 → 0577 + {u'Ո', u'ո'}, // 0548 → 0578 + {u'Չ', u'չ'}, // 0549 → 0579 + {u'Պ', u'պ'}, // 054a → 057a + {u'Ջ', u'ջ'}, // 054b → 057b + {u'Ռ', u'ռ'}, // 054c → 057c + {u'Ս', u'ս'}, // 054d → 057d + {u'Վ', u'վ'}, // 054e → 057e + {u'Տ', u'տ'}, // 054f → 057f + {u'Ր', u'ր'}, // 0550 → 0580 + {u'Ց', u'ց'}, // 0551 → 0581 + {u'Ւ', u'ւ'}, // 0552 → 0582 + {u'Փ', u'փ'}, // 0553 → 0583 + {u'Ք', u'ք'}, // 0554 → 0584 + {u'Օ', u'օ'}, // 0555 → 0585 + {u'Ֆ', u'ֆ'}, // 0556 → 0586 + {u'Ⴀ', u'ⴀ'}, // 10a0 → 2d00 + {u'Ⴁ', u'ⴁ'}, // 10a1 → 2d01 + {u'Ⴂ', u'ⴂ'}, // 10a2 → 2d02 + {u'Ⴃ', u'ⴃ'}, // 10a3 → 2d03 + {u'Ⴄ', u'ⴄ'}, // 10a4 → 2d04 + {u'Ⴅ', u'ⴅ'}, // 10a5 → 2d05 + {u'Ⴆ', u'ⴆ'}, // 10a6 → 2d06 + {u'Ⴇ', u'ⴇ'}, // 10a7 → 2d07 + {u'Ⴈ', u'ⴈ'}, // 10a8 → 2d08 + {u'Ⴉ', u'ⴉ'}, // 10a9 → 2d09 + {u'Ⴊ', u'ⴊ'}, // 10aa → 2d0a + {u'Ⴋ', u'ⴋ'}, // 10ab → 2d0b + {u'Ⴌ', u'ⴌ'}, // 10ac → 2d0c + {u'Ⴍ', u'ⴍ'}, // 10ad → 2d0d + {u'Ⴎ', u'ⴎ'}, // 10ae → 2d0e + {u'Ⴏ', u'ⴏ'}, // 10af → 2d0f + {u'Ⴐ', u'ⴐ'}, // 10b0 → 2d10 + {u'Ⴑ', u'ⴑ'}, // 10b1 → 2d11 + {u'Ⴒ', u'ⴒ'}, // 10b2 → 2d12 + {u'Ⴓ', u'ⴓ'}, // 10b3 → 2d13 + {u'Ⴔ', u'ⴔ'}, // 10b4 → 2d14 + {u'Ⴕ', u'ⴕ'}, // 10b5 → 2d15 + {u'Ⴖ', u'ⴖ'}, // 10b6 → 2d16 + {u'Ⴗ', u'ⴗ'}, // 10b7 → 2d17 + {u'Ⴘ', u'ⴘ'}, // 10b8 → 2d18 + {u'Ⴙ', u'ⴙ'}, // 10b9 → 2d19 + {u'Ⴚ', u'ⴚ'}, // 10ba → 2d1a + {u'Ⴛ', u'ⴛ'}, // 10bb → 2d1b + {u'Ⴜ', u'ⴜ'}, // 10bc → 2d1c + {u'Ⴝ', u'ⴝ'}, // 10bd → 2d1d + {u'Ⴞ', u'ⴞ'}, // 10be → 2d1e + {u'Ⴟ', u'ⴟ'}, // 10bf → 2d1f + {u'Ⴠ', u'ⴠ'}, // 10c0 → 2d20 + {u'Ⴡ', u'ⴡ'}, // 10c1 → 2d21 + {u'Ⴢ', u'ⴢ'}, // 10c2 → 2d22 + {u'Ⴣ', u'ⴣ'}, // 10c3 → 2d23 + {u'Ⴤ', u'ⴤ'}, // 10c4 → 2d24 + {u'Ⴥ', u'ⴥ'}, // 10c5 → 2d25 + {u'Ⴧ', u'ⴧ'}, // 10c7 → 2d27 + {u'Ⴭ', u'ⴭ'}, // 10cd → 2d2d + {u'Ᏸ', u'ᏸ'}, // 13f0 → 13f8 + {u'Ᏹ', u'ᏹ'}, // 13f1 → 13f9 + {u'Ᏺ', u'ᏺ'}, // 13f2 → 13fa + {u'Ᏻ', u'ᏻ'}, // 13f3 → 13fb + {u'Ᏼ', u'ᏼ'}, // 13f4 → 13fc + {u'Ᏽ', u'ᏽ'}, // 13f5 → 13fd + {u'Ḁ', u'ḁ'}, // 1e00 → 1e01 + {u'Ḃ', u'ḃ'}, // 1e02 → 1e03 + {u'Ḅ', u'ḅ'}, // 1e04 → 1e05 + {u'Ḇ', u'ḇ'}, // 1e06 → 1e07 + {u'Ḉ', u'ḉ'}, // 1e08 → 1e09 + {u'Ḋ', u'ḋ'}, // 1e0a → 1e0b + {u'Ḍ', u'ḍ'}, // 1e0c → 1e0d + {u'Ḏ', u'ḏ'}, // 1e0e → 1e0f + {u'Ḑ', u'ḑ'}, // 1e10 → 1e11 + {u'Ḓ', u'ḓ'}, // 1e12 → 1e13 + {u'Ḕ', u'ḕ'}, // 1e14 → 1e15 + {u'Ḗ', u'ḗ'}, // 1e16 → 1e17 + {u'Ḙ', u'ḙ'}, // 1e18 → 1e19 + {u'Ḛ', u'ḛ'}, // 1e1a → 1e1b + {u'Ḝ', u'ḝ'}, // 1e1c → 1e1d + {u'Ḟ', u'ḟ'}, // 1e1e → 1e1f + {u'Ḡ', u'ḡ'}, // 1e20 → 1e21 + {u'Ḣ', u'ḣ'}, // 1e22 → 1e23 + {u'Ḥ', u'ḥ'}, // 1e24 → 1e25 + {u'Ḧ', u'ḧ'}, // 1e26 → 1e27 + {u'Ḩ', u'ḩ'}, // 1e28 → 1e29 + {u'Ḫ', u'ḫ'}, // 1e2a → 1e2b + {u'Ḭ', u'ḭ'}, // 1e2c → 1e2d + {u'Ḯ', u'ḯ'}, // 1e2e → 1e2f + {u'Ḱ', u'ḱ'}, // 1e30 → 1e31 + {u'Ḳ', u'ḳ'}, // 1e32 → 1e33 + {u'Ḵ', u'ḵ'}, // 1e34 → 1e35 + {u'Ḷ', u'ḷ'}, // 1e36 → 1e37 + {u'Ḹ', u'ḹ'}, // 1e38 → 1e39 + {u'Ḻ', u'ḻ'}, // 1e3a → 1e3b + {u'Ḽ', u'ḽ'}, // 1e3c → 1e3d + {u'Ḿ', u'ḿ'}, // 1e3e → 1e3f + {u'Ṁ', u'ṁ'}, // 1e40 → 1e41 + {u'Ṃ', u'ṃ'}, // 1e42 → 1e43 + {u'Ṅ', u'ṅ'}, // 1e44 → 1e45 + {u'Ṇ', u'ṇ'}, // 1e46 → 1e47 + {u'Ṉ', u'ṉ'}, // 1e48 → 1e49 + {u'Ṋ', u'ṋ'}, // 1e4a → 1e4b + {u'Ṍ', u'ṍ'}, // 1e4c → 1e4d + {u'Ṏ', u'ṏ'}, // 1e4e → 1e4f + {u'Ṑ', u'ṑ'}, // 1e50 → 1e51 + {u'Ṓ', u'ṓ'}, // 1e52 → 1e53 + {u'Ṕ', u'ṕ'}, // 1e54 → 1e55 + {u'Ṗ', u'ṗ'}, // 1e56 → 1e57 + {u'Ṙ', u'ṙ'}, // 1e58 → 1e59 + {u'Ṛ', u'ṛ'}, // 1e5a → 1e5b + {u'Ṝ', u'ṝ'}, // 1e5c → 1e5d + {u'Ṟ', u'ṟ'}, // 1e5e → 1e5f + {u'Ṡ', u'ṡ'}, // 1e60 → 1e61 + {u'Ṣ', u'ṣ'}, // 1e62 → 1e63 + {u'Ṥ', u'ṥ'}, // 1e64 → 1e65 + {u'Ṧ', u'ṧ'}, // 1e66 → 1e67 + {u'Ṩ', u'ṩ'}, // 1e68 → 1e69 + {u'Ṫ', u'ṫ'}, // 1e6a → 1e6b + {u'Ṭ', u'ṭ'}, // 1e6c → 1e6d + {u'Ṯ', u'ṯ'}, // 1e6e → 1e6f + {u'Ṱ', u'ṱ'}, // 1e70 → 1e71 + {u'Ṳ', u'ṳ'}, // 1e72 → 1e73 + {u'Ṵ', u'ṵ'}, // 1e74 → 1e75 + {u'Ṷ', u'ṷ'}, // 1e76 → 1e77 + {u'Ṹ', u'ṹ'}, // 1e78 → 1e79 + {u'Ṻ', u'ṻ'}, // 1e7a → 1e7b + {u'Ṽ', u'ṽ'}, // 1e7c → 1e7d + {u'Ṿ', u'ṿ'}, // 1e7e → 1e7f + {u'Ẁ', u'ẁ'}, // 1e80 → 1e81 + {u'Ẃ', u'ẃ'}, // 1e82 → 1e83 + {u'Ẅ', u'ẅ'}, // 1e84 → 1e85 + {u'Ẇ', u'ẇ'}, // 1e86 → 1e87 + {u'Ẉ', u'ẉ'}, // 1e88 → 1e89 + {u'Ẋ', u'ẋ'}, // 1e8a → 1e8b + {u'Ẍ', u'ẍ'}, // 1e8c → 1e8d + {u'Ẏ', u'ẏ'}, // 1e8e → 1e8f + {u'Ẑ', u'ẑ'}, // 1e90 → 1e91 + {u'Ẓ', u'ẓ'}, // 1e92 → 1e93 + {u'Ẕ', u'ẕ'}, // 1e94 → 1e95 + {u'ẞ', u'ß'}, // 1e9e → 00df + {u'Ạ', u'ạ'}, // 1ea0 → 1ea1 + {u'Ả', u'ả'}, // 1ea2 → 1ea3 + {u'Ấ', u'ấ'}, // 1ea4 → 1ea5 + {u'Ầ', u'ầ'}, // 1ea6 → 1ea7 + {u'Ẩ', u'ẩ'}, // 1ea8 → 1ea9 + {u'Ẫ', u'ẫ'}, // 1eaa → 1eab + {u'Ậ', u'ậ'}, // 1eac → 1ead + {u'Ắ', u'ắ'}, // 1eae → 1eaf + {u'Ằ', u'ằ'}, // 1eb0 → 1eb1 + {u'Ẳ', u'ẳ'}, // 1eb2 → 1eb3 + {u'Ẵ', u'ẵ'}, // 1eb4 → 1eb5 + {u'Ặ', u'ặ'}, // 1eb6 → 1eb7 + {u'Ẹ', u'ẹ'}, // 1eb8 → 1eb9 + {u'Ẻ', u'ẻ'}, // 1eba → 1ebb + {u'Ẽ', u'ḛ'}, // 1ebc → 1e1b + {u'Ế', u'ế'}, // 1ebe → 1ebf + {u'Ề', u'ề'}, // 1ec0 → 1ec1 + {u'Ể', u'ể'}, // 1ec2 → 1ec3 + {u'Ễ', u'ễ'}, // 1ec4 → 1ec5 + {u'Ệ', u'ệ'}, // 1ec6 → 1ec7 + {u'Ỉ', u'ỉ'}, // 1ec8 → 1ec9 + {u'Ị', u'ị'}, // 1eca → 1ecb + {u'Ọ', u'ọ'}, // 1ecc → 1ecd + {u'Ỏ', u'ỏ'}, // 1ece → 1ecf + {u'Ố', u'ố'}, // 1ed0 → 1ed1 + {u'Ồ', u'ồ'}, // 1ed2 → 1ed3 + {u'Ổ', u'ổ'}, // 1ed4 → 1ed5 + {u'Ỗ', u'ỗ'}, // 1ed6 → 1ed7 + {u'Ộ', u'ộ'}, // 1ed8 → 1ed9 + {u'Ớ', u'ớ'}, // 1eda → 1edb + {u'Ờ', u'ờ'}, // 1edc → 1edd + {u'Ở', u'ở'}, // 1ede → 1edf + {u'Ỡ', u'ỡ'}, // 1ee0 → 1ee1 + {u'Ợ', u'ợ'}, // 1ee2 → 1ee3 + {u'Ụ', u'ụ'}, // 1ee4 → 1ee5 + {u'Ủ', u'ủ'}, // 1ee6 → 1ee7 + {u'Ứ', u'ứ'}, // 1ee8 → 1ee9 + {u'Ừ', u'ừ'}, // 1eea → 1eeb + {u'Ử', u'ử'}, // 1eec → 1eed + {u'Ữ', u'ữ'}, // 1eee → 1eef + {u'Ự', u'ự'}, // 1ef0 → 1ef1 + {u'Ỳ', u'ỳ'}, // 1ef2 → 1ef3 + {u'Ỵ', u'ỵ'}, // 1ef4 → 1ef5 + {u'Ỷ', u'ỷ'}, // 1ef6 → 1ef7 + {u'Ỹ', u'ỹ'}, // 1ef8 → 1ef9 + {u'Ỻ', u'ỻ'}, // 1efa → 1efb + {u'Ỽ', u'ỽ'}, // 1efc → 1efd + {u'Ỿ', u'ỿ'}, // 1efe → 1eff + {u'Ἀ', u'ἀ'}, // 1f08 → 1f00 + {u'Ἁ', u'ἁ'}, // 1f09 → 1f01 + {u'Ἂ', u'ἂ'}, // 1f0a → 1f02 + {u'Ἃ', u'ἃ'}, // 1f0b → 1f03 + {u'Ἄ', u'ἄ'}, // 1f0c → 1f04 + {u'Ἅ', u'ἅ'}, // 1f0d → 1f05 + {u'Ἆ', u'ἆ'}, // 1f0e → 1f06 + {u'Ἇ', u'ἇ'}, // 1f0f → 1f07 + {u'Ἐ', u'ἐ'}, // 1f18 → 1f10 + {u'Ἑ', u'ἑ'}, // 1f19 → 1f11 + {u'Ἒ', u'ἒ'}, // 1f1a → 1f12 + {u'Ἓ', u'ἓ'}, // 1f1b → 1f13 + {u'Ἔ', u'ἔ'}, // 1f1c → 1f14 + {u'Ἕ', u'ἕ'}, // 1f1d → 1f15 + {u'Ἠ', u'ἠ'}, // 1f28 → 1f20 + {u'Ἡ', u'ἡ'}, // 1f29 → 1f21 + {u'Ἢ', u'ἢ'}, // 1f2a → 1f22 + {u'Ἣ', u'ἣ'}, // 1f2b → 1f23 + {u'Ἤ', u'ἤ'}, // 1f2c → 1f24 + {u'Ἥ', u'ἥ'}, // 1f2d → 1f25 + {u'Ἦ', u'ἦ'}, // 1f2e → 1f26 + {u'Ἧ', u'ἧ'}, // 1f2f → 1f27 + {u'Ἰ', u'ἰ'}, // 1f38 → 1f30 + {u'Ἱ', u'ἱ'}, // 1f39 → 1f31 + {u'Ἲ', u'ἲ'}, // 1f3a → 1f32 + {u'Ἳ', u'ἳ'}, // 1f3b → 1f33 + {u'Ἴ', u'ἴ'}, // 1f3c → 1f34 + {u'Ἵ', u'ἵ'}, // 1f3d → 1f35 + {u'Ἶ', u'ἶ'}, // 1f3e → 1f36 + {u'Ἷ', u'ἷ'}, // 1f3f → 1f37 + {u'Ὀ', u'ὀ'}, // 1f48 → 1f40 + {u'Ὁ', u'ὁ'}, // 1f49 → 1f41 + {u'Ὂ', u'ὂ'}, // 1f4a → 1f42 + {u'Ὃ', u'ὃ'}, // 1f4b → 1f43 + {u'Ὄ', u'ὄ'}, // 1f4c → 1f44 + {u'Ὅ', u'ὅ'}, // 1f4d → 1f45 + {u'Ὑ', u'ὑ'}, // 1f59 → 1f51 + {u'Ὓ', u'ὓ'}, // 1f5b → 1f53 + {u'Ὕ', u'ὕ'}, // 1f5d → 1f55 + {u'Ὗ', u'ὗ'}, // 1f5f → 1f57 + {u'Ὠ', u'ὠ'}, // 1f68 → 1f60 + {u'Ὡ', u'ὡ'}, // 1f69 → 1f61 + {u'Ὢ', u'ὢ'}, // 1f6a → 1f62 + {u'Ὣ', u'ὣ'}, // 1f6b → 1f63 + {u'Ὤ', u'ὤ'}, // 1f6c → 1f64 + {u'Ὥ', u'ὥ'}, // 1f6d → 1f65 + {u'Ὦ', u'ὦ'}, // 1f6e → 1f66 + {u'Ὧ', u'ὧ'}, // 1f6f → 1f67 + {u'Ᾰ', u'ᾰ'}, // 1fb8 → 1fb0 + {u'Ᾱ', u'ᾱ'}, // 1fb9 → 1fb1 + {u'Ὰ', u'ὰ'}, // 1fba → 1f70 + {u'Ά', u'ά'}, // 1fbb → 1f71 + {u'Ὲ', u'ὲ'}, // 1fc8 → 1f72 + {u'Έ', u'έ'}, // 1fc9 → 1f73 + {u'Ὴ', u'ὴ'}, // 1fca → 1f74 + {u'Ή', u'ή'}, // 1fcb → 1f75 + {u'Ῐ', u'ῐ'}, // 1fd8 → 1fd0 + {u'Ῑ', u'ῑ'}, // 1fd9 → 1fd1 + {u'Ὶ', u'ὶ'}, // 1fda → 1f76 + {u'Ί', u'ί'}, // 1fdb → 1f77 + {u'Ῠ', u'ῠ'}, // 1fe8 → 1fe0 + {u'Ῡ', u'ῡ'}, // 1fe9 → 1fe1 + {u'Ὺ', u'ὺ'}, // 1fea → 1f7a + {u'Ύ', u'ύ'}, // 1feb → 1f7b + {u'Ῥ', u'ῥ'}, // 1fec → 1fe5 + {u'Ὸ', u'ὸ'}, // 1ff8 → 1f78 + {u'Ό', u'ό'}, // 1ff9 → 1f79 + {u'Ὼ', u'ὼ'}, // 1ffa → 1f7c + {u'Ώ', u'ώ'}, // 1ffb → 1f7d + {u'ℒ', u'ℓ'}, // 2112 → 2113 + {u'ℙ', u'ℼ'}, // 2119 → 213c + {u'ℰ', u'ℯ'}, // 2130 → 212f + {u'Ⅎ', u'ⅎ'}, // 2132 → 214e + {u'ℾ', u'ℽ'}, // 213e → 213d + {u'ℿ', u'ℼ'}, // 213f → 213c + {u'ⅅ', u'ⅆ'}, // 2145 → 2146 + {u'Ⰰ', u'ⰰ'}, // 2c00 → 2c30 + {u'Ⰱ', u'ⰱ'}, // 2c01 → 2c31 + {u'Ⰲ', u'ⰲ'}, // 2c02 → 2c32 + {u'Ⰳ', u'ⰳ'}, // 2c03 → 2c33 + {u'Ⰴ', u'ⰴ'}, // 2c04 → 2c34 + {u'Ⰵ', u'ⰵ'}, // 2c05 → 2c35 + {u'Ⰶ', u'ⰶ'}, // 2c06 → 2c36 + {u'Ⰷ', u'ⰷ'}, // 2c07 → 2c37 + {u'Ⰸ', u'ⰸ'}, // 2c08 → 2c38 + {u'Ⰹ', u'ⰹ'}, // 2c09 → 2c39 + {u'Ⰺ', u'ⰺ'}, // 2c0a → 2c3a + {u'Ⰻ', u'ⰹ'}, // 2c0b → 2c39 + {u'Ⰼ', u'ⰼ'}, // 2c0c → 2c3c + {u'Ⰽ', u'ⰽ'}, // 2c0d → 2c3d + {u'Ⰾ', u'ⰾ'}, // 2c0e → 2c3e + {u'Ⰿ', u'ⰿ'}, // 2c0f → 2c3f + {u'Ⱀ', u'ⱀ'}, // 2c10 → 2c40 + {u'Ⱁ', u'ⱁ'}, // 2c11 → 2c41 + {u'Ⱂ', u'ⱂ'}, // 2c12 → 2c42 + {u'Ⱃ', u'ⱃ'}, // 2c13 → 2c43 + {u'Ⱄ', u'ⱄ'}, // 2c14 → 2c44 + {u'Ⱅ', u'ⱅ'}, // 2c15 → 2c45 + {u'Ⱆ', u'ⱆ'}, // 2c16 → 2c46 + {u'Ⱇ', u'ⱇ'}, // 2c17 → 2c47 + {u'Ⱈ', u'ⱈ'}, // 2c18 → 2c48 + {u'Ⱉ', u'ⱉ'}, // 2c19 → 2c49 + {u'Ⱊ', u'ⱊ'}, // 2c1a → 2c4a + {u'Ⱋ', u'ⱋ'}, // 2c1b → 2c4b + {u'Ⱌ', u'ⱌ'}, // 2c1c → 2c4c + {u'Ⱍ', u'ⱍ'}, // 2c1d → 2c4d + {u'Ⱎ', u'ⱎ'}, // 2c1e → 2c4e + {u'Ⱏ', u'ⱏ'}, // 2c1f → 2c4f + {u'Ⱐ', u'ⱐ'}, // 2c20 → 2c50 + {u'Ⱑ', u'ⱑ'}, // 2c21 → 2c51 + {u'Ⱒ', u'ⱒ'}, // 2c22 → 2c52 + {u'Ⱓ', u'ⱓ'}, // 2c23 → 2c53 + {u'Ⱔ', u'ⱔ'}, // 2c24 → 2c54 + {u'Ⱕ', u'ⱕ'}, // 2c25 → 2c55 + {u'Ⱖ', u'ⱖ'}, // 2c26 → 2c56 + {u'Ⱗ', u'ⱗ'}, // 2c27 → 2c57 + {u'Ⱘ', u'ⱘ'}, // 2c28 → 2c58 + {u'Ⱙ', u'ⱙ'}, // 2c29 → 2c59 + {u'Ⱚ', u'ⱚ'}, // 2c2a → 2c5a + {u'Ⱛ', u'ⱛ'}, // 2c2b → 2c5b + {u'Ⱜ', u'ⱜ'}, // 2c2c → 2c5c + {u'Ⱝ', u'ⱝ'}, // 2c2d → 2c5d + {u'Ⱞ', u'ⱞ'}, // 2c2e → 2c5e + {u'Ⱡ', u'ⱡ'}, // 2c60 → 2c61 + {u'Ɫ', u'ɫ'}, // 2c62 → 026b + {u'Ᵽ', u'ᵽ'}, // 2c63 → 1d7d + {u'Ɽ', u'ɽ'}, // 2c64 → 027d + {u'Ⱨ', u'ⱨ'}, // 2c67 → 2c68 + {u'Ⱪ', u'ⱪ'}, // 2c69 → 2c6a + {u'Ⱬ', u'ⱬ'}, // 2c6b → 2c6c + {u'Ɑ', u'ɑ'}, // 2c6d → 0251 + {u'Ɱ', u'ɱ'}, // 2c6e → 0271 + {u'Ɐ', u'ɐ'}, // 2c6f → 0250 + {u'Ɒ', u'ɒ'}, // 2c70 → 0252 + {u'Ⱳ', u'ⱳ'}, // 2c72 → 2c73 + {u'Ⱶ', u'ⱶ'}, // 2c75 → 2c76 + {u'Ȿ', u'ȿ'}, // 2c7e → 023f + {u'Ɀ', u'ɀ'}, // 2c7f → 0240 + {u'Ⲁ', u'ⲁ'}, // 2c80 → 2c81 + {u'Ⲃ', u'ⲃ'}, // 2c82 → 2c83 + {u'Ⲅ', u'ⲅ'}, // 2c84 → 2c85 + {u'Ⲇ', u'ⲇ'}, // 2c86 → 2c87 + {u'Ⲉ', u'ⲉ'}, // 2c88 → 2c89 + {u'Ⲋ', u'ⲋ'}, // 2c8a → 2c8b + {u'Ⲍ', u'ⲍ'}, // 2c8c → 2c8d + {u'Ⲏ', u'ⲏ'}, // 2c8e → 2c8f + {u'Ⲑ', u'ⲑ'}, // 2c90 → 2c91 + {u'Ⲓ', u'ⲓ'}, // 2c92 → 2c93 + {u'Ⲕ', u'ⲕ'}, // 2c94 → 2c95 + {u'Ⲗ', u'ⲗ'}, // 2c96 → 2c97 + {u'Ⲙ', u'ⲙ'}, // 2c98 → 2c99 + {u'Ⲛ', u'ⲛ'}, // 2c9a → 2c9b + {u'Ⲝ', u'ⲝ'}, // 2c9c → 2c9d + {u'Ⲟ', u'ⲟ'}, // 2c9e → 2c9f + {u'Ⲡ', u'ⲡ'}, // 2ca0 → 2ca1 + {u'Ⲣ', u'ⲣ'}, // 2ca2 → 2ca3 + {u'Ⲥ', u'ⲥ'}, // 2ca4 → 2ca5 + {u'Ⲧ', u'ⲧ'}, // 2ca6 → 2ca7 + {u'Ⲩ', u'ⲩ'}, // 2ca8 → 2ca9 + {u'Ⲫ', u'ⲫ'}, // 2caa → 2cab + {u'Ⲭ', u'ⲭ'}, // 2cac → 2cad + {u'Ⲯ', u'ⲯ'}, // 2cae → 2caf + {u'Ⲱ', u'ⲱ'}, // 2cb0 → 2cb1 + {u'Ⲳ', u'ⲳ'}, // 2cb2 → 2cb3 + {u'Ⲵ', u'ⲵ'}, // 2cb4 → 2cb5 + {u'Ⲷ', u'ⲷ'}, // 2cb6 → 2cb7 + {u'Ⲹ', u'ⲹ'}, // 2cb8 → 2cb9 + {u'Ⲻ', u'ⲻ'}, // 2cba → 2cbb + {u'Ⲽ', u'ⲽ'}, // 2cbc → 2cbd + {u'Ⲿ', u'ⲿ'}, // 2cbe → 2cbf + {u'Ⳁ', u'ⳁ'}, // 2cc0 → 2cc1 + {u'Ⳃ', u'ⳃ'}, // 2cc2 → 2cc3 + {u'Ⳅ', u'ⳅ'}, // 2cc4 → 2cc5 + {u'Ⳇ', u'ⳇ'}, // 2cc6 → 2cc7 + {u'Ⳉ', u'ⳉ'}, // 2cc8 → 2cc9 + {u'Ⳋ', u'ⳋ'}, // 2cca → 2ccb + {u'Ⳍ', u'ⳍ'}, // 2ccc → 2ccd + {u'Ⳏ', u'ⳏ'}, // 2cce → 2ccf + {u'Ⳑ', u'ⳑ'}, // 2cd0 → 2cd1 + {u'Ⳓ', u'ⳓ'}, // 2cd2 → 2cd3 + {u'Ⳕ', u'ⳕ'}, // 2cd4 → 2cd5 + {u'Ⳗ', u'ⳗ'}, // 2cd6 → 2cd7 + {u'Ⳙ', u'ⳙ'}, // 2cd8 → 2cd9 + {u'Ⳛ', u'ⳛ'}, // 2cda → 2cdb + {u'Ⳝ', u'ⳝ'}, // 2cdc → 2cdd + {u'Ⳟ', u'ⳟ'}, // 2cde → 2cdf + {u'Ⳡ', u'ⳡ'}, // 2ce0 → 2ce1 + {u'Ⳣ', u'ⳣ'}, // 2ce2 → 2ce3 + {u'Ⳬ', u'ⳬ'}, // 2ceb → 2cec + {u'Ⳮ', u'ⳮ'}, // 2ced → 2cee + {u'Ⳳ', u'ⳳ'}, // 2cf2 → 2cf3 + {u'Ꙁ', u'ꙁ'}, // a640 → a641 + {u'Ꙃ', u'ꙃ'}, // a642 → a643 + {u'Ꙅ', u'ꙅ'}, // a644 → a645 + {u'Ꙇ', u'ꙇ'}, // a646 → a647 + {u'Ꙉ', u'ꙉ'}, // a648 → a649 + {u'Ꙋ', u'ꙋ'}, // a64a → a64b + {u'Ꙍ', u'ꙍ'}, // a64c → a64d + {u'Ꙏ', u'ꙏ'}, // a64e → a64f + {u'Ꙑ', u'ꙑ'}, // a650 → a651 + {u'Ꙓ', u'ꙓ'}, // a652 → a653 + {u'Ꙕ', u'ꙕ'}, // a654 → a655 + {u'Ꙗ', u'ꙗ'}, // a656 → a657 + {u'Ꙙ', u'ꙙ'}, // a658 → a659 + {u'Ꙛ', u'ꙛ'}, // a65a → a65b + {u'Ꙝ', u'ꙝ'}, // a65c → a65d + {u'Ꙟ', u'ꙟ'}, // a65e → a65f + {u'Ꙡ', u'ꙡ'}, // a660 → a661 + {u'Ꙣ', u'ꙣ'}, // a662 → a663 + {u'Ꙥ', u'ꙥ'}, // a664 → a665 + {u'Ꙧ', u'ꙧ'}, // a666 → a667 + {u'Ꙩ', u'ꙩ'}, // a668 → a669 + {u'Ꙫ', u'ꙫ'}, // a66a → a66b + {u'Ꙭ', u'ꙭ'}, // a66c → a66d + {u'Ꚁ', u'ꚁ'}, // a680 → a681 + {u'Ꚃ', u'ꚃ'}, // a682 → a683 + {u'Ꚅ', u'ꚅ'}, // a684 → a685 + {u'Ꚇ', u'ꚇ'}, // a686 → a687 + {u'Ꚉ', u'ꚉ'}, // a688 → a689 + {u'Ꚋ', u'ꚋ'}, // a68a → a68b + {u'Ꚍ', u'ꚍ'}, // a68c → a68d + {u'Ꚏ', u'ꚏ'}, // a68e → a68f + {u'Ꚑ', u'ꚑ'}, // a690 → a691 + {u'Ꚓ', u'ꚓ'}, // a692 → a693 + {u'Ꚕ', u'ꚕ'}, // a694 → a695 + {u'Ꚗ', u'ꚗ'}, // a696 → a697 + {u'Ꚙ', u'ꚙ'}, // a698 → a699 + {u'Ꚛ', u'ꚛ'}, // a69a → a69b + {u'Ꜣ', u'ꜣ'}, // a722 → a723 + {u'Ꜥ', u'ꜥ'}, // a724 → a725 + {u'Ꜧ', u'ꜧ'}, // a726 → a727 + {u'Ꜩ', u'ꜩ'}, // a728 → a729 + {u'Ꜫ', u'ꜫ'}, // a72a → a72b + {u'Ꜭ', u'ꜭ'}, // a72c → a72d + {u'Ꜯ', u'ꜯ'}, // a72e → a72f + {u'Ꜳ', u'ꜳ'}, // a732 → a733 + {u'Ꜵ', u'ꜵ'}, // a734 → a735 + {u'Ꜷ', u'ꜷ'}, // a736 → a737 + {u'Ꜹ', u'ꜹ'}, // a738 → a739 + {u'Ꜻ', u'ꜻ'}, // a73a → a73b + {u'Ꜽ', u'ꜽ'}, // a73c → a73d + {u'Ꜿ', u'ꜿ'}, // a73e → a73f + {u'Ꝁ', u'ꝁ'}, // a740 → a741 + {u'Ꝃ', u'ꝃ'}, // a742 → a743 + {u'Ꝅ', u'ꝅ'}, // a744 → a745 + {u'Ꝇ', u'ꝇ'}, // a746 → a747 + {u'Ꝉ', u'ꝉ'}, // a748 → a749 + {u'Ꝋ', u'ꝋ'}, // a74a → a74b + {u'Ꝍ', u'ꝍ'}, // a74c → a74d + {u'Ꝏ', u'ꝏ'}, // a74e → a74f + {u'Ꝑ', u'ꝑ'}, // a750 → a751 + {u'Ꝓ', u'ꝓ'}, // a752 → a753 + {u'Ꝕ', u'ꝕ'}, // a754 → a755 + {u'Ꝗ', u'ꝑ'}, // a756 → a751 + {u'Ꝙ', u'ꝙ'}, // a758 → a759 + {u'Ꝛ', u'ꝛ'}, // a75a → a75b + {u'Ꝝ', u'ꝝ'}, // a75c → a75d + {u'Ꝟ', u'ꝟ'}, // a75e → a75f + {u'Ꝡ', u'ꝡ'}, // a760 → a761 + {u'Ꝣ', u'ꝣ'}, // a762 → a763 + {u'Ꝥ', u'ꝥ'}, // a764 → a765 + {u'Ꝧ', u'ꝑ'}, // a766 → a751 + {u'Ꝩ', u'ꝩ'}, // a768 → a769 + {u'Ꝫ', u'ꝫ'}, // a76a → a76b + {u'Ꝭ', u'ꝭ'}, // a76c → a76d + {u'Ꝯ', u'ꝯ'}, // a76e → a76f + {u'Ꝺ', u'ꝺ'}, // a779 → a77a + {u'Ꝼ', u'ꝼ'}, // a77b → a77c + {u'Ᵹ', u'ᵹ'}, // a77d → 1d79 + {u'Ꝿ', u'ꝿ'}, // a77e → a77f + {u'Ꞁ', u'ꞁ'}, // a780 → a781 + {u'Ꞃ', u'ꞃ'}, // a782 → a783 + {u'Ꞅ', u'ꞅ'}, // a784 → a785 + {u'Ꞇ', u'ꞇ'}, // a786 → a787 + {u'Ꞌ', u'ꞌ'}, // a78b → a78c + {u'Ɥ', u'ɥ'}, // a78d → 0265 + {u'Ꞑ', u'ꞑ'}, // a790 → a791 + {u'Ꞓ', u'ꞓ'}, // a792 → a793 + {u'Ꞗ', u'ꞗ'}, // a796 → a797 + {u'Ꞙ', u'ꞙ'}, // a798 → a799 + {u'Ꞛ', u'ꞛ'}, // a79a → a79b + {u'Ꞝ', u'ꞝ'}, // a79c → a79d + {u'Ꞟ', u'ꞟ'}, // a79e → a79f + {u'Ꞡ', u'ꞡ'}, // a7a0 → a7a1 + {u'Ꞣ', u'ꞣ'}, // a7a2 → a7a3 + {u'Ꞥ', u'ꞥ'}, // a7a4 → a7a5 + {u'Ꞧ', u'ꞧ'}, // a7a6 → a7a7 + {u'Ꞩ', u'ꞩ'}, // a7a8 → a7a9 + {u'Ɦ', u'ɦ'}, // a7aa → 0266 + {u'Ɜ', u'ɜ'}, // a7ab → 025c + {u'Ɡ', u'ɡ'}, // a7ac → 0261 + {u'Ɬ', u'ɬ'}, // a7ad → 026c + {u'Ʞ', u'ʞ'}, // a7b0 → 029e + {u'Ʇ', u'ʇ'}, // a7b1 → 0287 + {u'Ʝ', u'ʝ'}, // a7b2 → 029d + {u'Ꭓ', u'ꭓ'}, // a7b3 → ab53 + {u'Ꞵ', u'ꞵ'}, // a7b4 → a7b5 + {u'Ꞷ', u'ꞷ'}, // a7b6 → a7b7 + {u'Ꞹ', u'ꞹ'}, // a7b8 → a7b9 + {u'Ꞻ', u'ꞻ'}, // a7ba → a7bb + {u'Ꞽ', u'ꞽ'}, // a7bc → a7bd + {u'Ꞿ', u'ꞿ'}, // a7be → a7bf + {u'Ꟃ', u'ꟃ'}, // a7c2 → a7c3 + {u'Ꞔ', u'ꞔ'}, // a7c4 → a794 + {u'Ʂ', u'ʂ'}, // a7c5 → 0282 + {u'Ᶎ', u'ᶎ'}, // a7c6 → 1d8e + {u'Ａ', u'ａ'}, // ff21 → ff41 + {u'Ｂ', u'ｂ'}, // ff22 → ff42 + {u'Ｃ', u'ｃ'}, // ff23 → ff43 + {u'Ｄ', u'ｄ'}, // ff24 → ff44 + {u'Ｅ', u'ｅ'}, // ff25 → ff45 + {u'Ｆ', u'ｆ'}, // ff26 → ff46 + {u'Ｇ', u'ｇ'}, // ff27 → ff47 + {u'Ｈ', u'ｈ'}, // ff28 → ff48 + {u'Ｉ', u'ｉ'}, // ff29 → ff49 + {u'Ｊ', u'ｊ'}, // ff2a → ff4a + {u'Ｋ', u'ｋ'}, // ff2b → ff4b + {u'Ｌ', u'ｌ'}, // ff2c → ff4c + {u'Ｍ', u'ｍ'}, // ff2d → ff4d + {u'Ｎ', u'ｎ'}, // ff2e → ff4e + {u'Ｏ', u'ｏ'}, // ff2f → ff4f + {u'Ｐ', u'ｐ'}, // ff30 → ff50 + {u'Ｑ', u'ｑ'}, // ff31 → ff51 + {u'Ｒ', u'ｒ'}, // ff32 → ff52 + {u'Ｓ', u'ｓ'}, // ff33 → ff53 + {u'Ｔ', u'ｔ'}, // ff34 → ff54 + {u'Ｕ', u'ｕ'}, // ff35 → ff55 + {u'Ｖ', u'ｖ'}, // ff36 → ff56 + {u'Ｗ', u'ｗ'}, // ff37 → ff57 + {u'Ｘ', u'ｘ'}, // ff38 → ff58 + {u'Ｙ', u'ｙ'}, // ff39 → ff59 + {u'Ｚ', u'ｚ'}, // ff3a → ff5a +#endif +}; + +static const char32_t kUpperToLowerAstral[][2] = { + {L'ℂ', L'𝕔'}, // 00002102 → 0001d554 + {L'ℋ', L'𝒽'}, // 0000210b → 0001d4bd + {L'ℍ', L'𝕙'}, // 0000210d → 0001d559 + {L'ℐ', L'𝒾'}, // 00002110 → 0001d4be + {L'ℕ', L'𝕟'}, // 00002115 → 0001d55f + {L'ℚ', L'𝕢'}, // 0000211a → 0001d562 + {L'ℛ', L'𝓇'}, // 0000211b → 0001d4c7 + {L'ℝ', L'𝕣'}, // 0000211d → 0001d563 + {L'ℤ', L'𝕫'}, // 00002124 → 0001d56b + {L'ℬ', L'𝒷'}, // 0000212c → 0001d4b7 + {L'ℱ', L'𝒻'}, // 00002131 → 0001d4bb + {L'ℳ', L'𝓂'}, // 00002133 → 0001d4c2 + {L'𐐀', L'𐐨'}, // 00010400 → 00010428 + {L'𐐁', L'𐐩'}, // 00010401 → 00010429 + {L'𐐂', L'𐐪'}, // 00010402 → 0001042a + {L'𐐃', L'𐐫'}, // 00010403 → 0001042b + {L'𐐄', L'𐐬'}, // 00010404 → 0001042c + {L'𐐅', L'𐐭'}, // 00010405 → 0001042d + {L'𐐆', L'𐐮'}, // 00010406 → 0001042e + {L'𐐇', L'𐐯'}, // 00010407 → 0001042f + {L'𐐈', L'𐐰'}, // 00010408 → 00010430 + {L'𐐉', L'𐐱'}, // 00010409 → 00010431 + {L'𐐊', L'𐐲'}, // 0001040a → 00010432 + {L'𐐋', L'𐐳'}, // 0001040b → 00010433 + {L'𐐌', L'𐐴'}, // 0001040c → 00010434 + {L'𐐍', L'𐐵'}, // 0001040d → 00010435 + {L'𐐎', L'𐐶'}, // 0001040e → 00010436 + {L'𐐏', L'𐐷'}, // 0001040f → 00010437 + {L'𐐐', L'𐐸'}, // 00010410 → 00010438 + {L'𐐑', L'𐐹'}, // 00010411 → 00010439 + {L'𐐒', L'𐐺'}, // 00010412 → 0001043a + {L'𐐓', L'𐐻'}, // 00010413 → 0001043b + {L'𐐔', L'𐐼'}, // 00010414 → 0001043c + {L'𐐕', L'𐐽'}, // 00010415 → 0001043d + {L'𐐖', L'𐐾'}, // 00010416 → 0001043e + {L'𐐗', L'𐐿'}, // 00010417 → 0001043f + {L'𐐘', L'𐑀'}, // 00010418 → 00010440 + {L'𐐙', L'𐑁'}, // 00010419 → 00010441 + {L'𐐚', L'𐑂'}, // 0001041a → 00010442 + {L'𐐛', L'𐑃'}, // 0001041b → 00010443 + {L'𐐜', L'𐑄'}, // 0001041c → 00010444 + {L'𐐝', L'𐑅'}, // 0001041d → 00010445 + {L'𐐞', L'𐑆'}, // 0001041e → 00010446 + {L'𐐟', L'𐑇'}, // 0001041f → 00010447 + {L'𐐠', L'𐑈'}, // 00010420 → 00010448 + {L'𐐡', L'𐑉'}, // 00010421 → 00010449 + {L'𐐢', L'𐑊'}, // 00010422 → 0001044a + {L'𐐣', L'𐑋'}, // 00010423 → 0001044b + {L'𐐤', L'𐑌'}, // 00010424 → 0001044c + {L'𐐥', L'𐑍'}, // 00010425 → 0001044d + {L'𐐦', L'𐑎'}, // 00010426 → 0001044e + {L'𐐧', L'𐑏'}, // 00010427 → 0001044f + {L'𐒰', L'𐓘'}, // 000104b0 → 000104d8 + {L'𐒱', L'𐓙'}, // 000104b1 → 000104d9 + {L'𐒲', L'𐓚'}, // 000104b2 → 000104da + {L'𐒳', L'𐓛'}, // 000104b3 → 000104db + {L'𐒴', L'𐓜'}, // 000104b4 → 000104dc + {L'𐒵', L'𐓝'}, // 000104b5 → 000104dd + {L'𐒶', L'𐓞'}, // 000104b6 → 000104de + {L'𐒷', L'𐓟'}, // 000104b7 → 000104df + {L'𐒸', L'𐓠'}, // 000104b8 → 000104e0 + {L'𐒹', L'𐓡'}, // 000104b9 → 000104e1 + {L'𐒺', L'𐓢'}, // 000104ba → 000104e2 + {L'𐒻', L'𐓣'}, // 000104bb → 000104e3 + {L'𐒼', L'𐓤'}, // 000104bc → 000104e4 + {L'𐒽', L'𐓥'}, // 000104bd → 000104e5 + {L'𐒾', L'𐓦'}, // 000104be → 000104e6 + {L'𐒿', L'𐓧'}, // 000104bf → 000104e7 + {L'𐓀', L'𐓨'}, // 000104c0 → 000104e8 + {L'𐓁', L'𐓩'}, // 000104c1 → 000104e9 + {L'𐓂', L'𐓪'}, // 000104c2 → 000104ea + {L'𐓃', L'𐓫'}, // 000104c3 → 000104eb + {L'𐓄', L'𐓬'}, // 000104c4 → 000104ec + {L'𐓅', L'𐓭'}, // 000104c5 → 000104ed + {L'𐓆', L'𐓮'}, // 000104c6 → 000104ee + {L'𐓇', L'𐓯'}, // 000104c7 → 000104ef + {L'𐓈', L'𐓰'}, // 000104c8 → 000104f0 + {L'𐓉', L'𐓱'}, // 000104c9 → 000104f1 + {L'𐓊', L'𐓲'}, // 000104ca → 000104f2 + {L'𐓋', L'𐓳'}, // 000104cb → 000104f3 + {L'𐓌', L'𐓴'}, // 000104cc → 000104f4 + {L'𐓍', L'𐓵'}, // 000104cd → 000104f5 + {L'𐓎', L'𐓶'}, // 000104ce → 000104f6 + {L'𐓏', L'𐓷'}, // 000104cf → 000104f7 + {L'𐓐', L'𐓸'}, // 000104d0 → 000104f8 + {L'𐓑', L'𐓹'}, // 000104d1 → 000104f9 + {L'𐓒', L'𐓺'}, // 000104d2 → 000104fa + {L'𐓓', L'𐓻'}, // 000104d3 → 000104fb + {L'𐲀', L'𐳀'}, // 00010c80 → 00010cc0 + {L'𐲁', L'𐳁'}, // 00010c81 → 00010cc1 + {L'𐲂', L'𐳂'}, // 00010c82 → 00010cc2 + {L'𐲃', L'𐳃'}, // 00010c83 → 00010cc3 + {L'𐲄', L'𐳄'}, // 00010c84 → 00010cc4 + {L'𐲅', L'𐳅'}, // 00010c85 → 00010cc5 + {L'𐲆', L'𐳆'}, // 00010c86 → 00010cc6 + {L'𐲇', L'𐳇'}, // 00010c87 → 00010cc7 + {L'𐲈', L'𐳈'}, // 00010c88 → 00010cc8 + {L'𐲉', L'𐳂'}, // 00010c89 → 00010cc2 + {L'𐲊', L'𐳊'}, // 00010c8a → 00010cca + {L'𐲋', L'𐳋'}, // 00010c8b → 00010ccb + {L'𐲌', L'𐳌'}, // 00010c8c → 00010ccc + {L'𐲍', L'𐳍'}, // 00010c8d → 00010ccd + {L'𐲎', L'𐳎'}, // 00010c8e → 00010cce + {L'𐲏', L'𐳏'}, // 00010c8f → 00010ccf + {L'𐲐', L'𐳐'}, // 00010c90 → 00010cd0 + {L'𐲑', L'𐳑'}, // 00010c91 → 00010cd1 + {L'𐲒', L'𐳒'}, // 00010c92 → 00010cd2 + {L'𐲓', L'𐳓'}, // 00010c93 → 00010cd3 + {L'𐲔', L'𐳔'}, // 00010c94 → 00010cd4 + {L'𐲕', L'𐳕'}, // 00010c95 → 00010cd5 + {L'𐲖', L'𐳖'}, // 00010c96 → 00010cd6 + {L'𐲗', L'𐳗'}, // 00010c97 → 00010cd7 + {L'𐲘', L'𐳘'}, // 00010c98 → 00010cd8 + {L'𐲙', L'𐳅'}, // 00010c99 → 00010cc5 + {L'𐲚', L'𐳚'}, // 00010c9a → 00010cda + {L'𐲛', L'𐳛'}, // 00010c9b → 00010cdb + {L'𐲜', L'𐳜'}, // 00010c9c → 00010cdc + {L'𐲝', L'𐳝'}, // 00010c9d → 00010cdd + {L'𐲞', L'𐳞'}, // 00010c9e → 00010cde + {L'𐲟', L'𐳟'}, // 00010c9f → 00010cdf + {L'𐲠', L'𐳠'}, // 00010ca0 → 00010ce0 + {L'𐲡', L'𐳡'}, // 00010ca1 → 00010ce1 + {L'𐲢', L'𐳢'}, // 00010ca2 → 00010ce2 + {L'𐲣', L'𐳣'}, // 00010ca3 → 00010ce3 + {L'𐲤', L'𐳤'}, // 00010ca4 → 00010ce4 + {L'𐲥', L'𐳥'}, // 00010ca5 → 00010ce5 + {L'𐲦', L'𐳦'}, // 00010ca6 → 00010ce6 + {L'𐲧', L'𐳧'}, // 00010ca7 → 00010ce7 + {L'𐲨', L'𐳨'}, // 00010ca8 → 00010ce8 + {L'𐲩', L'𐳩'}, // 00010ca9 → 00010ce9 + {L'𐲪', L'𐳕'}, // 00010caa → 00010cd5 + {L'𐲫', L'𐳫'}, // 00010cab → 00010ceb + {L'𐲬', L'𐳬'}, // 00010cac → 00010cec + {L'𐲭', L'𐳭'}, // 00010cad → 00010ced + {L'𐲮', L'𐳮'}, // 00010cae → 00010cee + {L'𐲯', L'𐳯'}, // 00010caf → 00010cef + {L'𐲰', L'𐳰'}, // 00010cb0 → 00010cf0 + {L'𐲱', L'𐳱'}, // 00010cb1 → 00010cf1 + {L'𐲲', L'𐳲'}, // 00010cb2 → 00010cf2 + {L'𑢠', L'𑣀'}, // 000118a0 → 000118c0 + {L'𑢡', L'𑣁'}, // 000118a1 → 000118c1 + {L'𑢢', L'𑣂'}, // 000118a2 → 000118c2 + {L'𑢣', L'𑣃'}, // 000118a3 → 000118c3 + {L'𑢤', L'𑣄'}, // 000118a4 → 000118c4 + {L'𑢥', L'𑣅'}, // 000118a5 → 000118c5 + {L'𑢦', L'𑣆'}, // 000118a6 → 000118c6 + {L'𑢧', L'𑣇'}, // 000118a7 → 000118c7 + {L'𑢨', L'𑣈'}, // 000118a8 → 000118c8 + {L'𑢩', L'𑣉'}, // 000118a9 → 000118c9 + {L'𑢪', L'𑣊'}, // 000118aa → 000118ca + {L'𑢫', L'𑣋'}, // 000118ab → 000118cb + {L'𑢬', L'𑣌'}, // 000118ac → 000118cc + {L'𑢭', L'𑣍'}, // 000118ad → 000118cd + {L'𑢮', L'𑣎'}, // 000118ae → 000118ce + {L'𑢯', L'𑣏'}, // 000118af → 000118cf + {L'𑢰', L'𑣐'}, // 000118b0 → 000118d0 + {L'𑢱', L'𑣑'}, // 000118b1 → 000118d1 + {L'𑢲', L'𑣒'}, // 000118b2 → 000118d2 + {L'𑢳', L'𑣓'}, // 000118b3 → 000118d3 + {L'𑢴', L'𑣔'}, // 000118b4 → 000118d4 + {L'𑢵', L'𑣕'}, // 000118b5 → 000118d5 + {L'𑢶', L'𑣖'}, // 000118b6 → 000118d6 + {L'𑢷', L'𑣗'}, // 000118b7 → 000118d7 + {L'𑢸', L'𑣘'}, // 000118b8 → 000118d8 + {L'𑢹', L'𑣙'}, // 000118b9 → 000118d9 + {L'𑢺', L'𑣚'}, // 000118ba → 000118da + {L'𑢻', L'𑣛'}, // 000118bb → 000118db + {L'𑢼', L'𑣜'}, // 000118bc → 000118dc + {L'𑢽', L'𑣝'}, // 000118bd → 000118dd + {L'𑢾', L'𑣞'}, // 000118be → 000118de + {L'𑢿', L'𑣟'}, // 000118bf → 000118df + {L'𖹀', L'𖹠'}, // 00016e40 → 00016e60 + {L'𖹁', L'𖹡'}, // 00016e41 → 00016e61 + {L'𖹂', L'𖹢'}, // 00016e42 → 00016e62 + {L'𖹃', L'𖹣'}, // 00016e43 → 00016e63 + {L'𖹄', L'𖹤'}, // 00016e44 → 00016e64 + {L'𖹅', L'𖹥'}, // 00016e45 → 00016e65 + {L'𖹆', L'𖹦'}, // 00016e46 → 00016e66 + {L'𖹇', L'𖹧'}, // 00016e47 → 00016e67 + {L'𖹈', L'𖹨'}, // 00016e48 → 00016e68 + {L'𖹉', L'𖹩'}, // 00016e49 → 00016e69 + {L'𖹊', L'𖹪'}, // 00016e4a → 00016e6a + {L'𖹋', L'𖹫'}, // 00016e4b → 00016e6b + {L'𖹌', L'𖹬'}, // 00016e4c → 00016e6c + {L'𖹍', L'𖹭'}, // 00016e4d → 00016e6d + {L'𖹎', L'𖹮'}, // 00016e4e → 00016e6e + {L'𖹏', L'𖹯'}, // 00016e4f → 00016e6f + {L'𖹐', L'𖹰'}, // 00016e50 → 00016e70 + {L'𖹑', L'𖹱'}, // 00016e51 → 00016e71 + {L'𖹒', L'𖹲'}, // 00016e52 → 00016e72 + {L'𖹓', L'𖹳'}, // 00016e53 → 00016e73 + {L'𖹔', L'𖹴'}, // 00016e54 → 00016e74 + {L'𖹕', L'𖹵'}, // 00016e55 → 00016e75 + {L'𖹖', L'𖹶'}, // 00016e56 → 00016e76 + {L'𖹗', L'𖹷'}, // 00016e57 → 00016e77 + {L'𖹘', L'𖹸'}, // 00016e58 → 00016e78 + {L'𖹙', L'𖹹'}, // 00016e59 → 00016e79 + {L'𖹚', L'𖹺'}, // 00016e5a → 00016e7a + {L'𖹛', L'𖹻'}, // 00016e5b → 00016e7b + {L'𖹜', L'𖹼'}, // 00016e5c → 00016e7c + {L'𖹝', L'𖹽'}, // 00016e5d → 00016e7d + {L'𖹞', L'𖹾'}, // 00016e5e → 00016e7e + {L'𖹟', L'𖹿'}, // 00016e5f → 00016e7f + {L'𝐀', L'𝐚'}, // 0001d400 → 0001d41a + {L'𝐁', L'𝐛'}, // 0001d401 → 0001d41b + {L'𝐂', L'𝐜'}, // 0001d402 → 0001d41c + {L'𝐃', L'𝐝'}, // 0001d403 → 0001d41d + {L'𝐄', L'𝐞'}, // 0001d404 → 0001d41e + {L'𝐅', L'𝐟'}, // 0001d405 → 0001d41f + {L'𝐆', L'𝐠'}, // 0001d406 → 0001d420 + {L'𝐇', L'𝐡'}, // 0001d407 → 0001d421 + {L'𝐈', L'𝐢'}, // 0001d408 → 0001d422 + {L'𝐉', L'𝐣'}, // 0001d409 → 0001d423 + {L'𝐊', L'𝐤'}, // 0001d40a → 0001d424 + {L'𝐋', L'𝐥'}, // 0001d40b → 0001d425 + {L'𝐌', L'𝐦'}, // 0001d40c → 0001d426 + {L'𝐍', L'𝐧'}, // 0001d40d → 0001d427 + {L'𝐎', L'𝐨'}, // 0001d40e → 0001d428 + {L'𝐏', L'𝐩'}, // 0001d40f → 0001d429 + {L'𝐐', L'𝐪'}, // 0001d410 → 0001d42a + {L'𝐑', L'𝐫'}, // 0001d411 → 0001d42b + {L'𝐒', L'𝐬'}, // 0001d412 → 0001d42c + {L'𝐓', L'𝐭'}, // 0001d413 → 0001d42d + {L'𝐔', L'𝐮'}, // 0001d414 → 0001d42e + {L'𝐕', L'𝐯'}, // 0001d415 → 0001d42f + {L'𝐖', L'𝐰'}, // 0001d416 → 0001d430 + {L'𝐗', L'𝐱'}, // 0001d417 → 0001d431 + {L'𝐘', L'𝐲'}, // 0001d418 → 0001d432 + {L'𝐙', L'𝐳'}, // 0001d419 → 0001d433 + {L'𝐴', L'𝑎'}, // 0001d434 → 0001d44e + {L'𝐵', L'𝑏'}, // 0001d435 → 0001d44f + {L'𝐶', L'𝑐'}, // 0001d436 → 0001d450 + {L'𝐷', L'𝑑'}, // 0001d437 → 0001d451 + {L'𝐸', L'𝑒'}, // 0001d438 → 0001d452 + {L'𝐹', L'𝑓'}, // 0001d439 → 0001d453 + {L'𝐺', L'𝑔'}, // 0001d43a → 0001d454 + {L'𝐼', L'𝑖'}, // 0001d43c → 0001d456 + {L'𝐽', L'𝑗'}, // 0001d43d → 0001d457 + {L'𝐾', L'𝑘'}, // 0001d43e → 0001d458 + {L'𝐿', L'𝑙'}, // 0001d43f → 0001d459 + {L'𝑀', L'𝑚'}, // 0001d440 → 0001d45a + {L'𝑁', L'𝑛'}, // 0001d441 → 0001d45b + {L'𝑂', L'𝑜'}, // 0001d442 → 0001d45c + {L'𝑃', L'𝑝'}, // 0001d443 → 0001d45d + {L'𝑄', L'𝑞'}, // 0001d444 → 0001d45e + {L'𝑅', L'𝑟'}, // 0001d445 → 0001d45f + {L'𝑆', L'𝑠'}, // 0001d446 → 0001d460 + {L'𝑇', L'𝑡'}, // 0001d447 → 0001d461 + {L'𝑈', L'𝑢'}, // 0001d448 → 0001d462 + {L'𝑉', L'𝑣'}, // 0001d449 → 0001d463 + {L'𝑊', L'𝑤'}, // 0001d44a → 0001d464 + {L'𝑋', L'𝑥'}, // 0001d44b → 0001d465 + {L'𝑌', L'𝑦'}, // 0001d44c → 0001d466 + {L'𝑍', L'𝑧'}, // 0001d44d → 0001d467 + {L'𝑨', L'𝒂'}, // 0001d468 → 0001d482 + {L'𝑩', L'𝒃'}, // 0001d469 → 0001d483 + {L'𝑪', L'𝒄'}, // 0001d46a → 0001d484 + {L'𝑫', L'𝒅'}, // 0001d46b → 0001d485 + {L'𝑬', L'𝒆'}, // 0001d46c → 0001d486 + {L'𝑭', L'𝒇'}, // 0001d46d → 0001d487 + {L'𝑮', L'𝒈'}, // 0001d46e → 0001d488 + {L'𝑯', L'𝒉'}, // 0001d46f → 0001d489 + {L'𝑰', L'𝒊'}, // 0001d470 → 0001d48a + {L'𝑱', L'𝒋'}, // 0001d471 → 0001d48b + {L'𝑲', L'𝒌'}, // 0001d472 → 0001d48c + {L'𝑳', L'𝒍'}, // 0001d473 → 0001d48d + {L'𝑴', L'𝒎'}, // 0001d474 → 0001d48e + {L'𝑵', L'𝒏'}, // 0001d475 → 0001d48f + {L'𝑶', L'𝒐'}, // 0001d476 → 0001d490 + {L'𝑷', L'𝒑'}, // 0001d477 → 0001d491 + {L'𝑸', L'𝒒'}, // 0001d478 → 0001d492 + {L'𝑹', L'𝒓'}, // 0001d479 → 0001d493 + {L'𝑺', L'𝒔'}, // 0001d47a → 0001d494 + {L'𝑻', L'𝒕'}, // 0001d47b → 0001d495 + {L'𝑼', L'𝒖'}, // 0001d47c → 0001d496 + {L'𝑽', L'𝒗'}, // 0001d47d → 0001d497 + {L'𝑾', L'𝒘'}, // 0001d47e → 0001d498 + {L'𝑿', L'𝒙'}, // 0001d47f → 0001d499 + {L'𝒀', L'𝒚'}, // 0001d480 → 0001d49a + {L'𝒁', L'𝒛'}, // 0001d481 → 0001d49b + {L'𝒜', L'𝒶'}, // 0001d49c → 0001d4b6 + {L'𝒞', L'𝒸'}, // 0001d49e → 0001d4b8 + {L'𝒟', L'𝒹'}, // 0001d49f → 0001d4b9 + {L'𝒥', L'𝒿'}, // 0001d4a5 → 0001d4bf + {L'𝒦', L'𝓀'}, // 0001d4a6 → 0001d4c0 + {L'𝒩', L'𝓃'}, // 0001d4a9 → 0001d4c3 + {L'𝒫', L'𝓅'}, // 0001d4ab → 0001d4c5 + {L'𝒬', L'𝓆'}, // 0001d4ac → 0001d4c6 + {L'𝒮', L'𝓈'}, // 0001d4ae → 0001d4c8 + {L'𝒯', L'𝓉'}, // 0001d4af → 0001d4c9 + {L'𝒰', L'𝓊'}, // 0001d4b0 → 0001d4ca + {L'𝒱', L'𝓋'}, // 0001d4b1 → 0001d4cb + {L'𝒲', L'𝓌'}, // 0001d4b2 → 0001d4cc + {L'𝒳', L'𝓍'}, // 0001d4b3 → 0001d4cd + {L'𝒴', L'𝓎'}, // 0001d4b4 → 0001d4ce + {L'𝒵', L'𝓏'}, // 0001d4b5 → 0001d4cf + {L'𝓐', L'𝓪'}, // 0001d4d0 → 0001d4ea + {L'𝓑', L'𝓫'}, // 0001d4d1 → 0001d4eb + {L'𝓒', L'𝓬'}, // 0001d4d2 → 0001d4ec + {L'𝓓', L'𝓭'}, // 0001d4d3 → 0001d4ed + {L'𝓔', L'𝓮'}, // 0001d4d4 → 0001d4ee + {L'𝓕', L'𝓯'}, // 0001d4d5 → 0001d4ef + {L'𝓖', L'𝓰'}, // 0001d4d6 → 0001d4f0 + {L'𝓗', L'𝓱'}, // 0001d4d7 → 0001d4f1 + {L'𝓘', L'𝓲'}, // 0001d4d8 → 0001d4f2 + {L'𝓙', L'𝓳'}, // 0001d4d9 → 0001d4f3 + {L'𝓚', L'𝓴'}, // 0001d4da → 0001d4f4 + {L'𝓛', L'𝓵'}, // 0001d4db → 0001d4f5 + {L'𝓜', L'𝓶'}, // 0001d4dc → 0001d4f6 + {L'𝓝', L'𝓷'}, // 0001d4dd → 0001d4f7 + {L'𝓞', L'𝓸'}, // 0001d4de → 0001d4f8 + {L'𝓟', L'𝓹'}, // 0001d4df → 0001d4f9 + {L'𝓠', L'𝓺'}, // 0001d4e0 → 0001d4fa + {L'𝓡', L'𝓻'}, // 0001d4e1 → 0001d4fb + {L'𝓢', L'𝓼'}, // 0001d4e2 → 0001d4fc + {L'𝓣', L'𝓽'}, // 0001d4e3 → 0001d4fd + {L'𝓤', L'𝓾'}, // 0001d4e4 → 0001d4fe + {L'𝓥', L'𝓿'}, // 0001d4e5 → 0001d4ff + {L'𝓦', L'𝔀'}, // 0001d4e6 → 0001d500 + {L'𝓧', L'𝔁'}, // 0001d4e7 → 0001d501 + {L'𝓨', L'𝔂'}, // 0001d4e8 → 0001d502 + {L'𝓩', L'𝔃'}, // 0001d4e9 → 0001d503 + {L'𝔄', L'𝔞'}, // 0001d504 → 0001d51e + {L'𝔅', L'𝔟'}, // 0001d505 → 0001d51f + {L'𝔇', L'𝔡'}, // 0001d507 → 0001d521 + {L'𝔈', L'𝔢'}, // 0001d508 → 0001d522 + {L'𝔉', L'𝔣'}, // 0001d509 → 0001d523 + {L'𝔊', L'𝔤'}, // 0001d50a → 0001d524 + {L'𝔍', L'𝔧'}, // 0001d50d → 0001d527 + {L'𝔎', L'𝔨'}, // 0001d50e → 0001d528 + {L'𝔏', L'𝔩'}, // 0001d50f → 0001d529 + {L'𝔐', L'𝔪'}, // 0001d510 → 0001d52a + {L'𝔑', L'𝔫'}, // 0001d511 → 0001d52b + {L'𝔒', L'𝔬'}, // 0001d512 → 0001d52c + {L'𝔓', L'𝔭'}, // 0001d513 → 0001d52d + {L'𝔔', L'𝔮'}, // 0001d514 → 0001d52e + {L'𝔖', L'𝔰'}, // 0001d516 → 0001d530 + {L'𝔗', L'𝔱'}, // 0001d517 → 0001d531 + {L'𝔘', L'𝔲'}, // 0001d518 → 0001d532 + {L'𝔙', L'𝔳'}, // 0001d519 → 0001d533 + {L'𝔚', L'𝔴'}, // 0001d51a → 0001d534 + {L'𝔛', L'𝔵'}, // 0001d51b → 0001d535 + {L'𝔜', L'𝔶'}, // 0001d51c → 0001d536 + {L'𝔸', L'𝕒'}, // 0001d538 → 0001d552 + {L'𝔹', L'𝕓'}, // 0001d539 → 0001d553 + {L'𝔻', L'𝕕'}, // 0001d53b → 0001d555 + {L'𝔼', L'𝕖'}, // 0001d53c → 0001d556 + {L'𝔽', L'𝕗'}, // 0001d53d → 0001d557 + {L'𝕀', L'𝕚'}, // 0001d540 → 0001d55a + {L'𝕁', L'𝕛'}, // 0001d541 → 0001d55b + {L'𝕂', L'𝕜'}, // 0001d542 → 0001d55c + {L'𝕃', L'𝕝'}, // 0001d543 → 0001d55d + {L'𝕄', L'𝕞'}, // 0001d544 → 0001d55e + {L'𝕆', L'𝕠'}, // 0001d546 → 0001d560 + {L'𝕊', L'𝕤'}, // 0001d54a → 0001d564 + {L'𝕋', L'𝕥'}, // 0001d54b → 0001d565 + {L'𝕌', L'𝕦'}, // 0001d54c → 0001d566 + {L'𝕍', L'𝕧'}, // 0001d54d → 0001d567 + {L'𝕎', L'𝕨'}, // 0001d54e → 0001d568 + {L'𝕏', L'𝕩'}, // 0001d54f → 0001d569 + {L'𝕐', L'𝕪'}, // 0001d550 → 0001d56a + {L'𝕬', L'𝖆'}, // 0001d56c → 0001d586 + {L'𝕭', L'𝖇'}, // 0001d56d → 0001d587 + {L'𝕮', L'𝖈'}, // 0001d56e → 0001d588 + {L'𝕯', L'𝖉'}, // 0001d56f → 0001d589 + {L'𝕰', L'𝖊'}, // 0001d570 → 0001d58a + {L'𝕱', L'𝖋'}, // 0001d571 → 0001d58b + {L'𝕲', L'𝖌'}, // 0001d572 → 0001d58c + {L'𝕳', L'𝖍'}, // 0001d573 → 0001d58d + {L'𝕴', L'𝖎'}, // 0001d574 → 0001d58e + {L'𝕵', L'𝖏'}, // 0001d575 → 0001d58f + {L'𝕶', L'𝖐'}, // 0001d576 → 0001d590 + {L'𝕷', L'𝖑'}, // 0001d577 → 0001d591 + {L'𝕸', L'𝖒'}, // 0001d578 → 0001d592 + {L'𝕹', L'𝖓'}, // 0001d579 → 0001d593 + {L'𝕺', L'𝖔'}, // 0001d57a → 0001d594 + {L'𝕻', L'𝖕'}, // 0001d57b → 0001d595 + {L'𝕼', L'𝖖'}, // 0001d57c → 0001d596 + {L'𝕽', L'𝖗'}, // 0001d57d → 0001d597 + {L'𝕾', L'𝖘'}, // 0001d57e → 0001d598 + {L'𝕿', L'𝖙'}, // 0001d57f → 0001d599 + {L'𝖀', L'𝖚'}, // 0001d580 → 0001d59a + {L'𝖁', L'𝖛'}, // 0001d581 → 0001d59b + {L'𝖂', L'𝖜'}, // 0001d582 → 0001d59c + {L'𝖃', L'𝖝'}, // 0001d583 → 0001d59d + {L'𝖄', L'𝖞'}, // 0001d584 → 0001d59e + {L'𝖅', L'𝖟'}, // 0001d585 → 0001d59f + {L'𝖠', L'𝖺'}, // 0001d5a0 → 0001d5ba + {L'𝖡', L'𝖻'}, // 0001d5a1 → 0001d5bb + {L'𝖢', L'𝖼'}, // 0001d5a2 → 0001d5bc + {L'𝖣', L'𝖽'}, // 0001d5a3 → 0001d5bd + {L'𝖤', L'𝖾'}, // 0001d5a4 → 0001d5be + {L'𝖥', L'𝖿'}, // 0001d5a5 → 0001d5bf + {L'𝖦', L'𝗀'}, // 0001d5a6 → 0001d5c0 + {L'𝖧', L'𝗁'}, // 0001d5a7 → 0001d5c1 + {L'𝖨', L'𝗂'}, // 0001d5a8 → 0001d5c2 + {L'𝖩', L'𝗃'}, // 0001d5a9 → 0001d5c3 + {L'𝖪', L'𝗄'}, // 0001d5aa → 0001d5c4 + {L'𝖫', L'𝗅'}, // 0001d5ab → 0001d5c5 + {L'𝖬', L'𝗆'}, // 0001d5ac → 0001d5c6 + {L'𝖭', L'𝗇'}, // 0001d5ad → 0001d5c7 + {L'𝖮', L'𝗈'}, // 0001d5ae → 0001d5c8 + {L'𝖯', L'𝗉'}, // 0001d5af → 0001d5c9 + {L'𝖰', L'𝗊'}, // 0001d5b0 → 0001d5ca + {L'𝖱', L'𝗋'}, // 0001d5b1 → 0001d5cb + {L'𝖲', L'𝗌'}, // 0001d5b2 → 0001d5cc + {L'𝖳', L'𝗍'}, // 0001d5b3 → 0001d5cd + {L'𝖴', L'𝗎'}, // 0001d5b4 → 0001d5ce + {L'𝖵', L'𝗏'}, // 0001d5b5 → 0001d5cf + {L'𝖶', L'𝗐'}, // 0001d5b6 → 0001d5d0 + {L'𝖷', L'𝗑'}, // 0001d5b7 → 0001d5d1 + {L'𝖸', L'𝗒'}, // 0001d5b8 → 0001d5d2 + {L'𝖹', L'𝗓'}, // 0001d5b9 → 0001d5d3 + {L'𝗔', L'𝗮'}, // 0001d5d4 → 0001d5ee + {L'𝗕', L'𝗯'}, // 0001d5d5 → 0001d5ef + {L'𝗖', L'𝗰'}, // 0001d5d6 → 0001d5f0 + {L'𝗗', L'𝗱'}, // 0001d5d7 → 0001d5f1 + {L'𝗘', L'𝗲'}, // 0001d5d8 → 0001d5f2 + {L'𝗙', L'𝗳'}, // 0001d5d9 → 0001d5f3 + {L'𝗚', L'𝗴'}, // 0001d5da → 0001d5f4 + {L'𝗛', L'𝗵'}, // 0001d5db → 0001d5f5 + {L'𝗜', L'𝗶'}, // 0001d5dc → 0001d5f6 + {L'𝗝', L'𝗷'}, // 0001d5dd → 0001d5f7 + {L'𝗞', L'𝗸'}, // 0001d5de → 0001d5f8 + {L'𝗟', L'𝗹'}, // 0001d5df → 0001d5f9 + {L'𝗠', L'𝗺'}, // 0001d5e0 → 0001d5fa + {L'𝗡', L'𝗻'}, // 0001d5e1 → 0001d5fb + {L'𝗢', L'𝗼'}, // 0001d5e2 → 0001d5fc + {L'𝗣', L'𝗽'}, // 0001d5e3 → 0001d5fd + {L'𝗤', L'𝗾'}, // 0001d5e4 → 0001d5fe + {L'𝗥', L'𝗿'}, // 0001d5e5 → 0001d5ff + {L'𝗦', L'𝘀'}, // 0001d5e6 → 0001d600 + {L'𝗧', L'𝘁'}, // 0001d5e7 → 0001d601 + {L'𝗨', L'𝘂'}, // 0001d5e8 → 0001d602 + {L'𝗩', L'𝘃'}, // 0001d5e9 → 0001d603 + {L'𝗪', L'𝘄'}, // 0001d5ea → 0001d604 + {L'𝗫', L'𝘅'}, // 0001d5eb → 0001d605 + {L'𝗬', L'𝘆'}, // 0001d5ec → 0001d606 + {L'𝗭', L'𝘇'}, // 0001d5ed → 0001d607 + {L'𝘈', L'𝘢'}, // 0001d608 → 0001d622 + {L'𝘉', L'𝘣'}, // 0001d609 → 0001d623 + {L'𝘊', L'𝘤'}, // 0001d60a → 0001d624 + {L'𝘋', L'𝘥'}, // 0001d60b → 0001d625 + {L'𝘌', L'𝘦'}, // 0001d60c → 0001d626 + {L'𝘍', L'𝘧'}, // 0001d60d → 0001d627 + {L'𝘎', L'𝘨'}, // 0001d60e → 0001d628 + {L'𝘏', L'𝘩'}, // 0001d60f → 0001d629 + {L'𝘐', L'𝘪'}, // 0001d610 → 0001d62a + {L'𝘑', L'𝘫'}, // 0001d611 → 0001d62b + {L'𝘒', L'𝘬'}, // 0001d612 → 0001d62c + {L'𝘓', L'𝘭'}, // 0001d613 → 0001d62d + {L'𝘔', L'𝘮'}, // 0001d614 → 0001d62e + {L'𝘕', L'𝘯'}, // 0001d615 → 0001d62f + {L'𝘖', L'𝘰'}, // 0001d616 → 0001d630 + {L'𝘗', L'𝘱'}, // 0001d617 → 0001d631 + {L'𝘘', L'𝘲'}, // 0001d618 → 0001d632 + {L'𝘙', L'𝘳'}, // 0001d619 → 0001d633 + {L'𝘚', L'𝘴'}, // 0001d61a → 0001d634 + {L'𝘛', L'𝘵'}, // 0001d61b → 0001d635 + {L'𝘜', L'𝘶'}, // 0001d61c → 0001d636 + {L'𝘝', L'𝘷'}, // 0001d61d → 0001d637 + {L'𝘞', L'𝘸'}, // 0001d61e → 0001d638 + {L'𝘟', L'𝘹'}, // 0001d61f → 0001d639 + {L'𝘠', L'𝘺'}, // 0001d620 → 0001d63a + {L'𝘡', L'𝘻'}, // 0001d621 → 0001d63b + {L'𝘼', L'𝒂'}, // 0001d63c → 0001d482 + {L'𝘽', L'𝒃'}, // 0001d63d → 0001d483 + {L'𝘾', L'𝒄'}, // 0001d63e → 0001d484 + {L'𝘿', L'𝒅'}, // 0001d63f → 0001d485 + {L'𝙀', L'𝒆'}, // 0001d640 → 0001d486 + {L'𝙁', L'𝒇'}, // 0001d641 → 0001d487 + {L'𝙂', L'𝒈'}, // 0001d642 → 0001d488 + {L'𝙃', L'𝒉'}, // 0001d643 → 0001d489 + {L'𝙄', L'𝒊'}, // 0001d644 → 0001d48a + {L'𝙅', L'𝒋'}, // 0001d645 → 0001d48b + {L'𝙆', L'𝒌'}, // 0001d646 → 0001d48c + {L'𝙇', L'𝒍'}, // 0001d647 → 0001d48d + {L'𝙈', L'𝒎'}, // 0001d648 → 0001d48e + {L'𝙉', L'𝒏'}, // 0001d649 → 0001d48f + {L'𝙊', L'𝒐'}, // 0001d64a → 0001d490 + {L'𝙋', L'𝒑'}, // 0001d64b → 0001d491 + {L'𝙌', L'𝒒'}, // 0001d64c → 0001d492 + {L'𝙍', L'𝒓'}, // 0001d64d → 0001d493 + {L'𝙎', L'𝒔'}, // 0001d64e → 0001d494 + {L'𝙏', L'𝒕'}, // 0001d64f → 0001d495 + {L'𝙐', L'𝒖'}, // 0001d650 → 0001d496 + {L'𝙑', L'𝒗'}, // 0001d651 → 0001d497 + {L'𝙒', L'𝒘'}, // 0001d652 → 0001d498 + {L'𝙓', L'𝒙'}, // 0001d653 → 0001d499 + {L'𝙔', L'𝒚'}, // 0001d654 → 0001d49a + {L'𝙕', L'𝒛'}, // 0001d655 → 0001d49b + {L'𝙰', L'𝚊'}, // 0001d670 → 0001d68a + {L'𝙱', L'𝚋'}, // 0001d671 → 0001d68b + {L'𝙲', L'𝚌'}, // 0001d672 → 0001d68c + {L'𝙳', L'𝚍'}, // 0001d673 → 0001d68d + {L'𝙴', L'𝚎'}, // 0001d674 → 0001d68e + {L'𝙵', L'𝚏'}, // 0001d675 → 0001d68f + {L'𝙶', L'𝚐'}, // 0001d676 → 0001d690 + {L'𝙷', L'𝚑'}, // 0001d677 → 0001d691 + {L'𝙸', L'𝚒'}, // 0001d678 → 0001d692 + {L'𝙹', L'𝚓'}, // 0001d679 → 0001d693 + {L'𝙺', L'𝚔'}, // 0001d67a → 0001d694 + {L'𝙻', L'𝚕'}, // 0001d67b → 0001d695 + {L'𝙼', L'𝚖'}, // 0001d67c → 0001d696 + {L'𝙽', L'𝚗'}, // 0001d67d → 0001d697 + {L'𝙾', L'𝚘'}, // 0001d67e → 0001d698 + {L'𝙿', L'𝚙'}, // 0001d67f → 0001d699 + {L'𝚀', L'𝚚'}, // 0001d680 → 0001d69a + {L'𝚁', L'𝚛'}, // 0001d681 → 0001d69b + {L'𝚂', L'𝚜'}, // 0001d682 → 0001d69c + {L'𝚃', L'𝚝'}, // 0001d683 → 0001d69d + {L'𝚄', L'𝚞'}, // 0001d684 → 0001d69e + {L'𝚅', L'𝚟'}, // 0001d685 → 0001d69f + {L'𝚆', L'𝚠'}, // 0001d686 → 0001d6a0 + {L'𝚇', L'𝚡'}, // 0001d687 → 0001d6a1 + {L'𝚈', L'𝚢'}, // 0001d688 → 0001d6a2 + {L'𝚉', L'𝚣'}, // 0001d689 → 0001d6a3 + {L'𝚨', L'𝛂'}, // 0001d6a8 → 0001d6c2 + {L'𝚩', L'𝛃'}, // 0001d6a9 → 0001d6c3 + {L'𝚪', L'𝛄'}, // 0001d6aa → 0001d6c4 + {L'𝚫', L'𝛅'}, // 0001d6ab → 0001d6c5 + {L'𝚬', L'𝛆'}, // 0001d6ac → 0001d6c6 + {L'𝚭', L'𝛇'}, // 0001d6ad → 0001d6c7 + {L'𝚮', L'𝛈'}, // 0001d6ae → 0001d6c8 + {L'𝚯', L'𝛉'}, // 0001d6af → 0001d6c9 + {L'𝚰', L'𝛊'}, // 0001d6b0 → 0001d6ca + {L'𝚱', L'𝛋'}, // 0001d6b1 → 0001d6cb + {L'𝚲', L'𝛌'}, // 0001d6b2 → 0001d6cc + {L'𝚳', L'𝛍'}, // 0001d6b3 → 0001d6cd + {L'𝚴', L'𝛎'}, // 0001d6b4 → 0001d6ce + {L'𝚵', L'𝛏'}, // 0001d6b5 → 0001d6cf + {L'𝚶', L'𝛐'}, // 0001d6b6 → 0001d6d0 + {L'𝚷', L'𝛑'}, // 0001d6b7 → 0001d6d1 + {L'𝚸', L'𝛒'}, // 0001d6b8 → 0001d6d2 + {L'𝚺', L'𝛔'}, // 0001d6ba → 0001d6d4 + {L'𝚻', L'𝛕'}, // 0001d6bb → 0001d6d5 + {L'𝚼', L'𝛖'}, // 0001d6bc → 0001d6d6 + {L'𝚽', L'𝛗'}, // 0001d6bd → 0001d6d7 + {L'𝚾', L'𝛘'}, // 0001d6be → 0001d6d8 + {L'𝚿', L'𝛙'}, // 0001d6bf → 0001d6d9 + {L'𝛀', L'𝛚'}, // 0001d6c0 → 0001d6da + {L'𝛢', L'𝛼'}, // 0001d6e2 → 0001d6fc + {L'𝛣', L'𝛽'}, // 0001d6e3 → 0001d6fd + {L'𝛤', L'𝛾'}, // 0001d6e4 → 0001d6fe + {L'𝛥', L'𝛿'}, // 0001d6e5 → 0001d6ff + {L'𝛦', L'𝜀'}, // 0001d6e6 → 0001d700 + {L'𝛧', L'𝜁'}, // 0001d6e7 → 0001d701 + {L'𝛨', L'𝜂'}, // 0001d6e8 → 0001d702 + {L'𝛩', L'𝜃'}, // 0001d6e9 → 0001d703 + {L'𝛪', L'𝜄'}, // 0001d6ea → 0001d704 + {L'𝛫', L'𝜅'}, // 0001d6eb → 0001d705 + {L'𝛬', L'𝜆'}, // 0001d6ec → 0001d706 + {L'𝛭', L'𝜇'}, // 0001d6ed → 0001d707 + {L'𝛮', L'𝜈'}, // 0001d6ee → 0001d708 + {L'𝛯', L'𝜉'}, // 0001d6ef → 0001d709 + {L'𝛰', L'𝜊'}, // 0001d6f0 → 0001d70a + {L'𝛱', L'𝜋'}, // 0001d6f1 → 0001d70b + {L'𝛲', L'𝜌'}, // 0001d6f2 → 0001d70c + {L'𝛳', L'𝜗'}, // 0001d6f3 → 0001d717 + {L'𝛴', L'𝜎'}, // 0001d6f4 → 0001d70e + {L'𝛵', L'𝜏'}, // 0001d6f5 → 0001d70f + {L'𝛶', L'𝜐'}, // 0001d6f6 → 0001d710 + {L'𝛷', L'𝜑'}, // 0001d6f7 → 0001d711 + {L'𝛸', L'𝜒'}, // 0001d6f8 → 0001d712 + {L'𝛹', L'𝜓'}, // 0001d6f9 → 0001d713 + {L'𝛺', L'𝜔'}, // 0001d6fa → 0001d714 + {L'𝜜', L'𝜶'}, // 0001d71c → 0001d736 + {L'𝜝', L'𝜷'}, // 0001d71d → 0001d737 + {L'𝜞', L'𝜸'}, // 0001d71e → 0001d738 + {L'𝜟', L'𝜹'}, // 0001d71f → 0001d739 + {L'𝜠', L'𝜺'}, // 0001d720 → 0001d73a + {L'𝜡', L'𝜻'}, // 0001d721 → 0001d73b + {L'𝜢', L'𝜼'}, // 0001d722 → 0001d73c + {L'𝜣', L'𝜽'}, // 0001d723 → 0001d73d + {L'𝜤', L'𝜾'}, // 0001d724 → 0001d73e + {L'𝜥', L'𝜿'}, // 0001d725 → 0001d73f + {L'𝜦', L'𝝀'}, // 0001d726 → 0001d740 + {L'𝜧', L'𝝁'}, // 0001d727 → 0001d741 + {L'𝜨', L'𝝂'}, // 0001d728 → 0001d742 + {L'𝜩', L'𝝃'}, // 0001d729 → 0001d743 + {L'𝜪', L'𝝄'}, // 0001d72a → 0001d744 + {L'𝜫', L'𝝅'}, // 0001d72b → 0001d745 + {L'𝜬', L'𝝆'}, // 0001d72c → 0001d746 + {L'𝜭', L'𝝑'}, // 0001d72d → 0001d751 + {L'𝜮', L'𝝈'}, // 0001d72e → 0001d748 + {L'𝜯', L'𝝉'}, // 0001d72f → 0001d749 + {L'𝜰', L'𝝊'}, // 0001d730 → 0001d74a + {L'𝜱', L'𝝋'}, // 0001d731 → 0001d74b + {L'𝜲', L'𝝌'}, // 0001d732 → 0001d74c + {L'𝜳', L'𝝍'}, // 0001d733 → 0001d74d + {L'𝜴', L'𝝎'}, // 0001d734 → 0001d74e + {L'𝝖', L'𝝰'}, // 0001d756 → 0001d770 + {L'𝝗', L'𝝱'}, // 0001d757 → 0001d771 + {L'𝝘', L'𝝲'}, // 0001d758 → 0001d772 + {L'𝝙', L'𝝳'}, // 0001d759 → 0001d773 + {L'𝝚', L'𝝴'}, // 0001d75a → 0001d774 + {L'𝝛', L'𝝵'}, // 0001d75b → 0001d775 + {L'𝝜', L'𝝶'}, // 0001d75c → 0001d776 + {L'𝝝', L'𝝷'}, // 0001d75d → 0001d777 + {L'𝝞', L'𝝸'}, // 0001d75e → 0001d778 + {L'𝝟', L'𝝹'}, // 0001d75f → 0001d779 + {L'𝝠', L'𝝺'}, // 0001d760 → 0001d77a + {L'𝝡', L'𝝻'}, // 0001d761 → 0001d77b + {L'𝝢', L'𝝼'}, // 0001d762 → 0001d77c + {L'𝝣', L'𝝽'}, // 0001d763 → 0001d77d + {L'𝝤', L'𝝾'}, // 0001d764 → 0001d77e + {L'𝝥', L'𝝿'}, // 0001d765 → 0001d77f + {L'𝝦', L'𝞀'}, // 0001d766 → 0001d780 + {L'𝝧', L'𝟅'}, // 0001d767 → 0001d7c5 + {L'𝝨', L'𝞂'}, // 0001d768 → 0001d782 + {L'𝝩', L'𝞃'}, // 0001d769 → 0001d783 + {L'𝝪', L'𝞄'}, // 0001d76a → 0001d784 + {L'𝝫', L'𝞅'}, // 0001d76b → 0001d785 + {L'𝝬', L'𝞆'}, // 0001d76c → 0001d786 + {L'𝝭', L'𝞇'}, // 0001d76d → 0001d787 + {L'𝝮', L'𝞈'}, // 0001d76e → 0001d788 + {L'𝞐', L'𝜶'}, // 0001d790 → 0001d736 + {L'𝞑', L'𝜷'}, // 0001d791 → 0001d737 + {L'𝞒', L'𝜸'}, // 0001d792 → 0001d738 + {L'𝞓', L'𝜹'}, // 0001d793 → 0001d739 + {L'𝞔', L'𝜺'}, // 0001d794 → 0001d73a + {L'𝞕', L'𝜻'}, // 0001d795 → 0001d73b + {L'𝞖', L'𝜼'}, // 0001d796 → 0001d73c + {L'𝞗', L'𝜽'}, // 0001d797 → 0001d73d + {L'𝞘', L'𝜾'}, // 0001d798 → 0001d73e + {L'𝞙', L'𝜿'}, // 0001d799 → 0001d73f + {L'𝞚', L'𝝀'}, // 0001d79a → 0001d740 + {L'𝞛', L'𝝁'}, // 0001d79b → 0001d741 + {L'𝞜', L'𝝂'}, // 0001d79c → 0001d742 + {L'𝞝', L'𝝃'}, // 0001d79d → 0001d743 + {L'𝞞', L'𝝄'}, // 0001d79e → 0001d744 + {L'𝞟', L'𝝅'}, // 0001d79f → 0001d745 + {L'𝞠', L'𝝆'}, // 0001d7a0 → 0001d746 + {L'𝞡', L'𝟅'}, // 0001d7a1 → 0001d7c5 + {L'𝞢', L'𝝈'}, // 0001d7a2 → 0001d748 + {L'𝞣', L'𝝉'}, // 0001d7a3 → 0001d749 + {L'𝞤', L'𝝊'}, // 0001d7a4 → 0001d74a + {L'𝞥', L'𝝋'}, // 0001d7a5 → 0001d74b + {L'𝞦', L'𝝌'}, // 0001d7a6 → 0001d74c + {L'𝞧', L'𝝍'}, // 0001d7a7 → 0001d74d + {L'𝞨', L'𝝎'}, // 0001d7a8 → 0001d74e + {L'𝟊', L'𝟋'}, // 0001d7ca → 0001d7cb + {L'𞤀', L'𞤢'}, // 0001e900 → 0001e922 + {L'𞤁', L'𞤣'}, // 0001e901 → 0001e923 + {L'𞤂', L'𞤤'}, // 0001e902 → 0001e924 + {L'𞤃', L'𞤥'}, // 0001e903 → 0001e925 + {L'𞤄', L'𞤦'}, // 0001e904 → 0001e926 + {L'𞤅', L'𞤧'}, // 0001e905 → 0001e927 + {L'𞤆', L'𞤨'}, // 0001e906 → 0001e928 + {L'𞤇', L'𞤩'}, // 0001e907 → 0001e929 + {L'𞤈', L'𞤪'}, // 0001e908 → 0001e92a + {L'𞤉', L'𞤫'}, // 0001e909 → 0001e92b + {L'𞤊', L'𞤬'}, // 0001e90a → 0001e92c + {L'𞤋', L'𞤭'}, // 0001e90b → 0001e92d + {L'𞤌', L'𞤮'}, // 0001e90c → 0001e92e + {L'𞤍', L'𞤯'}, // 0001e90d → 0001e92f + {L'𞤎', L'𞤰'}, // 0001e90e → 0001e930 + {L'𞤏', L'𞤱'}, // 0001e90f → 0001e931 + {L'𞤐', L'𞤲'}, // 0001e910 → 0001e932 + {L'𞤑', L'𞤳'}, // 0001e911 → 0001e933 + {L'𞤒', L'𞤴'}, // 0001e912 → 0001e934 + {L'𞤓', L'𞤵'}, // 0001e913 → 0001e935 + {L'𞤔', L'𞤶'}, // 0001e914 → 0001e936 + {L'𞤕', L'𞤷'}, // 0001e915 → 0001e937 + {L'𞤖', L'𞤸'}, // 0001e916 → 0001e938 + {L'𞤗', L'𞤹'}, // 0001e917 → 0001e939 + {L'𞤘', L'𞤺'}, // 0001e918 → 0001e93a + {L'𞤙', L'𞤻'}, // 0001e919 → 0001e93b + {L'𞤚', L'𞤼'}, // 0001e91a → 0001e93c + {L'𞤛', L'𞤽'}, // 0001e91b → 0001e93d + {L'𞤜', L'𞤾'}, // 0001e91c → 0001e93e + {L'𞤝', L'𞤿'}, // 0001e91d → 0001e93f + {L'𞤞', L'𞥀'}, // 0001e91e → 0001e940 + {L'𞤟', L'𞥁'}, // 0001e91f → 0001e941 + {L'𞤠', L'𞥂'}, // 0001e920 → 0001e942 + {L'𞤡', L'𞥃'}, // 0001e921 → 0001e943 +}; /** * Converts wide character to lower case. */ -wint_t towlower(wint_t wc) { - return tolower(wc); +wint_t towlower(wint_t c) { + int m, l, r; + if (isascii(c)) return tolower(c); + if (!IsTiny()) { + l = 0; + r = ARRAYLEN(kUpperToLowerAstral) - 1; + while (l <= r) { + m = (l + r) >> 1; + if (kUpperToLowerAstral[m][0] < c) { + l = m + 1; + } else if (kUpperToLowerAstral[m][0] > c) { + r = m - 1; + } else { + return kUpperToLowerAstral[m][1]; + } + } + } + l = 0; + r = ARRAYLEN(kUpperToLower) - 1; + while (l <= r) { + m = (l + r) >> 1; + if (kUpperToLower[m][0] < c) { + l = m + 1; + } else if (kUpperToLower[m][0] > c) { + r = m - 1; + } else { + return kUpperToLower[m][1]; + } + } + return c; } diff --git a/libc/str/towupper.c b/libc/str/towupper.c index 1418a9703..b6bab958e 100644 --- a/libc/str/towupper.c +++ b/libc/str/towupper.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,11 +16,1650 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/macros.internal.h" #include "libc/str/str.h" +#include "libc/dce.h" + +static const char16_t kLowerToUpper[][2] = { + {u'ß', u'ẞ'}, // 00df → 1e9e + {u'à', u'À'}, // 00e0 → 00c0 + {u'á', u'Á'}, // 00e1 → 00c1 + {u'â', u'Â'}, // 00e2 → 00c2 + {u'ã', u'Ã'}, // 00e3 → 00c3 + {u'ä', u'Ä'}, // 00e4 → 00c4 + {u'å', u'Å'}, // 00e5 → 00c5 + {u'æ', u'Æ'}, // 00e6 → 00c6 + {u'ç', u'Ç'}, // 00e7 → 00c7 + {u'è', u'È'}, // 00e8 → 00c8 + {u'é', u'É'}, // 00e9 → 00c9 + {u'ê', u'Ê'}, // 00ea → 00ca + {u'ë', u'Ë'}, // 00eb → 00cb + {u'ì', u'Ì'}, // 00ec → 00cc + {u'í', u'Í'}, // 00ed → 00cd + {u'î', u'Î'}, // 00ee → 00ce + {u'ï', u'Ï'}, // 00ef → 00cf + {u'ð', u'Ð'}, // 00f0 → 00d0 + {u'ñ', u'Ñ'}, // 00f1 → 00d1 + {u'ò', u'Ò'}, // 00f2 → 00d2 + {u'ó', u'Ó'}, // 00f3 → 00d3 + {u'ô', u'Ô'}, // 00f4 → 00d4 + {u'õ', u'Õ'}, // 00f5 → 00d5 + {u'ö', u'Ö'}, // 00f6 → 00d6 + {u'ø', u'Ø'}, // 00f8 → 00d8 + {u'ù', u'Ù'}, // 00f9 → 00d9 + {u'ú', u'Ú'}, // 00fa → 00da + {u'û', u'Û'}, // 00fb → 00db + {u'ü', u'Ü'}, // 00fc → 00dc + {u'ý', u'Ý'}, // 00fd → 00dd + {u'þ', u'Þ'}, // 00fe → 00de + {u'ÿ', u'Ÿ'}, // 00ff → 0178 +#ifndef TINY + {u'ā', u'Ā'}, // 0101 → 0100 + {u'ă', u'Ă'}, // 0103 → 0102 + {u'ą', u'Ą'}, // 0105 → 0104 + {u'ć', u'Ć'}, // 0107 → 0106 + {u'ĉ', u'Ĉ'}, // 0109 → 0108 + {u'ċ', u'Ċ'}, // 010b → 010a + {u'č', u'Č'}, // 010d → 010c + {u'ď', u'Ď'}, // 010f → 010e + {u'đ', u'Đ'}, // 0111 → 0110 + {u'ē', u'Ē'}, // 0113 → 0112 + {u'ĕ', u'Ĕ'}, // 0115 → 0114 + {u'ė', u'Ė'}, // 0117 → 0116 + {u'ę', u'Ę'}, // 0119 → 0118 + {u'ě', u'Ě'}, // 011b → 011a + {u'ĝ', u'Ĝ'}, // 011d → 011c + {u'ğ', u'Ğ'}, // 011f → 011e + {u'ġ', u'Ġ'}, // 0121 → 0120 + {u'ģ', u'Ģ'}, // 0123 → 0122 + {u'ĥ', u'Ĥ'}, // 0125 → 0124 + {u'ħ', u'Ħ'}, // 0127 → 0126 + {u'ĩ', u'Ĩ'}, // 0129 → 0128 + {u'ī', u'Ī'}, // 012b → 012a + {u'ĭ', u'Ĭ'}, // 012d → 012c + {u'į', u'Į'}, // 012f → 012e + {u'ij', u'IJ'}, // 0133 → 0132 + {u'ĵ', u'Ĵ'}, // 0135 → 0134 + {u'ķ', u'Ķ'}, // 0137 → 0136 + {u'ĺ', u'Ĺ'}, // 013a → 0139 + {u'ļ', u'Ļ'}, // 013c → 013b + {u'ľ', u'Ľ'}, // 013e → 013d + {u'ŀ', u'Ŀ'}, // 0140 → 013f + {u'ł', u'Ł'}, // 0142 → 0141 + {u'ń', u'Ń'}, // 0144 → 0143 + {u'ņ', u'Ņ'}, // 0146 → 0145 + {u'ň', u'Ň'}, // 0148 → 0147 + {u'ŋ', u'Ŋ'}, // 014b → 014a + {u'ō', u'Ō'}, // 014d → 014c + {u'ŏ', u'Ŏ'}, // 014f → 014e + {u'ő', u'Ő'}, // 0151 → 0150 + {u'œ', u'Œ'}, // 0153 → 0152 + {u'ŕ', u'Ŕ'}, // 0155 → 0154 + {u'ŗ', u'Ŗ'}, // 0157 → 0156 + {u'ř', u'Ř'}, // 0159 → 0158 + {u'ś', u'Ś'}, // 015b → 015a + {u'ŝ', u'Ŝ'}, // 015d → 015c + {u'ş', u'Ş'}, // 015f → 015e + {u'š', u'Š'}, // 0161 → 0160 + {u'ţ', u'Ţ'}, // 0163 → 0162 + {u'ť', u'Ť'}, // 0165 → 0164 + {u'ŧ', u'Ŧ'}, // 0167 → 0166 + {u'ũ', u'Ũ'}, // 0169 → 0168 + {u'ū', u'Ū'}, // 016b → 016a + {u'ŭ', u'Ŭ'}, // 016d → 016c + {u'ů', u'Ů'}, // 016f → 016e + {u'ű', u'Ű'}, // 0171 → 0170 + {u'ų', u'Ų'}, // 0173 → 0172 + {u'ŵ', u'Ŵ'}, // 0175 → 0174 + {u'ŷ', u'Ŷ'}, // 0177 → 0176 + {u'ź', u'Ź'}, // 017a → 0179 + {u'ż', u'Ż'}, // 017c → 017b + {u'ž', u'Ž'}, // 017e → 017d + {u'ƀ', u'Ƀ'}, // 0180 → 0243 + {u'ƃ', u'Ƃ'}, // 0183 → 0182 + {u'ƅ', u'Ƅ'}, // 0185 → 0184 + {u'ƈ', u'Ƈ'}, // 0188 → 0187 + {u'ƌ', u'Ƌ'}, // 018c → 018b + {u'ƒ', u'Ƒ'}, // 0192 → 0191 + {u'ƙ', u'Ƙ'}, // 0199 → 0198 + {u'ƚ', u'Ƚ'}, // 019a → 023d + {u'ƞ', u'Ƞ'}, // 019e → 0220 + {u'ơ', u'Ơ'}, // 01a1 → 01a0 + {u'ƣ', u'Ƣ'}, // 01a3 → 01a2 + {u'ƥ', u'Ƥ'}, // 01a5 → 01a4 + {u'ƨ', u'Ƨ'}, // 01a8 → 01a7 + {u'ƭ', u'Ƭ'}, // 01ad → 01ac + {u'ư', u'Ư'}, // 01b0 → 01af + {u'ƴ', u'Ƴ'}, // 01b4 → 01b3 + {u'ƶ', u'Ƶ'}, // 01b6 → 01b5 + {u'ƹ', u'Ƹ'}, // 01b9 → 01b8 + {u'ƹ', u'Ʒ'}, // 01b9 → 01b7 + {u'ƽ', u'Ƽ'}, // 01bd → 01bc + {u'dž', u'DZ'}, // 01c6 → 01f1 + {u'dž', u'DŽ'}, // 01c6 → 01c4 + {u'lj', u'LJ'}, // 01c9 → 01c7 + {u'nj', u'NJ'}, // 01cc → 01ca + {u'ǎ', u'Ǎ'}, // 01ce → 01cd + {u'ǐ', u'Ǐ'}, // 01d0 → 01cf + {u'ǒ', u'Ǒ'}, // 01d2 → 01d1 + {u'ǔ', u'Ǔ'}, // 01d4 → 01d3 + {u'ǖ', u'Ǖ'}, // 01d6 → 01d5 + {u'ǘ', u'Ǘ'}, // 01d8 → 01d7 + {u'ǚ', u'Ǚ'}, // 01da → 01d9 + {u'ǜ', u'Ǜ'}, // 01dc → 01db + {u'ǟ', u'Ǟ'}, // 01df → 01de + {u'ǡ', u'Ȧ'}, // 01e1 → 0226 + {u'ǡ', u'Ǡ'}, // 01e1 → 01e0 + {u'ǣ', u'Ǣ'}, // 01e3 → 01e2 + {u'ǥ', u'Ǥ'}, // 01e5 → 01e4 + {u'ǧ', u'Ǧ'}, // 01e7 → 01e6 + {u'ǩ', u'Ǩ'}, // 01e9 → 01e8 + {u'ǫ', u'Ǫ'}, // 01eb → 01ea + {u'ǭ', u'Ǭ'}, // 01ed → 01ec + {u'ǯ', u'Ǯ'}, // 01ef → 01ee + {u'ǵ', u'Ǵ'}, // 01f5 → 01f4 + {u'ǹ', u'Ǹ'}, // 01f9 → 01f8 + {u'ǻ', u'Ǻ'}, // 01fb → 01fa + {u'ǽ', u'Ǽ'}, // 01fd → 01fc + {u'ǿ', u'Ǿ'}, // 01ff → 01fe + {u'ȁ', u'Ȁ'}, // 0201 → 0200 + {u'ȃ', u'Ȃ'}, // 0203 → 0202 + {u'ȅ', u'Ȅ'}, // 0205 → 0204 + {u'ȇ', u'Ȇ'}, // 0207 → 0206 + {u'ȉ', u'Ȉ'}, // 0209 → 0208 + {u'ȋ', u'Ȋ'}, // 020b → 020a + {u'ȍ', u'Ȍ'}, // 020d → 020c + {u'ȏ', u'Ȏ'}, // 020f → 020e + {u'ȑ', u'Ȑ'}, // 0211 → 0210 + {u'ȓ', u'Ȓ'}, // 0213 → 0212 + {u'ȕ', u'Ȕ'}, // 0215 → 0214 + {u'ȗ', u'Ȗ'}, // 0217 → 0216 + {u'ș', u'Ș'}, // 0219 → 0218 + {u'ț', u'Ț'}, // 021b → 021a + {u'ȝ', u'Ȝ'}, // 021d → 021c + {u'ȟ', u'Ȟ'}, // 021f → 021e + {u'ȣ', u'Ȣ'}, // 0223 → 0222 + {u'ȥ', u'Ȥ'}, // 0225 → 0224 + {u'ȩ', u'Ȩ'}, // 0229 → 0228 + {u'ȫ', u'Ȫ'}, // 022b → 022a + {u'ȭ', u'Ȭ'}, // 022d → 022c + {u'ȯ', u'Ȯ'}, // 022f → 022e + {u'ȱ', u'Ȱ'}, // 0231 → 0230 + {u'ȳ', u'Ȳ'}, // 0233 → 0232 + {u'ȼ', u'Ȼ'}, // 023c → 023b + {u'ȿ', u'Ȿ'}, // 023f → 2c7e + {u'ɀ', u'Ɀ'}, // 0240 → 2c7f + {u'ɂ', u'Ɂ'}, // 0242 → 0241 + {u'ɇ', u'Ɇ'}, // 0247 → 0246 + {u'ɉ', u'Ɉ'}, // 0249 → 0248 + {u'ɍ', u'Ɍ'}, // 024d → 024c + {u'ɏ', u'Ɏ'}, // 024f → 024e + {u'ɐ', u'Ɐ'}, // 0250 → 2c6f + {u'ɑ', u'Ɑ'}, // 0251 → 2c6d + {u'ɒ', u'Ɒ'}, // 0252 → 2c70 + {u'ɓ', u'Ɓ'}, // 0253 → 0181 + {u'ɔ', u'Ɔ'}, // 0254 → 0186 + {u'ɗ', u'Ɗ'}, // 0257 → 018a + {u'ɘ', u'Ǝ'}, // 0258 → 018e + {u'ə', u'Ə'}, // 0259 → 018f + {u'ɛ', u'Ɛ'}, // 025b → 0190 + {u'ɜ', u'Ɜ'}, // 025c → a7ab + {u'ɠ', u'Ɠ'}, // 0260 → 0193 + {u'ɡ', u'Ɡ'}, // 0261 → a7ac + {u'ɣ', u'Ɣ'}, // 0263 → 0194 + {u'ɥ', u'Ɥ'}, // 0265 → a78d + {u'ɦ', u'Ɦ'}, // 0266 → a7aa + {u'ɨ', u'Ɨ'}, // 0268 → 0197 + {u'ɩ', u'Ɩ'}, // 0269 → 0196 + {u'ɫ', u'Ɫ'}, // 026b → 2c62 + {u'ɬ', u'Ɬ'}, // 026c → a7ad + {u'ɯ', u'Ɯ'}, // 026f → 019c + {u'ɱ', u'Ɱ'}, // 0271 → 2c6e + {u'ɲ', u'Ɲ'}, // 0272 → 019d + {u'ɽ', u'Ɽ'}, // 027d → 2c64 + {u'ʂ', u'Ʂ'}, // 0282 → a7c5 + {u'ʃ', u'Ʃ'}, // 0283 → 01a9 + {u'ʇ', u'Ʇ'}, // 0287 → a7b1 + {u'ʈ', u'Ʈ'}, // 0288 → 01ae + {u'ʉ', u'Ʉ'}, // 0289 → 0244 + {u'ʊ', u'Ʊ'}, // 028a → 01b1 + {u'ʋ', u'Ʋ'}, // 028b → 01b2 + {u'ʌ', u'Ʌ'}, // 028c → 0245 + {u'ʝ', u'Ʝ'}, // 029d → a7b2 + {u'ʞ', u'Ʞ'}, // 029e → a7b0 + {u'ͱ', u'Ͱ'}, // 0371 → 0370 + {u'ͳ', u'Ͳ'}, // 0373 → 0372 + {u'ͷ', u'Ͷ'}, // 0377 → 0376 + {u'ͻ', u'Ͻ'}, // 037b → 03fd + {u'ͼ', u'Ͼ'}, // 037c → 03fe + {u'ͽ', u'Ͽ'}, // 037d → 03ff + {u'ΐ', u'Ι'}, // 0390 → 0399 + {u'ΐ', u'Ϊ'}, // 0390 → 03aa + {u'ά', u'Α'}, // 03ac → 0391 + {u'ά', u'Ά'}, // 03ac → 0386 + {u'έ', u'Έ'}, // 03ad → 0388 + {u'έ', u'Ε'}, // 03ad → 0395 + {u'ή', u'Η'}, // 03ae → 0397 + {u'ή', u'Ή'}, // 03ae → 0389 + {u'ί', u'Ί'}, // 03af → 038a + {u'ΰ', u'Υ'}, // 03b0 → 03a5 + {u'ΰ', u'Ϋ'}, // 03b0 → 03ab + {u'β', u'Β'}, // 03b2 → 0392 + {u'γ', u'Γ'}, // 03b3 → 0393 + {u'δ', u'Δ'}, // 03b4 → 0394 + {u'ζ', u'Ζ'}, // 03b6 → 0396 + {u'θ', u'Θ'}, // 03b8 → 0398 + {u'κ', u'Κ'}, // 03ba → 039a + {u'λ', u'Λ'}, // 03bb → 039b + {u'μ', u'Μ'}, // 03bc → 039c + {u'ν', u'Ν'}, // 03bd → 039d + {u'ξ', u'Ξ'}, // 03be → 039e + {u'ο', u'Ο'}, // 03bf → 039f + {u'π', u'Π'}, // 03c0 → 03a0 + {u'ρ', u'Ρ'}, // 03c1 → 03a1 + {u'σ', u'Σ'}, // 03c3 → 03a3 + {u'τ', u'Τ'}, // 03c4 → 03a4 + {u'φ', u'Φ'}, // 03c6 → 03a6 + {u'χ', u'Χ'}, // 03c7 → 03a7 + {u'ψ', u'Ψ'}, // 03c8 → 03a8 + {u'ω', u'Ω'}, // 03c9 → 03a9 + {u'ό', u'Ό'}, // 03cc → 038c + {u'ύ', u'Ύ'}, // 03cd → 038e + {u'ώ', u'Ώ'}, // 03ce → 038f + {u'ϣ', u'Ϣ'}, // 03e3 → 03e2 + {u'ϥ', u'Ϥ'}, // 03e5 → 03e4 + {u'ϧ', u'Ϧ'}, // 03e7 → 03e6 + {u'ϩ', u'Ϩ'}, // 03e9 → 03e8 + {u'ϫ', u'Ϫ'}, // 03eb → 03ea + {u'ϭ', u'Ϭ'}, // 03ed → 03ec + {u'ϯ', u'Ϯ'}, // 03ef → 03ee + {u'ϸ', u'Ϸ'}, // 03f8 → 03f7 + {u'ϻ', u'Ϻ'}, // 03fb → 03fa + {u'а', u'А'}, // 0430 → 0410 + {u'б', u'Б'}, // 0431 → 0411 + {u'в', u'В'}, // 0432 → 0412 + {u'г', u'Г'}, // 0433 → 0413 + {u'д', u'Д'}, // 0434 → 0414 + {u'е', u'Е'}, // 0435 → 0415 + {u'е', u'И'}, // 0435 → 0418 + {u'ж', u'Ж'}, // 0436 → 0416 + {u'з', u'З'}, // 0437 → 0417 + {u'й', u'Й'}, // 0439 → 0419 + {u'к', u'К'}, // 043a → 041a + {u'л', u'Л'}, // 043b → 041b + {u'л', u'Э'}, // 043b → 042d + {u'м', u'М'}, // 043c → 041c + {u'н', u'Н'}, // 043d → 041d + {u'о', u'О'}, // 043e → 041e + {u'п', u'П'}, // 043f → 041f + {u'р', u'Р'}, // 0440 → 0420 + {u'с', u'С'}, // 0441 → 0421 + {u'т', u'Т'}, // 0442 → 0422 + {u'у', u'У'}, // 0443 → 0423 + {u'ф', u'Ф'}, // 0444 → 0424 + {u'х', u'Х'}, // 0445 → 0425 + {u'ц', u'Ц'}, // 0446 → 0426 + {u'ч', u'Ч'}, // 0447 → 0427 + {u'ш', u'Ш'}, // 0448 → 0428 + {u'щ', u'Щ'}, // 0449 → 0429 + {u'ъ', u'Ъ'}, // 044a → 042a + {u'ы', u'Ы'}, // 044b → 042b + {u'ь', u'Ь'}, // 044c → 042c + {u'ю', u'Ю'}, // 044e → 042e + {u'я', u'Я'}, // 044f → 042f + {u'ѐ', u'Ѐ'}, // 0450 → 0400 + {u'ё', u'Ё'}, // 0451 → 0401 + {u'ђ', u'Ђ'}, // 0452 → 0402 + {u'ѓ', u'Ѓ'}, // 0453 → 0403 + {u'є', u'Ѹ'}, // 0454 → 0478 + {u'є', u'Є'}, // 0454 → 0404 + {u'ѕ', u'Ѕ'}, // 0455 → 0405 + {u'і', u'І'}, // 0456 → 0406 + {u'ї', u'Ї'}, // 0457 → 0407 + {u'ј', u'Ј'}, // 0458 → 0408 + {u'љ', u'Љ'}, // 0459 → 0409 + {u'њ', u'Њ'}, // 045a → 040a + {u'ћ', u'Ћ'}, // 045b → 040b + {u'ќ', u'Ќ'}, // 045c → 040c + {u'ѝ', u'Ѝ'}, // 045d → 040d + {u'ў', u'Ў'}, // 045e → 040e + {u'џ', u'Џ'}, // 045f → 040f + {u'ѡ', u'Ѡ'}, // 0461 → 0460 + {u'ѣ', u'Ѣ'}, // 0463 → 0462 + {u'ѥ', u'Ѥ'}, // 0465 → 0464 + {u'ѧ', u'Ѧ'}, // 0467 → 0466 + {u'ѩ', u'Ѩ'}, // 0469 → 0468 + {u'ѫ', u'Ѫ'}, // 046b → 046a + {u'ѭ', u'Ѭ'}, // 046d → 046c + {u'ѯ', u'Ѯ'}, // 046f → 046e + {u'ѱ', u'Ѱ'}, // 0471 → 0470 + {u'ѳ', u'Ѳ'}, // 0473 → 0472 + {u'ѵ', u'Ѵ'}, // 0475 → 0474 + {u'ѷ', u'Ѷ'}, // 0477 → 0476 + {u'ѻ', u'Ѻ'}, // 047b → 047a + {u'ѽ', u'Ѽ'}, // 047d → 047c + {u'ѿ', u'Ѿ'}, // 047f → 047e + {u'ҁ', u'Ҁ'}, // 0481 → 0480 + {u'ҋ', u'Ҋ'}, // 048b → 048a + {u'ҍ', u'Ҍ'}, // 048d → 048c + {u'ҏ', u'Ҏ'}, // 048f → 048e + {u'ґ', u'Ґ'}, // 0491 → 0490 + {u'ғ', u'Ғ'}, // 0493 → 0492 + {u'ҕ', u'Ҕ'}, // 0495 → 0494 + {u'җ', u'Җ'}, // 0497 → 0496 + {u'ҙ', u'Ҙ'}, // 0499 → 0498 + {u'қ', u'Қ'}, // 049b → 049a + {u'ҝ', u'Ҝ'}, // 049d → 049c + {u'ҟ', u'Ҟ'}, // 049f → 049e + {u'ҡ', u'Ҡ'}, // 04a1 → 04a0 + {u'ң', u'Ң'}, // 04a3 → 04a2 + {u'ҥ', u'Ҥ'}, // 04a5 → 04a4 + {u'ҧ', u'Ҧ'}, // 04a7 → 04a6 + {u'ҩ', u'Ҩ'}, // 04a9 → 04a8 + {u'ҫ', u'Ҫ'}, // 04ab → 04aa + {u'ҭ', u'Ҭ'}, // 04ad → 04ac + {u'ү', u'Ү'}, // 04af → 04ae + {u'ұ', u'Ұ'}, // 04b1 → 04b0 + {u'ҳ', u'Ҳ'}, // 04b3 → 04b2 + {u'ҵ', u'Ҵ'}, // 04b5 → 04b4 + {u'ҷ', u'Ҷ'}, // 04b7 → 04b6 + {u'ҹ', u'Ҹ'}, // 04b9 → 04b8 + {u'һ', u'Һ'}, // 04bb → 04ba + {u'ҽ', u'Ҽ'}, // 04bd → 04bc + {u'ҿ', u'Ҿ'}, // 04bf → 04be + {u'ӂ', u'Ӂ'}, // 04c2 → 04c1 + {u'ӄ', u'Ӄ'}, // 04c4 → 04c3 + {u'ӆ', u'Ӆ'}, // 04c6 → 04c5 + {u'ӈ', u'Ӈ'}, // 04c8 → 04c7 + {u'ӊ', u'Ӊ'}, // 04ca → 04c9 + {u'ӌ', u'Ӌ'}, // 04cc → 04cb + {u'ӎ', u'Ӎ'}, // 04ce → 04cd + {u'ӑ', u'Ӑ'}, // 04d1 → 04d0 + {u'ӓ', u'Ӓ'}, // 04d3 → 04d2 + {u'ӕ', u'Ӕ'}, // 04d5 → 04d4 + {u'ӗ', u'Ӗ'}, // 04d7 → 04d6 + {u'ә', u'Ә'}, // 04d9 → 04d8 + {u'ӛ', u'Ӛ'}, // 04db → 04da + {u'ӝ', u'Ӝ'}, // 04dd → 04dc + {u'ӟ', u'Ӟ'}, // 04df → 04de + {u'ӡ', u'Ӡ'}, // 04e1 → 04e0 + {u'ӣ', u'Ӣ'}, // 04e3 → 04e2 + {u'ӥ', u'Ӥ'}, // 04e5 → 04e4 + {u'ӧ', u'Ӧ'}, // 04e7 → 04e6 + {u'ө', u'Ө'}, // 04e9 → 04e8 + {u'ӫ', u'Ӫ'}, // 04eb → 04ea + {u'ӭ', u'Ӭ'}, // 04ed → 04ec + {u'ӯ', u'Ӯ'}, // 04ef → 04ee + {u'ӱ', u'Ӱ'}, // 04f1 → 04f0 + {u'ӳ', u'Ӳ'}, // 04f3 → 04f2 + {u'ӵ', u'Ӵ'}, // 04f5 → 04f4 + {u'ӷ', u'Ӷ'}, // 04f7 → 04f6 + {u'ӹ', u'Ӹ'}, // 04f9 → 04f8 + {u'ӻ', u'Ӻ'}, // 04fb → 04fa + {u'ӽ', u'Ӽ'}, // 04fd → 04fc + {u'ӿ', u'Ӿ'}, // 04ff → 04fe + {u'ԁ', u'Ԁ'}, // 0501 → 0500 + {u'ԃ', u'Ԃ'}, // 0503 → 0502 + {u'ԅ', u'Ԅ'}, // 0505 → 0504 + {u'ԇ', u'Ԇ'}, // 0507 → 0506 + {u'ԉ', u'Ԉ'}, // 0509 → 0508 + {u'ԋ', u'Ԋ'}, // 050b → 050a + {u'ԍ', u'Ԍ'}, // 050d → 050c + {u'ԏ', u'Ԏ'}, // 050f → 050e + {u'ԑ', u'Ԑ'}, // 0511 → 0510 + {u'ԓ', u'Ԓ'}, // 0513 → 0512 + {u'ԕ', u'Ԕ'}, // 0515 → 0514 + {u'ԗ', u'Ԗ'}, // 0517 → 0516 + {u'ԙ', u'Ԙ'}, // 0519 → 0518 + {u'ԛ', u'Ԛ'}, // 051b → 051a + {u'ԝ', u'Ԝ'}, // 051d → 051c + {u'ԟ', u'Ԟ'}, // 051f → 051e + {u'ԡ', u'Ԡ'}, // 0521 → 0520 + {u'ԣ', u'Ԣ'}, // 0523 → 0522 + {u'ԥ', u'Ԥ'}, // 0525 → 0524 + {u'ԧ', u'Ԧ'}, // 0527 → 0526 + {u'ԩ', u'Ԩ'}, // 0529 → 0528 + {u'ԫ', u'Ԫ'}, // 052b → 052a + {u'ԭ', u'Ԭ'}, // 052d → 052c + {u'ԯ', u'Ԯ'}, // 052f → 052e + {u'ա', u'Ա'}, // 0561 → 0531 + {u'բ', u'Բ'}, // 0562 → 0532 + {u'գ', u'Գ'}, // 0563 → 0533 + {u'դ', u'Դ'}, // 0564 → 0534 + {u'ե', u'Ե'}, // 0565 → 0535 + {u'զ', u'Զ'}, // 0566 → 0536 + {u'է', u'Է'}, // 0567 → 0537 + {u'ը', u'Ը'}, // 0568 → 0538 + {u'թ', u'Թ'}, // 0569 → 0539 + {u'ժ', u'Ժ'}, // 056a → 053a + {u'ի', u'Ի'}, // 056b → 053b + {u'լ', u'Լ'}, // 056c → 053c + {u'խ', u'Խ'}, // 056d → 053d + {u'ծ', u'Ծ'}, // 056e → 053e + {u'կ', u'Կ'}, // 056f → 053f + {u'հ', u'Հ'}, // 0570 → 0540 + {u'ձ', u'Ձ'}, // 0571 → 0541 + {u'ղ', u'Ղ'}, // 0572 → 0542 + {u'ճ', u'Ճ'}, // 0573 → 0543 + {u'մ', u'Մ'}, // 0574 → 0544 + {u'յ', u'Յ'}, // 0575 → 0545 + {u'ն', u'Ն'}, // 0576 → 0546 + {u'շ', u'Շ'}, // 0577 → 0547 + {u'ո', u'Ո'}, // 0578 → 0548 + {u'չ', u'Չ'}, // 0579 → 0549 + {u'պ', u'Պ'}, // 057a → 054a + {u'ջ', u'Ջ'}, // 057b → 054b + {u'ռ', u'Ռ'}, // 057c → 054c + {u'ս', u'Ս'}, // 057d → 054d + {u'վ', u'Վ'}, // 057e → 054e + {u'տ', u'Տ'}, // 057f → 054f + {u'ր', u'Ր'}, // 0580 → 0550 + {u'ց', u'Ց'}, // 0581 → 0551 + {u'ւ', u'Ւ'}, // 0582 → 0552 + {u'փ', u'Փ'}, // 0583 → 0553 + {u'ք', u'Ք'}, // 0584 → 0554 + {u'օ', u'Օ'}, // 0585 → 0555 + {u'ֆ', u'Ֆ'}, // 0586 → 0556 + {u'ᏸ', u'Ᏸ'}, // 13f8 → 13f0 + {u'ᏹ', u'Ᏹ'}, // 13f9 → 13f1 + {u'ᏺ', u'Ᏺ'}, // 13fa → 13f2 + {u'ᏻ', u'Ᏻ'}, // 13fb → 13f3 + {u'ᏼ', u'Ᏼ'}, // 13fc → 13f4 + {u'ᏽ', u'Ᏽ'}, // 13fd → 13f5 + {u'ᵹ', u'Ᵹ'}, // 1d79 → a77d + {u'ᵽ', u'Ᵽ'}, // 1d7d → 2c63 + {u'ᶎ', u'Ᶎ'}, // 1d8e → a7c6 + {u'ḁ', u'Ḁ'}, // 1e01 → 1e00 + {u'ḃ', u'Ḃ'}, // 1e03 → 1e02 + {u'ḅ', u'Ḅ'}, // 1e05 → 1e04 + {u'ḇ', u'Ḇ'}, // 1e07 → 1e06 + {u'ḉ', u'Ḉ'}, // 1e09 → 1e08 + {u'ḋ', u'Ḋ'}, // 1e0b → 1e0a + {u'ḍ', u'Ḍ'}, // 1e0d → 1e0c + {u'ḏ', u'Ḏ'}, // 1e0f → 1e0e + {u'ḑ', u'Ḑ'}, // 1e11 → 1e10 + {u'ḓ', u'Ḓ'}, // 1e13 → 1e12 + {u'ḕ', u'Ḕ'}, // 1e15 → 1e14 + {u'ḗ', u'Ḗ'}, // 1e17 → 1e16 + {u'ḙ', u'Ḙ'}, // 1e19 → 1e18 + {u'ḛ', u'Ḛ'}, // 1e1b → 1e1a + {u'ḛ', u'Ẽ'}, // 1e1b → 1ebc + {u'ḝ', u'Ḝ'}, // 1e1d → 1e1c + {u'ḟ', u'Ḟ'}, // 1e1f → 1e1e + {u'ḡ', u'Ḡ'}, // 1e21 → 1e20 + {u'ḣ', u'Ḣ'}, // 1e23 → 1e22 + {u'ḥ', u'Ḥ'}, // 1e25 → 1e24 + {u'ḧ', u'Ḧ'}, // 1e27 → 1e26 + {u'ḩ', u'Ḩ'}, // 1e29 → 1e28 + {u'ḫ', u'Ḫ'}, // 1e2b → 1e2a + {u'ḭ', u'Ḭ'}, // 1e2d → 1e2c + {u'ḯ', u'Ḯ'}, // 1e2f → 1e2e + {u'ḱ', u'Ḱ'}, // 1e31 → 1e30 + {u'ḳ', u'Ḳ'}, // 1e33 → 1e32 + {u'ḵ', u'Ḵ'}, // 1e35 → 1e34 + {u'ḷ', u'Ḷ'}, // 1e37 → 1e36 + {u'ḹ', u'Ḹ'}, // 1e39 → 1e38 + {u'ḻ', u'Ḻ'}, // 1e3b → 1e3a + {u'ḽ', u'Ḽ'}, // 1e3d → 1e3c + {u'ḿ', u'Ḿ'}, // 1e3f → 1e3e + {u'ṁ', u'Ṁ'}, // 1e41 → 1e40 + {u'ṃ', u'Ṃ'}, // 1e43 → 1e42 + {u'ṅ', u'Ṅ'}, // 1e45 → 1e44 + {u'ṇ', u'Ṇ'}, // 1e47 → 1e46 + {u'ṉ', u'Ṉ'}, // 1e49 → 1e48 + {u'ṋ', u'Ṋ'}, // 1e4b → 1e4a + {u'ṍ', u'Ṍ'}, // 1e4d → 1e4c + {u'ṏ', u'Ṏ'}, // 1e4f → 1e4e + {u'ṑ', u'Ṑ'}, // 1e51 → 1e50 + {u'ṓ', u'Ṓ'}, // 1e53 → 1e52 + {u'ṕ', u'Ṕ'}, // 1e55 → 1e54 + {u'ṗ', u'Ṗ'}, // 1e57 → 1e56 + {u'ṙ', u'Ṙ'}, // 1e59 → 1e58 + {u'ṛ', u'Ṛ'}, // 1e5b → 1e5a + {u'ṝ', u'Ṝ'}, // 1e5d → 1e5c + {u'ṟ', u'Ṟ'}, // 1e5f → 1e5e + {u'ṡ', u'Ṡ'}, // 1e61 → 1e60 + {u'ṣ', u'Ṣ'}, // 1e63 → 1e62 + {u'ṥ', u'Ṥ'}, // 1e65 → 1e64 + {u'ṧ', u'Ṧ'}, // 1e67 → 1e66 + {u'ṩ', u'Ṩ'}, // 1e69 → 1e68 + {u'ṫ', u'Ṫ'}, // 1e6b → 1e6a + {u'ṭ', u'Ṭ'}, // 1e6d → 1e6c + {u'ṯ', u'Ṯ'}, // 1e6f → 1e6e + {u'ṱ', u'Ṱ'}, // 1e71 → 1e70 + {u'ṳ', u'Ṳ'}, // 1e73 → 1e72 + {u'ṵ', u'Ṵ'}, // 1e75 → 1e74 + {u'ṷ', u'Ṷ'}, // 1e77 → 1e76 + {u'ṹ', u'Ṹ'}, // 1e79 → 1e78 + {u'ṻ', u'Ṻ'}, // 1e7b → 1e7a + {u'ṽ', u'Ṽ'}, // 1e7d → 1e7c + {u'ṿ', u'Ṿ'}, // 1e7f → 1e7e + {u'ẁ', u'Ẁ'}, // 1e81 → 1e80 + {u'ẃ', u'Ẃ'}, // 1e83 → 1e82 + {u'ẅ', u'Ẅ'}, // 1e85 → 1e84 + {u'ẇ', u'Ẇ'}, // 1e87 → 1e86 + {u'ẉ', u'Ẉ'}, // 1e89 → 1e88 + {u'ẋ', u'Ẋ'}, // 1e8b → 1e8a + {u'ẍ', u'Ẍ'}, // 1e8d → 1e8c + {u'ẏ', u'Ẏ'}, // 1e8f → 1e8e + {u'ẑ', u'Ẑ'}, // 1e91 → 1e90 + {u'ẓ', u'Ẓ'}, // 1e93 → 1e92 + {u'ẕ', u'Ẕ'}, // 1e95 → 1e94 + {u'ạ', u'Ạ'}, // 1ea1 → 1ea0 + {u'ả', u'Ả'}, // 1ea3 → 1ea2 + {u'ấ', u'Ấ'}, // 1ea5 → 1ea4 + {u'ầ', u'Ầ'}, // 1ea7 → 1ea6 + {u'ẩ', u'Ẩ'}, // 1ea9 → 1ea8 + {u'ẫ', u'Ẫ'}, // 1eab → 1eaa + {u'ậ', u'Ậ'}, // 1ead → 1eac + {u'ắ', u'Ắ'}, // 1eaf → 1eae + {u'ằ', u'Ằ'}, // 1eb1 → 1eb0 + {u'ẳ', u'Ẳ'}, // 1eb3 → 1eb2 + {u'ẵ', u'Ẵ'}, // 1eb5 → 1eb4 + {u'ặ', u'Ặ'}, // 1eb7 → 1eb6 + {u'ẹ', u'Ẹ'}, // 1eb9 → 1eb8 + {u'ẻ', u'Ẻ'}, // 1ebb → 1eba + {u'ế', u'Ế'}, // 1ebf → 1ebe + {u'ề', u'Ề'}, // 1ec1 → 1ec0 + {u'ể', u'Ể'}, // 1ec3 → 1ec2 + {u'ễ', u'Ễ'}, // 1ec5 → 1ec4 + {u'ệ', u'Ệ'}, // 1ec7 → 1ec6 + {u'ỉ', u'Ỉ'}, // 1ec9 → 1ec8 + {u'ị', u'Ị'}, // 1ecb → 1eca + {u'ọ', u'Ọ'}, // 1ecd → 1ecc + {u'ỏ', u'Ỏ'}, // 1ecf → 1ece + {u'ố', u'Ố'}, // 1ed1 → 1ed0 + {u'ồ', u'Ồ'}, // 1ed3 → 1ed2 + {u'ổ', u'Ổ'}, // 1ed5 → 1ed4 + {u'ỗ', u'Ỗ'}, // 1ed7 → 1ed6 + {u'ộ', u'Ộ'}, // 1ed9 → 1ed8 + {u'ớ', u'Ớ'}, // 1edb → 1eda + {u'ờ', u'Ờ'}, // 1edd → 1edc + {u'ở', u'Ở'}, // 1edf → 1ede + {u'ỡ', u'Ỡ'}, // 1ee1 → 1ee0 + {u'ợ', u'Ợ'}, // 1ee3 → 1ee2 + {u'ụ', u'Ụ'}, // 1ee5 → 1ee4 + {u'ủ', u'Ủ'}, // 1ee7 → 1ee6 + {u'ứ', u'Ứ'}, // 1ee9 → 1ee8 + {u'ừ', u'Ừ'}, // 1eeb → 1eea + {u'ử', u'Ử'}, // 1eed → 1eec + {u'ữ', u'Ữ'}, // 1eef → 1eee + {u'ự', u'Ự'}, // 1ef1 → 1ef0 + {u'ỳ', u'Ỳ'}, // 1ef3 → 1ef2 + {u'ỵ', u'Ỵ'}, // 1ef5 → 1ef4 + {u'ỷ', u'Ỷ'}, // 1ef7 → 1ef6 + {u'ỹ', u'Ỹ'}, // 1ef9 → 1ef8 + {u'ỻ', u'Ỻ'}, // 1efb → 1efa + {u'ỽ', u'Ỽ'}, // 1efd → 1efc + {u'ỿ', u'Ỿ'}, // 1eff → 1efe + {u'ἀ', u'Ἀ'}, // 1f00 → 1f08 + {u'ἁ', u'Ἁ'}, // 1f01 → 1f09 + {u'ἂ', u'Ἂ'}, // 1f02 → 1f0a + {u'ἃ', u'Ἃ'}, // 1f03 → 1f0b + {u'ἄ', u'Ἄ'}, // 1f04 → 1f0c + {u'ἅ', u'Ἅ'}, // 1f05 → 1f0d + {u'ἆ', u'Ἆ'}, // 1f06 → 1f0e + {u'ἇ', u'Ἇ'}, // 1f07 → 1f0f + {u'ἐ', u'Ἐ'}, // 1f10 → 1f18 + {u'ἑ', u'Ἑ'}, // 1f11 → 1f19 + {u'ἒ', u'Ἒ'}, // 1f12 → 1f1a + {u'ἓ', u'Ἓ'}, // 1f13 → 1f1b + {u'ἔ', u'Ἔ'}, // 1f14 → 1f1c + {u'ἕ', u'Ἕ'}, // 1f15 → 1f1d + {u'ἠ', u'Ἠ'}, // 1f20 → 1f28 + {u'ἡ', u'Ἡ'}, // 1f21 → 1f29 + {u'ἢ', u'Ἢ'}, // 1f22 → 1f2a + {u'ἣ', u'Ἣ'}, // 1f23 → 1f2b + {u'ἤ', u'Ἤ'}, // 1f24 → 1f2c + {u'ἥ', u'Ἥ'}, // 1f25 → 1f2d + {u'ἦ', u'Ἦ'}, // 1f26 → 1f2e + {u'ἧ', u'Ἧ'}, // 1f27 → 1f2f + {u'ἰ', u'Ἰ'}, // 1f30 → 1f38 + {u'ἱ', u'Ἱ'}, // 1f31 → 1f39 + {u'ἲ', u'Ἲ'}, // 1f32 → 1f3a + {u'ἳ', u'Ἳ'}, // 1f33 → 1f3b + {u'ἴ', u'Ἴ'}, // 1f34 → 1f3c + {u'ἵ', u'Ἵ'}, // 1f35 → 1f3d + {u'ἶ', u'Ἶ'}, // 1f36 → 1f3e + {u'ἷ', u'Ἷ'}, // 1f37 → 1f3f + {u'ὀ', u'Ὀ'}, // 1f40 → 1f48 + {u'ὁ', u'Ὁ'}, // 1f41 → 1f49 + {u'ὂ', u'Ὂ'}, // 1f42 → 1f4a + {u'ὃ', u'Ὃ'}, // 1f43 → 1f4b + {u'ὄ', u'Ὄ'}, // 1f44 → 1f4c + {u'ὅ', u'Ὅ'}, // 1f45 → 1f4d + {u'ὑ', u'Ὑ'}, // 1f51 → 1f59 + {u'ὓ', u'Ὓ'}, // 1f53 → 1f5b + {u'ὕ', u'Ὕ'}, // 1f55 → 1f5d + {u'ὗ', u'Ὗ'}, // 1f57 → 1f5f + {u'ὠ', u'Ὠ'}, // 1f60 → 1f68 + {u'ὡ', u'Ὡ'}, // 1f61 → 1f69 + {u'ὢ', u'Ὢ'}, // 1f62 → 1f6a + {u'ὣ', u'Ὣ'}, // 1f63 → 1f6b + {u'ὤ', u'Ὤ'}, // 1f64 → 1f6c + {u'ὥ', u'Ὥ'}, // 1f65 → 1f6d + {u'ὦ', u'Ὦ'}, // 1f66 → 1f6e + {u'ὧ', u'Ὧ'}, // 1f67 → 1f6f + {u'ὰ', u'Ὰ'}, // 1f70 → 1fba + {u'ά', u'Ά'}, // 1f71 → 1fbb + {u'ὲ', u'Ὲ'}, // 1f72 → 1fc8 + {u'έ', u'Έ'}, // 1f73 → 1fc9 + {u'ὴ', u'Ὴ'}, // 1f74 → 1fca + {u'ή', u'Ή'}, // 1f75 → 1fcb + {u'ὶ', u'Ὶ'}, // 1f76 → 1fda + {u'ί', u'Ί'}, // 1f77 → 1fdb + {u'ὸ', u'Ὸ'}, // 1f78 → 1ff8 + {u'ό', u'Ό'}, // 1f79 → 1ff9 + {u'ὺ', u'Ὺ'}, // 1f7a → 1fea + {u'ύ', u'Ύ'}, // 1f7b → 1feb + {u'ὼ', u'Ὼ'}, // 1f7c → 1ffa + {u'ώ', u'Ώ'}, // 1f7d → 1ffb + {u'ᾰ', u'Ᾰ'}, // 1fb0 → 1fb8 + {u'ᾱ', u'Ᾱ'}, // 1fb1 → 1fb9 + {u'ῐ', u'Ῐ'}, // 1fd0 → 1fd8 + {u'ῑ', u'Ῑ'}, // 1fd1 → 1fd9 + {u'ῠ', u'Ῠ'}, // 1fe0 → 1fe8 + {u'ῡ', u'Ῡ'}, // 1fe1 → 1fe9 + {u'ῥ', u'Ῥ'}, // 1fe5 → 1fec + {u'ℓ', u'ℒ'}, // 2113 → 2112 + {u'ℯ', u'ℰ'}, // 212f → 2130 + {u'ℼ', u'ℿ'}, // 213c → 213f + {u'ℼ', u'ℙ'}, // 213c → 2119 + {u'ℽ', u'ℾ'}, // 213d → 213e + {u'ⅆ', u'ⅅ'}, // 2146 → 2145 + {u'ⅎ', u'Ⅎ'}, // 214e → 2132 + {u'ⰰ', u'Ⰰ'}, // 2c30 → 2c00 + {u'ⰱ', u'Ⰱ'}, // 2c31 → 2c01 + {u'ⰲ', u'Ⰲ'}, // 2c32 → 2c02 + {u'ⰳ', u'Ⰳ'}, // 2c33 → 2c03 + {u'ⰴ', u'Ⰴ'}, // 2c34 → 2c04 + {u'ⰵ', u'Ⰵ'}, // 2c35 → 2c05 + {u'ⰶ', u'Ⰶ'}, // 2c36 → 2c06 + {u'ⰷ', u'Ⰷ'}, // 2c37 → 2c07 + {u'ⰸ', u'Ⰸ'}, // 2c38 → 2c08 + {u'ⰹ', u'Ⰹ'}, // 2c39 → 2c09 + {u'ⰹ', u'Ⰻ'}, // 2c39 → 2c0b + {u'ⰺ', u'Ⰺ'}, // 2c3a → 2c0a + {u'ⰼ', u'Ⰼ'}, // 2c3c → 2c0c + {u'ⰽ', u'Ⰽ'}, // 2c3d → 2c0d + {u'ⰾ', u'Ⰾ'}, // 2c3e → 2c0e + {u'ⰿ', u'Ⰿ'}, // 2c3f → 2c0f + {u'ⱀ', u'Ⱀ'}, // 2c40 → 2c10 + {u'ⱁ', u'Ⱁ'}, // 2c41 → 2c11 + {u'ⱂ', u'Ⱂ'}, // 2c42 → 2c12 + {u'ⱃ', u'Ⱃ'}, // 2c43 → 2c13 + {u'ⱄ', u'Ⱄ'}, // 2c44 → 2c14 + {u'ⱅ', u'Ⱅ'}, // 2c45 → 2c15 + {u'ⱆ', u'Ⱆ'}, // 2c46 → 2c16 + {u'ⱇ', u'Ⱇ'}, // 2c47 → 2c17 + {u'ⱈ', u'Ⱈ'}, // 2c48 → 2c18 + {u'ⱉ', u'Ⱉ'}, // 2c49 → 2c19 + {u'ⱊ', u'Ⱊ'}, // 2c4a → 2c1a + {u'ⱋ', u'Ⱋ'}, // 2c4b → 2c1b + {u'ⱌ', u'Ⱌ'}, // 2c4c → 2c1c + {u'ⱍ', u'Ⱍ'}, // 2c4d → 2c1d + {u'ⱎ', u'Ⱎ'}, // 2c4e → 2c1e + {u'ⱏ', u'Ⱏ'}, // 2c4f → 2c1f + {u'ⱐ', u'Ⱐ'}, // 2c50 → 2c20 + {u'ⱑ', u'Ⱑ'}, // 2c51 → 2c21 + {u'ⱒ', u'Ⱒ'}, // 2c52 → 2c22 + {u'ⱓ', u'Ⱓ'}, // 2c53 → 2c23 + {u'ⱔ', u'Ⱔ'}, // 2c54 → 2c24 + {u'ⱕ', u'Ⱕ'}, // 2c55 → 2c25 + {u'ⱖ', u'Ⱖ'}, // 2c56 → 2c26 + {u'ⱗ', u'Ⱗ'}, // 2c57 → 2c27 + {u'ⱘ', u'Ⱘ'}, // 2c58 → 2c28 + {u'ⱙ', u'Ⱙ'}, // 2c59 → 2c29 + {u'ⱚ', u'Ⱚ'}, // 2c5a → 2c2a + {u'ⱛ', u'Ⱛ'}, // 2c5b → 2c2b + {u'ⱜ', u'Ⱜ'}, // 2c5c → 2c2c + {u'ⱝ', u'Ⱝ'}, // 2c5d → 2c2d + {u'ⱞ', u'Ⱞ'}, // 2c5e → 2c2e + {u'ⱡ', u'Ⱡ'}, // 2c61 → 2c60 + {u'ⱥ', u'Ⱥ'}, // 2c65 → 023a + {u'ⱦ', u'Ⱦ'}, // 2c66 → 023e + {u'ⱨ', u'Ⱨ'}, // 2c68 → 2c67 + {u'ⱪ', u'Ⱪ'}, // 2c6a → 2c69 + {u'ⱬ', u'Ⱬ'}, // 2c6c → 2c6b + {u'ⱳ', u'Ⱳ'}, // 2c73 → 2c72 + {u'ⱶ', u'Ⱶ'}, // 2c76 → 2c75 + {u'ⲁ', u'Ⲁ'}, // 2c81 → 2c80 + {u'ⲃ', u'Ⲃ'}, // 2c83 → 2c82 + {u'ⲅ', u'Ⲅ'}, // 2c85 → 2c84 + {u'ⲇ', u'Ⲇ'}, // 2c87 → 2c86 + {u'ⲉ', u'Ⲉ'}, // 2c89 → 2c88 + {u'ⲋ', u'Ⲋ'}, // 2c8b → 2c8a + {u'ⲍ', u'Ⲍ'}, // 2c8d → 2c8c + {u'ⲏ', u'Ⲏ'}, // 2c8f → 2c8e + {u'ⲑ', u'Ⲑ'}, // 2c91 → 2c90 + {u'ⲓ', u'Ⲓ'}, // 2c93 → 2c92 + {u'ⲕ', u'Ⲕ'}, // 2c95 → 2c94 + {u'ⲗ', u'Ⲗ'}, // 2c97 → 2c96 + {u'ⲙ', u'Ⲙ'}, // 2c99 → 2c98 + {u'ⲛ', u'Ⲛ'}, // 2c9b → 2c9a + {u'ⲝ', u'Ⲝ'}, // 2c9d → 2c9c + {u'ⲟ', u'Ⲟ'}, // 2c9f → 2c9e + {u'ⲡ', u'Ⲡ'}, // 2ca1 → 2ca0 + {u'ⲣ', u'Ⲣ'}, // 2ca3 → 2ca2 + {u'ⲥ', u'Ⲥ'}, // 2ca5 → 2ca4 + {u'ⲧ', u'Ⲧ'}, // 2ca7 → 2ca6 + {u'ⲩ', u'Ⲩ'}, // 2ca9 → 2ca8 + {u'ⲫ', u'Ⲫ'}, // 2cab → 2caa + {u'ⲭ', u'Ⲭ'}, // 2cad → 2cac + {u'ⲯ', u'Ⲯ'}, // 2caf → 2cae + {u'ⲱ', u'Ⲱ'}, // 2cb1 → 2cb0 + {u'ⲳ', u'Ⲳ'}, // 2cb3 → 2cb2 + {u'ⲵ', u'Ⲵ'}, // 2cb5 → 2cb4 + {u'ⲷ', u'Ⲷ'}, // 2cb7 → 2cb6 + {u'ⲹ', u'Ⲹ'}, // 2cb9 → 2cb8 + {u'ⲻ', u'Ⲻ'}, // 2cbb → 2cba + {u'ⲽ', u'Ⲽ'}, // 2cbd → 2cbc + {u'ⲿ', u'Ⲿ'}, // 2cbf → 2cbe + {u'ⳁ', u'Ⳁ'}, // 2cc1 → 2cc0 + {u'ⳃ', u'Ⳃ'}, // 2cc3 → 2cc2 + {u'ⳅ', u'Ⳅ'}, // 2cc5 → 2cc4 + {u'ⳇ', u'Ⳇ'}, // 2cc7 → 2cc6 + {u'ⳉ', u'Ⳉ'}, // 2cc9 → 2cc8 + {u'ⳋ', u'Ⳋ'}, // 2ccb → 2cca + {u'ⳍ', u'Ⳍ'}, // 2ccd → 2ccc + {u'ⳏ', u'Ⳏ'}, // 2ccf → 2cce + {u'ⳑ', u'Ⳑ'}, // 2cd1 → 2cd0 + {u'ⳓ', u'Ⳓ'}, // 2cd3 → 2cd2 + {u'ⳕ', u'Ⳕ'}, // 2cd5 → 2cd4 + {u'ⳗ', u'Ⳗ'}, // 2cd7 → 2cd6 + {u'ⳙ', u'Ⳙ'}, // 2cd9 → 2cd8 + {u'ⳛ', u'Ⳛ'}, // 2cdb → 2cda + {u'ⳝ', u'Ⳝ'}, // 2cdd → 2cdc + {u'ⳟ', u'Ⳟ'}, // 2cdf → 2cde + {u'ⳡ', u'Ⳡ'}, // 2ce1 → 2ce0 + {u'ⳣ', u'Ⳣ'}, // 2ce3 → 2ce2 + {u'ⳬ', u'Ⳬ'}, // 2cec → 2ceb + {u'ⳮ', u'Ⳮ'}, // 2cee → 2ced + {u'ⳳ', u'Ⳳ'}, // 2cf3 → 2cf2 + {u'ⴀ', u'Ⴀ'}, // 2d00 → 10a0 + {u'ⴁ', u'Ⴁ'}, // 2d01 → 10a1 + {u'ⴂ', u'Ⴂ'}, // 2d02 → 10a2 + {u'ⴃ', u'Ⴃ'}, // 2d03 → 10a3 + {u'ⴄ', u'Ⴄ'}, // 2d04 → 10a4 + {u'ⴅ', u'Ⴅ'}, // 2d05 → 10a5 + {u'ⴆ', u'Ⴆ'}, // 2d06 → 10a6 + {u'ⴇ', u'Ⴇ'}, // 2d07 → 10a7 + {u'ⴈ', u'Ⴈ'}, // 2d08 → 10a8 + {u'ⴉ', u'Ⴉ'}, // 2d09 → 10a9 + {u'ⴊ', u'Ⴊ'}, // 2d0a → 10aa + {u'ⴋ', u'Ⴋ'}, // 2d0b → 10ab + {u'ⴌ', u'Ⴌ'}, // 2d0c → 10ac + {u'ⴍ', u'Ⴍ'}, // 2d0d → 10ad + {u'ⴎ', u'Ⴎ'}, // 2d0e → 10ae + {u'ⴏ', u'Ⴏ'}, // 2d0f → 10af + {u'ⴐ', u'Ⴐ'}, // 2d10 → 10b0 + {u'ⴑ', u'Ⴑ'}, // 2d11 → 10b1 + {u'ⴒ', u'Ⴒ'}, // 2d12 → 10b2 + {u'ⴓ', u'Ⴓ'}, // 2d13 → 10b3 + {u'ⴔ', u'Ⴔ'}, // 2d14 → 10b4 + {u'ⴕ', u'Ⴕ'}, // 2d15 → 10b5 + {u'ⴖ', u'Ⴖ'}, // 2d16 → 10b6 + {u'ⴗ', u'Ⴗ'}, // 2d17 → 10b7 + {u'ⴘ', u'Ⴘ'}, // 2d18 → 10b8 + {u'ⴙ', u'Ⴙ'}, // 2d19 → 10b9 + {u'ⴚ', u'Ⴚ'}, // 2d1a → 10ba + {u'ⴛ', u'Ⴛ'}, // 2d1b → 10bb + {u'ⴜ', u'Ⴜ'}, // 2d1c → 10bc + {u'ⴝ', u'Ⴝ'}, // 2d1d → 10bd + {u'ⴞ', u'Ⴞ'}, // 2d1e → 10be + {u'ⴟ', u'Ⴟ'}, // 2d1f → 10bf + {u'ⴠ', u'Ⴠ'}, // 2d20 → 10c0 + {u'ⴡ', u'Ⴡ'}, // 2d21 → 10c1 + {u'ⴢ', u'Ⴢ'}, // 2d22 → 10c2 + {u'ⴣ', u'Ⴣ'}, // 2d23 → 10c3 + {u'ⴤ', u'Ⴤ'}, // 2d24 → 10c4 + {u'ⴥ', u'Ⴥ'}, // 2d25 → 10c5 + {u'ⴧ', u'Ⴧ'}, // 2d27 → 10c7 + {u'ⴭ', u'Ⴭ'}, // 2d2d → 10cd + {u'ꙁ', u'Ꙁ'}, // a641 → a640 + {u'ꙃ', u'Ꙃ'}, // a643 → a642 + {u'ꙅ', u'Ꙅ'}, // a645 → a644 + {u'ꙇ', u'Ꙇ'}, // a647 → a646 + {u'ꙉ', u'Ꙉ'}, // a649 → a648 + {u'ꙋ', u'Ꙋ'}, // a64b → a64a + {u'ꙍ', u'Ꙍ'}, // a64d → a64c + {u'ꙏ', u'Ꙏ'}, // a64f → a64e + {u'ꙑ', u'Ꙑ'}, // a651 → a650 + {u'ꙓ', u'Ꙓ'}, // a653 → a652 + {u'ꙕ', u'Ꙕ'}, // a655 → a654 + {u'ꙗ', u'Ꙗ'}, // a657 → a656 + {u'ꙙ', u'Ꙙ'}, // a659 → a658 + {u'ꙛ', u'Ꙛ'}, // a65b → a65a + {u'ꙝ', u'Ꙝ'}, // a65d → a65c + {u'ꙟ', u'Ꙟ'}, // a65f → a65e + {u'ꙡ', u'Ꙡ'}, // a661 → a660 + {u'ꙣ', u'Ꙣ'}, // a663 → a662 + {u'ꙥ', u'Ꙥ'}, // a665 → a664 + {u'ꙧ', u'Ꙧ'}, // a667 → a666 + {u'ꙩ', u'Ꙩ'}, // a669 → a668 + {u'ꙫ', u'Ꙫ'}, // a66b → a66a + {u'ꙭ', u'Ꙭ'}, // a66d → a66c + {u'ꚁ', u'Ꚁ'}, // a681 → a680 + {u'ꚃ', u'Ꚃ'}, // a683 → a682 + {u'ꚅ', u'Ꚅ'}, // a685 → a684 + {u'ꚇ', u'Ꚇ'}, // a687 → a686 + {u'ꚉ', u'Ꚉ'}, // a689 → a688 + {u'ꚋ', u'Ꚋ'}, // a68b → a68a + {u'ꚍ', u'Ꚍ'}, // a68d → a68c + {u'ꚏ', u'Ꚏ'}, // a68f → a68e + {u'ꚑ', u'Ꚑ'}, // a691 → a690 + {u'ꚓ', u'Ꚓ'}, // a693 → a692 + {u'ꚕ', u'Ꚕ'}, // a695 → a694 + {u'ꚗ', u'Ꚗ'}, // a697 → a696 + {u'ꚙ', u'Ꚙ'}, // a699 → a698 + {u'ꚛ', u'Ꚛ'}, // a69b → a69a + {u'ꜣ', u'Ꜣ'}, // a723 → a722 + {u'ꜥ', u'Ꜥ'}, // a725 → a724 + {u'ꜧ', u'Ꜧ'}, // a727 → a726 + {u'ꜩ', u'Ꜩ'}, // a729 → a728 + {u'ꜫ', u'Ꜫ'}, // a72b → a72a + {u'ꜭ', u'Ꜭ'}, // a72d → a72c + {u'ꜯ', u'Ꜯ'}, // a72f → a72e + {u'ꜳ', u'Ꜳ'}, // a733 → a732 + {u'ꜵ', u'Ꜵ'}, // a735 → a734 + {u'ꜷ', u'Ꜷ'}, // a737 → a736 + {u'ꜹ', u'Ꜹ'}, // a739 → a738 + {u'ꜻ', u'Ꜻ'}, // a73b → a73a + {u'ꜽ', u'Ꜽ'}, // a73d → a73c + {u'ꜿ', u'Ꜿ'}, // a73f → a73e + {u'ꝁ', u'Ꝁ'}, // a741 → a740 + {u'ꝃ', u'Ꝃ'}, // a743 → a742 + {u'ꝅ', u'Ꝅ'}, // a745 → a744 + {u'ꝇ', u'Ꝇ'}, // a747 → a746 + {u'ꝉ', u'Ꝉ'}, // a749 → a748 + {u'ꝋ', u'Ꝋ'}, // a74b → a74a + {u'ꝍ', u'Ꝍ'}, // a74d → a74c + {u'ꝏ', u'Ꝏ'}, // a74f → a74e + {u'ꝑ', u'Ꝑ'}, // a751 → a750 + {u'ꝑ', u'Ꝧ'}, // a751 → a766 + {u'ꝑ', u'Ꝗ'}, // a751 → a756 + {u'ꝓ', u'Ꝓ'}, // a753 → a752 + {u'ꝕ', u'Ꝕ'}, // a755 → a754 + {u'ꝙ', u'Ꝙ'}, // a759 → a758 + {u'ꝛ', u'Ꝛ'}, // a75b → a75a + {u'ꝝ', u'Ꝝ'}, // a75d → a75c + {u'ꝟ', u'Ꝟ'}, // a75f → a75e + {u'ꝡ', u'Ꝡ'}, // a761 → a760 + {u'ꝣ', u'Ꝣ'}, // a763 → a762 + {u'ꝥ', u'Ꝥ'}, // a765 → a764 + {u'ꝩ', u'Ꝩ'}, // a769 → a768 + {u'ꝫ', u'Ꝫ'}, // a76b → a76a + {u'ꝭ', u'Ꝭ'}, // a76d → a76c + {u'ꝯ', u'Ꝯ'}, // a76f → a76e + {u'ꝺ', u'Ꝺ'}, // a77a → a779 + {u'ꝼ', u'Ꝼ'}, // a77c → a77b + {u'ꝿ', u'Ꝿ'}, // a77f → a77e + {u'ꞁ', u'Ꞁ'}, // a781 → a780 + {u'ꞃ', u'Ꞃ'}, // a783 → a782 + {u'ꞅ', u'Ꞅ'}, // a785 → a784 + {u'ꞇ', u'Ꞇ'}, // a787 → a786 + {u'ꞌ', u'Ꞌ'}, // a78c → a78b + {u'ꞑ', u'Ꞑ'}, // a791 → a790 + {u'ꞓ', u'Ꞓ'}, // a793 → a792 + {u'ꞔ', u'Ꞔ'}, // a794 → a7c4 + {u'ꞗ', u'Ꞗ'}, // a797 → a796 + {u'ꞙ', u'Ꞙ'}, // a799 → a798 + {u'ꞛ', u'Ꞛ'}, // a79b → a79a + {u'ꞝ', u'Ꞝ'}, // a79d → a79c + {u'ꞟ', u'Ꞟ'}, // a79f → a79e + {u'ꞡ', u'Ꞡ'}, // a7a1 → a7a0 + {u'ꞣ', u'Ꞣ'}, // a7a3 → a7a2 + {u'ꞥ', u'Ꞥ'}, // a7a5 → a7a4 + {u'ꞧ', u'Ꞧ'}, // a7a7 → a7a6 + {u'ꞩ', u'Ꞩ'}, // a7a9 → a7a8 + {u'ꞵ', u'Ꞵ'}, // a7b5 → a7b4 + {u'ꞷ', u'Ꞷ'}, // a7b7 → a7b6 + {u'ꞹ', u'Ꞹ'}, // a7b9 → a7b8 + {u'ꞻ', u'Ꞻ'}, // a7bb → a7ba + {u'ꞽ', u'Ꞽ'}, // a7bd → a7bc + {u'ꞿ', u'Ꞿ'}, // a7bf → a7be + {u'ꟃ', u'Ꟃ'}, // a7c3 → a7c2 + {u'ꭓ', u'Ꭓ'}, // ab53 → a7b3 + {u'ａ', u'Ａ'}, // ff41 → ff21 + {u'ｂ', u'Ｂ'}, // ff42 → ff22 + {u'ｃ', u'Ｃ'}, // ff43 → ff23 + {u'ｄ', u'Ｄ'}, // ff44 → ff24 + {u'ｅ', u'Ｅ'}, // ff45 → ff25 + {u'ｆ', u'Ｆ'}, // ff46 → ff26 + {u'ｇ', u'Ｇ'}, // ff47 → ff27 + {u'ｈ', u'Ｈ'}, // ff48 → ff28 + {u'ｉ', u'Ｉ'}, // ff49 → ff29 + {u'ｊ', u'Ｊ'}, // ff4a → ff2a + {u'ｋ', u'Ｋ'}, // ff4b → ff2b + {u'ｌ', u'Ｌ'}, // ff4c → ff2c + {u'ｍ', u'Ｍ'}, // ff4d → ff2d + {u'ｎ', u'Ｎ'}, // ff4e → ff2e + {u'ｏ', u'Ｏ'}, // ff4f → ff2f + {u'ｐ', u'Ｐ'}, // ff50 → ff30 + {u'ｑ', u'Ｑ'}, // ff51 → ff31 + {u'ｒ', u'Ｒ'}, // ff52 → ff32 + {u'ｓ', u'Ｓ'}, // ff53 → ff33 + {u'ｔ', u'Ｔ'}, // ff54 → ff34 + {u'ｕ', u'Ｕ'}, // ff55 → ff35 + {u'ｖ', u'Ｖ'}, // ff56 → ff36 + {u'ｗ', u'Ｗ'}, // ff57 → ff37 + {u'ｘ', u'Ｘ'}, // ff58 → ff38 + {u'ｙ', u'Ｙ'}, // ff59 → ff39 + {u'ｚ', u'Ｚ'}, // ff5a → ff3a +#endif +}; + +static const char32_t kLowerToUpperAstral[][2] = { + {L'𐐨', L'𐐀'}, // 00010428 → 00010400 + {L'𐐩', L'𐐁'}, // 00010429 → 00010401 + {L'𐐪', L'𐐂'}, // 0001042a → 00010402 + {L'𐐫', L'𐐃'}, // 0001042b → 00010403 + {L'𐐬', L'𐐄'}, // 0001042c → 00010404 + {L'𐐭', L'𐐅'}, // 0001042d → 00010405 + {L'𐐮', L'𐐆'}, // 0001042e → 00010406 + {L'𐐯', L'𐐇'}, // 0001042f → 00010407 + {L'𐐰', L'𐐈'}, // 00010430 → 00010408 + {L'𐐱', L'𐐉'}, // 00010431 → 00010409 + {L'𐐲', L'𐐊'}, // 00010432 → 0001040a + {L'𐐳', L'𐐋'}, // 00010433 → 0001040b + {L'𐐴', L'𐐌'}, // 00010434 → 0001040c + {L'𐐵', L'𐐍'}, // 00010435 → 0001040d + {L'𐐶', L'𐐎'}, // 00010436 → 0001040e + {L'𐐷', L'𐐏'}, // 00010437 → 0001040f + {L'𐐸', L'𐐐'}, // 00010438 → 00010410 + {L'𐐹', L'𐐑'}, // 00010439 → 00010411 + {L'𐐺', L'𐐒'}, // 0001043a → 00010412 + {L'𐐻', L'𐐓'}, // 0001043b → 00010413 + {L'𐐼', L'𐐔'}, // 0001043c → 00010414 + {L'𐐽', L'𐐕'}, // 0001043d → 00010415 + {L'𐐾', L'𐐖'}, // 0001043e → 00010416 + {L'𐐿', L'𐐗'}, // 0001043f → 00010417 + {L'𐑀', L'𐐘'}, // 00010440 → 00010418 + {L'𐑁', L'𐐙'}, // 00010441 → 00010419 + {L'𐑂', L'𐐚'}, // 00010442 → 0001041a + {L'𐑃', L'𐐛'}, // 00010443 → 0001041b + {L'𐑄', L'𐐜'}, // 00010444 → 0001041c + {L'𐑅', L'𐐝'}, // 00010445 → 0001041d + {L'𐑆', L'𐐞'}, // 00010446 → 0001041e + {L'𐑇', L'𐐟'}, // 00010447 → 0001041f + {L'𐑈', L'𐐠'}, // 00010448 → 00010420 + {L'𐑉', L'𐐡'}, // 00010449 → 00010421 + {L'𐑊', L'𐐢'}, // 0001044a → 00010422 + {L'𐑋', L'𐐣'}, // 0001044b → 00010423 + {L'𐑌', L'𐐤'}, // 0001044c → 00010424 + {L'𐑍', L'𐐥'}, // 0001044d → 00010425 + {L'𐑎', L'𐐦'}, // 0001044e → 00010426 + {L'𐑏', L'𐐧'}, // 0001044f → 00010427 + {L'𐓘', L'𐒰'}, // 000104d8 → 000104b0 + {L'𐓙', L'𐒱'}, // 000104d9 → 000104b1 + {L'𐓚', L'𐒲'}, // 000104da → 000104b2 + {L'𐓛', L'𐒳'}, // 000104db → 000104b3 + {L'𐓜', L'𐒴'}, // 000104dc → 000104b4 + {L'𐓝', L'𐒵'}, // 000104dd → 000104b5 + {L'𐓞', L'𐒶'}, // 000104de → 000104b6 + {L'𐓟', L'𐒷'}, // 000104df → 000104b7 + {L'𐓠', L'𐒸'}, // 000104e0 → 000104b8 + {L'𐓡', L'𐒹'}, // 000104e1 → 000104b9 + {L'𐓢', L'𐒺'}, // 000104e2 → 000104ba + {L'𐓣', L'𐒻'}, // 000104e3 → 000104bb + {L'𐓤', L'𐒼'}, // 000104e4 → 000104bc + {L'𐓥', L'𐒽'}, // 000104e5 → 000104bd + {L'𐓦', L'𐒾'}, // 000104e6 → 000104be + {L'𐓧', L'𐒿'}, // 000104e7 → 000104bf + {L'𐓨', L'𐓀'}, // 000104e8 → 000104c0 + {L'𐓩', L'𐓁'}, // 000104e9 → 000104c1 + {L'𐓪', L'𐓂'}, // 000104ea → 000104c2 + {L'𐓫', L'𐓃'}, // 000104eb → 000104c3 + {L'𐓬', L'𐓄'}, // 000104ec → 000104c4 + {L'𐓭', L'𐓅'}, // 000104ed → 000104c5 + {L'𐓮', L'𐓆'}, // 000104ee → 000104c6 + {L'𐓯', L'𐓇'}, // 000104ef → 000104c7 + {L'𐓰', L'𐓈'}, // 000104f0 → 000104c8 + {L'𐓱', L'𐓉'}, // 000104f1 → 000104c9 + {L'𐓲', L'𐓊'}, // 000104f2 → 000104ca + {L'𐓳', L'𐓋'}, // 000104f3 → 000104cb + {L'𐓴', L'𐓌'}, // 000104f4 → 000104cc + {L'𐓵', L'𐓍'}, // 000104f5 → 000104cd + {L'𐓶', L'𐓎'}, // 000104f6 → 000104ce + {L'𐓷', L'𐓏'}, // 000104f7 → 000104cf + {L'𐓸', L'𐓐'}, // 000104f8 → 000104d0 + {L'𐓹', L'𐓑'}, // 000104f9 → 000104d1 + {L'𐓺', L'𐓒'}, // 000104fa → 000104d2 + {L'𐓻', L'𐓓'}, // 000104fb → 000104d3 + {L'𐳀', L'𐲀'}, // 00010cc0 → 00010c80 + {L'𐳁', L'𐲁'}, // 00010cc1 → 00010c81 + {L'𐳂', L'𐲉'}, // 00010cc2 → 00010c89 + {L'𐳂', L'𐲂'}, // 00010cc2 → 00010c82 + {L'𐳃', L'𐲃'}, // 00010cc3 → 00010c83 + {L'𐳄', L'𐲄'}, // 00010cc4 → 00010c84 + {L'𐳅', L'𐲅'}, // 00010cc5 → 00010c85 + {L'𐳅', L'𐲙'}, // 00010cc5 → 00010c99 + {L'𐳆', L'𐲆'}, // 00010cc6 → 00010c86 + {L'𐳇', L'𐲇'}, // 00010cc7 → 00010c87 + {L'𐳈', L'𐲈'}, // 00010cc8 → 00010c88 + {L'𐳊', L'𐲊'}, // 00010cca → 00010c8a + {L'𐳋', L'𐲋'}, // 00010ccb → 00010c8b + {L'𐳌', L'𐲌'}, // 00010ccc → 00010c8c + {L'𐳍', L'𐲍'}, // 00010ccd → 00010c8d + {L'𐳎', L'𐲎'}, // 00010cce → 00010c8e + {L'𐳏', L'𐲏'}, // 00010ccf → 00010c8f + {L'𐳐', L'𐲐'}, // 00010cd0 → 00010c90 + {L'𐳑', L'𐲑'}, // 00010cd1 → 00010c91 + {L'𐳒', L'𐲒'}, // 00010cd2 → 00010c92 + {L'𐳓', L'𐲓'}, // 00010cd3 → 00010c93 + {L'𐳔', L'𐲔'}, // 00010cd4 → 00010c94 + {L'𐳕', L'𐲪'}, // 00010cd5 → 00010caa + {L'𐳕', L'𐲕'}, // 00010cd5 → 00010c95 + {L'𐳖', L'𐲖'}, // 00010cd6 → 00010c96 + {L'𐳗', L'𐲗'}, // 00010cd7 → 00010c97 + {L'𐳘', L'𐲘'}, // 00010cd8 → 00010c98 + {L'𐳚', L'𐲚'}, // 00010cda → 00010c9a + {L'𐳛', L'𐲛'}, // 00010cdb → 00010c9b + {L'𐳜', L'𐲜'}, // 00010cdc → 00010c9c + {L'𐳝', L'𐲝'}, // 00010cdd → 00010c9d + {L'𐳞', L'𐲞'}, // 00010cde → 00010c9e + {L'𐳟', L'𐲟'}, // 00010cdf → 00010c9f + {L'𐳠', L'𐲠'}, // 00010ce0 → 00010ca0 + {L'𐳡', L'𐲡'}, // 00010ce1 → 00010ca1 + {L'𐳢', L'𐲢'}, // 00010ce2 → 00010ca2 + {L'𐳣', L'𐲣'}, // 00010ce3 → 00010ca3 + {L'𐳤', L'𐲤'}, // 00010ce4 → 00010ca4 + {L'𐳥', L'𐲥'}, // 00010ce5 → 00010ca5 + {L'𐳦', L'𐲦'}, // 00010ce6 → 00010ca6 + {L'𐳧', L'𐲧'}, // 00010ce7 → 00010ca7 + {L'𐳨', L'𐲨'}, // 00010ce8 → 00010ca8 + {L'𐳩', L'𐲩'}, // 00010ce9 → 00010ca9 + {L'𐳫', L'𐲫'}, // 00010ceb → 00010cab + {L'𐳬', L'𐲬'}, // 00010cec → 00010cac + {L'𐳭', L'𐲭'}, // 00010ced → 00010cad + {L'𐳮', L'𐲮'}, // 00010cee → 00010cae + {L'𐳯', L'𐲯'}, // 00010cef → 00010caf + {L'𐳰', L'𐲰'}, // 00010cf0 → 00010cb0 + {L'𐳱', L'𐲱'}, // 00010cf1 → 00010cb1 + {L'𐳲', L'𐲲'}, // 00010cf2 → 00010cb2 + {L'𑣀', L'𑢠'}, // 000118c0 → 000118a0 + {L'𑣁', L'𑢡'}, // 000118c1 → 000118a1 + {L'𑣂', L'𑢢'}, // 000118c2 → 000118a2 + {L'𑣃', L'𑢣'}, // 000118c3 → 000118a3 + {L'𑣄', L'𑢤'}, // 000118c4 → 000118a4 + {L'𑣅', L'𑢥'}, // 000118c5 → 000118a5 + {L'𑣆', L'𑢦'}, // 000118c6 → 000118a6 + {L'𑣇', L'𑢧'}, // 000118c7 → 000118a7 + {L'𑣈', L'𑢨'}, // 000118c8 → 000118a8 + {L'𑣉', L'𑢩'}, // 000118c9 → 000118a9 + {L'𑣊', L'𑢪'}, // 000118ca → 000118aa + {L'𑣋', L'𑢫'}, // 000118cb → 000118ab + {L'𑣌', L'𑢬'}, // 000118cc → 000118ac + {L'𑣍', L'𑢭'}, // 000118cd → 000118ad + {L'𑣎', L'𑢮'}, // 000118ce → 000118ae + {L'𑣏', L'𑢯'}, // 000118cf → 000118af + {L'𑣐', L'𑢰'}, // 000118d0 → 000118b0 + {L'𑣑', L'𑢱'}, // 000118d1 → 000118b1 + {L'𑣒', L'𑢲'}, // 000118d2 → 000118b2 + {L'𑣓', L'𑢳'}, // 000118d3 → 000118b3 + {L'𑣔', L'𑢴'}, // 000118d4 → 000118b4 + {L'𑣕', L'𑢵'}, // 000118d5 → 000118b5 + {L'𑣖', L'𑢶'}, // 000118d6 → 000118b6 + {L'𑣗', L'𑢷'}, // 000118d7 → 000118b7 + {L'𑣘', L'𑢸'}, // 000118d8 → 000118b8 + {L'𑣙', L'𑢹'}, // 000118d9 → 000118b9 + {L'𑣚', L'𑢺'}, // 000118da → 000118ba + {L'𑣛', L'𑢻'}, // 000118db → 000118bb + {L'𑣜', L'𑢼'}, // 000118dc → 000118bc + {L'𑣝', L'𑢽'}, // 000118dd → 000118bd + {L'𑣞', L'𑢾'}, // 000118de → 000118be + {L'𑣟', L'𑢿'}, // 000118df → 000118bf + {L'𖹠', L'𖹀'}, // 00016e60 → 00016e40 + {L'𖹡', L'𖹁'}, // 00016e61 → 00016e41 + {L'𖹢', L'𖹂'}, // 00016e62 → 00016e42 + {L'𖹣', L'𖹃'}, // 00016e63 → 00016e43 + {L'𖹤', L'𖹄'}, // 00016e64 → 00016e44 + {L'𖹥', L'𖹅'}, // 00016e65 → 00016e45 + {L'𖹦', L'𖹆'}, // 00016e66 → 00016e46 + {L'𖹧', L'𖹇'}, // 00016e67 → 00016e47 + {L'𖹨', L'𖹈'}, // 00016e68 → 00016e48 + {L'𖹩', L'𖹉'}, // 00016e69 → 00016e49 + {L'𖹪', L'𖹊'}, // 00016e6a → 00016e4a + {L'𖹫', L'𖹋'}, // 00016e6b → 00016e4b + {L'𖹬', L'𖹌'}, // 00016e6c → 00016e4c + {L'𖹭', L'𖹍'}, // 00016e6d → 00016e4d + {L'𖹮', L'𖹎'}, // 00016e6e → 00016e4e + {L'𖹯', L'𖹏'}, // 00016e6f → 00016e4f + {L'𖹰', L'𖹐'}, // 00016e70 → 00016e50 + {L'𖹱', L'𖹑'}, // 00016e71 → 00016e51 + {L'𖹲', L'𖹒'}, // 00016e72 → 00016e52 + {L'𖹳', L'𖹓'}, // 00016e73 → 00016e53 + {L'𖹴', L'𖹔'}, // 00016e74 → 00016e54 + {L'𖹵', L'𖹕'}, // 00016e75 → 00016e55 + {L'𖹶', L'𖹖'}, // 00016e76 → 00016e56 + {L'𖹷', L'𖹗'}, // 00016e77 → 00016e57 + {L'𖹸', L'𖹘'}, // 00016e78 → 00016e58 + {L'𖹹', L'𖹙'}, // 00016e79 → 00016e59 + {L'𖹺', L'𖹚'}, // 00016e7a → 00016e5a + {L'𖹻', L'𖹛'}, // 00016e7b → 00016e5b + {L'𖹼', L'𖹜'}, // 00016e7c → 00016e5c + {L'𖹽', L'𖹝'}, // 00016e7d → 00016e5d + {L'𖹾', L'𖹞'}, // 00016e7e → 00016e5e + {L'𖹿', L'𖹟'}, // 00016e7f → 00016e5f + {L'𝐚', L'𝐀'}, // 0001d41a → 0001d400 + {L'𝐛', L'𝐁'}, // 0001d41b → 0001d401 + {L'𝐜', L'𝐂'}, // 0001d41c → 0001d402 + {L'𝐝', L'𝐃'}, // 0001d41d → 0001d403 + {L'𝐞', L'𝐄'}, // 0001d41e → 0001d404 + {L'𝐟', L'𝐅'}, // 0001d41f → 0001d405 + {L'𝐠', L'𝐆'}, // 0001d420 → 0001d406 + {L'𝐡', L'𝐇'}, // 0001d421 → 0001d407 + {L'𝐢', L'𝐈'}, // 0001d422 → 0001d408 + {L'𝐣', L'𝐉'}, // 0001d423 → 0001d409 + {L'𝐤', L'𝐊'}, // 0001d424 → 0001d40a + {L'𝐥', L'𝐋'}, // 0001d425 → 0001d40b + {L'𝐦', L'𝐌'}, // 0001d426 → 0001d40c + {L'𝐧', L'𝐍'}, // 0001d427 → 0001d40d + {L'𝐨', L'𝐎'}, // 0001d428 → 0001d40e + {L'𝐩', L'𝐏'}, // 0001d429 → 0001d40f + {L'𝐪', L'𝐐'}, // 0001d42a → 0001d410 + {L'𝐫', L'𝐑'}, // 0001d42b → 0001d411 + {L'𝐬', L'𝐒'}, // 0001d42c → 0001d412 + {L'𝐭', L'𝐓'}, // 0001d42d → 0001d413 + {L'𝐮', L'𝐔'}, // 0001d42e → 0001d414 + {L'𝐯', L'𝐕'}, // 0001d42f → 0001d415 + {L'𝐰', L'𝐖'}, // 0001d430 → 0001d416 + {L'𝐱', L'𝐗'}, // 0001d431 → 0001d417 + {L'𝐲', L'𝐘'}, // 0001d432 → 0001d418 + {L'𝐳', L'𝐙'}, // 0001d433 → 0001d419 + {L'𝑎', L'𝐴'}, // 0001d44e → 0001d434 + {L'𝑏', L'𝐵'}, // 0001d44f → 0001d435 + {L'𝑐', L'𝐶'}, // 0001d450 → 0001d436 + {L'𝑑', L'𝐷'}, // 0001d451 → 0001d437 + {L'𝑒', L'𝐸'}, // 0001d452 → 0001d438 + {L'𝑓', L'𝐹'}, // 0001d453 → 0001d439 + {L'𝑔', L'𝐺'}, // 0001d454 → 0001d43a + {L'𝑖', L'𝐼'}, // 0001d456 → 0001d43c + {L'𝑗', L'𝐽'}, // 0001d457 → 0001d43d + {L'𝑘', L'𝐾'}, // 0001d458 → 0001d43e + {L'𝑙', L'𝐿'}, // 0001d459 → 0001d43f + {L'𝑚', L'𝑀'}, // 0001d45a → 0001d440 + {L'𝑛', L'𝑁'}, // 0001d45b → 0001d441 + {L'𝑜', L'𝑂'}, // 0001d45c → 0001d442 + {L'𝑝', L'𝑃'}, // 0001d45d → 0001d443 + {L'𝑞', L'𝑄'}, // 0001d45e → 0001d444 + {L'𝑟', L'𝑅'}, // 0001d45f → 0001d445 + {L'𝑠', L'𝑆'}, // 0001d460 → 0001d446 + {L'𝑡', L'𝑇'}, // 0001d461 → 0001d447 + {L'𝑢', L'𝑈'}, // 0001d462 → 0001d448 + {L'𝑣', L'𝑉'}, // 0001d463 → 0001d449 + {L'𝑤', L'𝑊'}, // 0001d464 → 0001d44a + {L'𝑥', L'𝑋'}, // 0001d465 → 0001d44b + {L'𝑦', L'𝑌'}, // 0001d466 → 0001d44c + {L'𝑧', L'𝑍'}, // 0001d467 → 0001d44d + {L'𝒂', L'𝘼'}, // 0001d482 → 0001d63c + {L'𝒂', L'𝑨'}, // 0001d482 → 0001d468 + {L'𝒃', L'𝘽'}, // 0001d483 → 0001d63d + {L'𝒃', L'𝑩'}, // 0001d483 → 0001d469 + {L'𝒄', L'𝘾'}, // 0001d484 → 0001d63e + {L'𝒄', L'𝑪'}, // 0001d484 → 0001d46a + {L'𝒅', L'𝘿'}, // 0001d485 → 0001d63f + {L'𝒅', L'𝑫'}, // 0001d485 → 0001d46b + {L'𝒆', L'𝙀'}, // 0001d486 → 0001d640 + {L'𝒆', L'𝑬'}, // 0001d486 → 0001d46c + {L'𝒇', L'𝙁'}, // 0001d487 → 0001d641 + {L'𝒇', L'𝑭'}, // 0001d487 → 0001d46d + {L'𝒈', L'𝙂'}, // 0001d488 → 0001d642 + {L'𝒈', L'𝑮'}, // 0001d488 → 0001d46e + {L'𝒉', L'𝙃'}, // 0001d489 → 0001d643 + {L'𝒉', L'𝑯'}, // 0001d489 → 0001d46f + {L'𝒊', L'𝙄'}, // 0001d48a → 0001d644 + {L'𝒊', L'𝑰'}, // 0001d48a → 0001d470 + {L'𝒋', L'𝙅'}, // 0001d48b → 0001d645 + {L'𝒋', L'𝑱'}, // 0001d48b → 0001d471 + {L'𝒌', L'𝙆'}, // 0001d48c → 0001d646 + {L'𝒌', L'𝑲'}, // 0001d48c → 0001d472 + {L'𝒍', L'𝙇'}, // 0001d48d → 0001d647 + {L'𝒍', L'𝑳'}, // 0001d48d → 0001d473 + {L'𝒎', L'𝙈'}, // 0001d48e → 0001d648 + {L'𝒎', L'𝑴'}, // 0001d48e → 0001d474 + {L'𝒏', L'𝙉'}, // 0001d48f → 0001d649 + {L'𝒏', L'𝑵'}, // 0001d48f → 0001d475 + {L'𝒐', L'𝑶'}, // 0001d490 → 0001d476 + {L'𝒐', L'𝙊'}, // 0001d490 → 0001d64a + {L'𝒑', L'𝙋'}, // 0001d491 → 0001d64b + {L'𝒑', L'𝑷'}, // 0001d491 → 0001d477 + {L'𝒒', L'𝑸'}, // 0001d492 → 0001d478 + {L'𝒒', L'𝙌'}, // 0001d492 → 0001d64c + {L'𝒓', L'𝙍'}, // 0001d493 → 0001d64d + {L'𝒓', L'𝑹'}, // 0001d493 → 0001d479 + {L'𝒔', L'𝙎'}, // 0001d494 → 0001d64e + {L'𝒔', L'𝑺'}, // 0001d494 → 0001d47a + {L'𝒕', L'𝙏'}, // 0001d495 → 0001d64f + {L'𝒕', L'𝑻'}, // 0001d495 → 0001d47b + {L'𝒖', L'𝙐'}, // 0001d496 → 0001d650 + {L'𝒖', L'𝑼'}, // 0001d496 → 0001d47c + {L'𝒗', L'𝙑'}, // 0001d497 → 0001d651 + {L'𝒗', L'𝑽'}, // 0001d497 → 0001d47d + {L'𝒘', L'𝑾'}, // 0001d498 → 0001d47e + {L'𝒘', L'𝙒'}, // 0001d498 → 0001d652 + {L'𝒙', L'𝑿'}, // 0001d499 → 0001d47f + {L'𝒙', L'𝙓'}, // 0001d499 → 0001d653 + {L'𝒚', L'𝒀'}, // 0001d49a → 0001d480 + {L'𝒚', L'𝙔'}, // 0001d49a → 0001d654 + {L'𝒛', L'𝒁'}, // 0001d49b → 0001d481 + {L'𝒛', L'𝙕'}, // 0001d49b → 0001d655 + {L'𝒶', L'𝒜'}, // 0001d4b6 → 0001d49c + {L'𝒷', L'ℬ'}, // 0001d4b7 → 0000212c + {L'𝒸', L'𝒞'}, // 0001d4b8 → 0001d49e + {L'𝒹', L'𝒟'}, // 0001d4b9 → 0001d49f + {L'𝒻', L'ℱ'}, // 0001d4bb → 00002131 + {L'𝒽', L'ℋ'}, // 0001d4bd → 0000210b + {L'𝒾', L'ℐ'}, // 0001d4be → 00002110 + {L'𝒿', L'𝒥'}, // 0001d4bf → 0001d4a5 + {L'𝓀', L'𝒦'}, // 0001d4c0 → 0001d4a6 + {L'𝓂', L'ℳ'}, // 0001d4c2 → 00002133 + {L'𝓃', L'𝒩'}, // 0001d4c3 → 0001d4a9 + {L'𝓅', L'𝒫'}, // 0001d4c5 → 0001d4ab + {L'𝓆', L'𝒬'}, // 0001d4c6 → 0001d4ac + {L'𝓇', L'ℛ'}, // 0001d4c7 → 0000211b + {L'𝓈', L'𝒮'}, // 0001d4c8 → 0001d4ae + {L'𝓉', L'𝒯'}, // 0001d4c9 → 0001d4af + {L'𝓊', L'𝒰'}, // 0001d4ca → 0001d4b0 + {L'𝓋', L'𝒱'}, // 0001d4cb → 0001d4b1 + {L'𝓌', L'𝒲'}, // 0001d4cc → 0001d4b2 + {L'𝓍', L'𝒳'}, // 0001d4cd → 0001d4b3 + {L'𝓎', L'𝒴'}, // 0001d4ce → 0001d4b4 + {L'𝓏', L'𝒵'}, // 0001d4cf → 0001d4b5 + {L'𝓪', L'𝓐'}, // 0001d4ea → 0001d4d0 + {L'𝓫', L'𝓑'}, // 0001d4eb → 0001d4d1 + {L'𝓬', L'𝓒'}, // 0001d4ec → 0001d4d2 + {L'𝓭', L'𝓓'}, // 0001d4ed → 0001d4d3 + {L'𝓮', L'𝓔'}, // 0001d4ee → 0001d4d4 + {L'𝓯', L'𝓕'}, // 0001d4ef → 0001d4d5 + {L'𝓰', L'𝓖'}, // 0001d4f0 → 0001d4d6 + {L'𝓱', L'𝓗'}, // 0001d4f1 → 0001d4d7 + {L'𝓲', L'𝓘'}, // 0001d4f2 → 0001d4d8 + {L'𝓳', L'𝓙'}, // 0001d4f3 → 0001d4d9 + {L'𝓴', L'𝓚'}, // 0001d4f4 → 0001d4da + {L'𝓵', L'𝓛'}, // 0001d4f5 → 0001d4db + {L'𝓶', L'𝓜'}, // 0001d4f6 → 0001d4dc + {L'𝓷', L'𝓝'}, // 0001d4f7 → 0001d4dd + {L'𝓸', L'𝓞'}, // 0001d4f8 → 0001d4de + {L'𝓹', L'𝓟'}, // 0001d4f9 → 0001d4df + {L'𝓺', L'𝓠'}, // 0001d4fa → 0001d4e0 + {L'𝓻', L'𝓡'}, // 0001d4fb → 0001d4e1 + {L'𝓼', L'𝓢'}, // 0001d4fc → 0001d4e2 + {L'𝓽', L'𝓣'}, // 0001d4fd → 0001d4e3 + {L'𝓾', L'𝓤'}, // 0001d4fe → 0001d4e4 + {L'𝓿', L'𝓥'}, // 0001d4ff → 0001d4e5 + {L'𝔀', L'𝓦'}, // 0001d500 → 0001d4e6 + {L'𝔁', L'𝓧'}, // 0001d501 → 0001d4e7 + {L'𝔂', L'𝓨'}, // 0001d502 → 0001d4e8 + {L'𝔃', L'𝓩'}, // 0001d503 → 0001d4e9 + {L'𝔞', L'𝔄'}, // 0001d51e → 0001d504 + {L'𝔟', L'𝔅'}, // 0001d51f → 0001d505 + {L'𝔡', L'𝔇'}, // 0001d521 → 0001d507 + {L'𝔢', L'𝔈'}, // 0001d522 → 0001d508 + {L'𝔣', L'𝔉'}, // 0001d523 → 0001d509 + {L'𝔤', L'𝔊'}, // 0001d524 → 0001d50a + {L'𝔧', L'𝔍'}, // 0001d527 → 0001d50d + {L'𝔨', L'𝔎'}, // 0001d528 → 0001d50e + {L'𝔩', L'𝔏'}, // 0001d529 → 0001d50f + {L'𝔪', L'𝔐'}, // 0001d52a → 0001d510 + {L'𝔫', L'𝔑'}, // 0001d52b → 0001d511 + {L'𝔬', L'𝔒'}, // 0001d52c → 0001d512 + {L'𝔭', L'𝔓'}, // 0001d52d → 0001d513 + {L'𝔮', L'𝔔'}, // 0001d52e → 0001d514 + {L'𝔰', L'𝔖'}, // 0001d530 → 0001d516 + {L'𝔱', L'𝔗'}, // 0001d531 → 0001d517 + {L'𝔲', L'𝔘'}, // 0001d532 → 0001d518 + {L'𝔳', L'𝔙'}, // 0001d533 → 0001d519 + {L'𝔴', L'𝔚'}, // 0001d534 → 0001d51a + {L'𝔵', L'𝔛'}, // 0001d535 → 0001d51b + {L'𝔶', L'𝔜'}, // 0001d536 → 0001d51c + {L'𝕒', L'𝔸'}, // 0001d552 → 0001d538 + {L'𝕓', L'𝔹'}, // 0001d553 → 0001d539 + {L'𝕔', L'ℂ'}, // 0001d554 → 00002102 + {L'𝕕', L'𝔻'}, // 0001d555 → 0001d53b + {L'𝕖', L'𝔼'}, // 0001d556 → 0001d53c + {L'𝕗', L'𝔽'}, // 0001d557 → 0001d53d + {L'𝕙', L'ℍ'}, // 0001d559 → 0000210d + {L'𝕚', L'𝕀'}, // 0001d55a → 0001d540 + {L'𝕛', L'𝕁'}, // 0001d55b → 0001d541 + {L'𝕜', L'𝕂'}, // 0001d55c → 0001d542 + {L'𝕝', L'𝕃'}, // 0001d55d → 0001d543 + {L'𝕞', L'𝕄'}, // 0001d55e → 0001d544 + {L'𝕟', L'ℕ'}, // 0001d55f → 00002115 + {L'𝕠', L'𝕆'}, // 0001d560 → 0001d546 + {L'𝕢', L'ℚ'}, // 0001d562 → 0000211a + {L'𝕣', L'ℝ'}, // 0001d563 → 0000211d + {L'𝕤', L'𝕊'}, // 0001d564 → 0001d54a + {L'𝕥', L'𝕋'}, // 0001d565 → 0001d54b + {L'𝕦', L'𝕌'}, // 0001d566 → 0001d54c + {L'𝕧', L'𝕍'}, // 0001d567 → 0001d54d + {L'𝕨', L'𝕎'}, // 0001d568 → 0001d54e + {L'𝕩', L'𝕏'}, // 0001d569 → 0001d54f + {L'𝕪', L'𝕐'}, // 0001d56a → 0001d550 + {L'𝕫', L'ℤ'}, // 0001d56b → 00002124 + {L'𝖆', L'𝕬'}, // 0001d586 → 0001d56c + {L'𝖇', L'𝕭'}, // 0001d587 → 0001d56d + {L'𝖈', L'𝕮'}, // 0001d588 → 0001d56e + {L'𝖉', L'𝕯'}, // 0001d589 → 0001d56f + {L'𝖊', L'𝕰'}, // 0001d58a → 0001d570 + {L'𝖋', L'𝕱'}, // 0001d58b → 0001d571 + {L'𝖌', L'𝕲'}, // 0001d58c → 0001d572 + {L'𝖍', L'𝕳'}, // 0001d58d → 0001d573 + {L'𝖎', L'𝕴'}, // 0001d58e → 0001d574 + {L'𝖏', L'𝕵'}, // 0001d58f → 0001d575 + {L'𝖐', L'𝕶'}, // 0001d590 → 0001d576 + {L'𝖑', L'𝕷'}, // 0001d591 → 0001d577 + {L'𝖒', L'𝕸'}, // 0001d592 → 0001d578 + {L'𝖓', L'𝕹'}, // 0001d593 → 0001d579 + {L'𝖔', L'𝕺'}, // 0001d594 → 0001d57a + {L'𝖕', L'𝕻'}, // 0001d595 → 0001d57b + {L'𝖖', L'𝕼'}, // 0001d596 → 0001d57c + {L'𝖗', L'𝕽'}, // 0001d597 → 0001d57d + {L'𝖘', L'𝕾'}, // 0001d598 → 0001d57e + {L'𝖙', L'𝕿'}, // 0001d599 → 0001d57f + {L'𝖚', L'𝖀'}, // 0001d59a → 0001d580 + {L'𝖛', L'𝖁'}, // 0001d59b → 0001d581 + {L'𝖜', L'𝖂'}, // 0001d59c → 0001d582 + {L'𝖝', L'𝖃'}, // 0001d59d → 0001d583 + {L'𝖞', L'𝖄'}, // 0001d59e → 0001d584 + {L'𝖟', L'𝖅'}, // 0001d59f → 0001d585 + {L'𝖺', L'𝖠'}, // 0001d5ba → 0001d5a0 + {L'𝖻', L'𝖡'}, // 0001d5bb → 0001d5a1 + {L'𝖼', L'𝖢'}, // 0001d5bc → 0001d5a2 + {L'𝖽', L'𝖣'}, // 0001d5bd → 0001d5a3 + {L'𝖾', L'𝖤'}, // 0001d5be → 0001d5a4 + {L'𝖿', L'𝖥'}, // 0001d5bf → 0001d5a5 + {L'𝗀', L'𝖦'}, // 0001d5c0 → 0001d5a6 + {L'𝗁', L'𝖧'}, // 0001d5c1 → 0001d5a7 + {L'𝗂', L'𝖨'}, // 0001d5c2 → 0001d5a8 + {L'𝗃', L'𝖩'}, // 0001d5c3 → 0001d5a9 + {L'𝗄', L'𝖪'}, // 0001d5c4 → 0001d5aa + {L'𝗅', L'𝖫'}, // 0001d5c5 → 0001d5ab + {L'𝗆', L'𝖬'}, // 0001d5c6 → 0001d5ac + {L'𝗇', L'𝖭'}, // 0001d5c7 → 0001d5ad + {L'𝗈', L'𝖮'}, // 0001d5c8 → 0001d5ae + {L'𝗉', L'𝖯'}, // 0001d5c9 → 0001d5af + {L'𝗊', L'𝖰'}, // 0001d5ca → 0001d5b0 + {L'𝗋', L'𝖱'}, // 0001d5cb → 0001d5b1 + {L'𝗌', L'𝖲'}, // 0001d5cc → 0001d5b2 + {L'𝗍', L'𝖳'}, // 0001d5cd → 0001d5b3 + {L'𝗎', L'𝖴'}, // 0001d5ce → 0001d5b4 + {L'𝗏', L'𝖵'}, // 0001d5cf → 0001d5b5 + {L'𝗐', L'𝖶'}, // 0001d5d0 → 0001d5b6 + {L'𝗑', L'𝖷'}, // 0001d5d1 → 0001d5b7 + {L'𝗒', L'𝖸'}, // 0001d5d2 → 0001d5b8 + {L'𝗓', L'𝖹'}, // 0001d5d3 → 0001d5b9 + {L'𝗮', L'𝗔'}, // 0001d5ee → 0001d5d4 + {L'𝗯', L'𝗕'}, // 0001d5ef → 0001d5d5 + {L'𝗰', L'𝗖'}, // 0001d5f0 → 0001d5d6 + {L'𝗱', L'𝗗'}, // 0001d5f1 → 0001d5d7 + {L'𝗲', L'𝗘'}, // 0001d5f2 → 0001d5d8 + {L'𝗳', L'𝗙'}, // 0001d5f3 → 0001d5d9 + {L'𝗴', L'𝗚'}, // 0001d5f4 → 0001d5da + {L'𝗵', L'𝗛'}, // 0001d5f5 → 0001d5db + {L'𝗶', L'𝗜'}, // 0001d5f6 → 0001d5dc + {L'𝗷', L'𝗝'}, // 0001d5f7 → 0001d5dd + {L'𝗸', L'𝗞'}, // 0001d5f8 → 0001d5de + {L'𝗹', L'𝗟'}, // 0001d5f9 → 0001d5df + {L'𝗺', L'𝗠'}, // 0001d5fa → 0001d5e0 + {L'𝗻', L'𝗡'}, // 0001d5fb → 0001d5e1 + {L'𝗼', L'𝗢'}, // 0001d5fc → 0001d5e2 + {L'𝗽', L'𝗣'}, // 0001d5fd → 0001d5e3 + {L'𝗾', L'𝗤'}, // 0001d5fe → 0001d5e4 + {L'𝗿', L'𝗥'}, // 0001d5ff → 0001d5e5 + {L'𝘀', L'𝗦'}, // 0001d600 → 0001d5e6 + {L'𝘁', L'𝗧'}, // 0001d601 → 0001d5e7 + {L'𝘂', L'𝗨'}, // 0001d602 → 0001d5e8 + {L'𝘃', L'𝗩'}, // 0001d603 → 0001d5e9 + {L'𝘄', L'𝗪'}, // 0001d604 → 0001d5ea + {L'𝘅', L'𝗫'}, // 0001d605 → 0001d5eb + {L'𝘆', L'𝗬'}, // 0001d606 → 0001d5ec + {L'𝘇', L'𝗭'}, // 0001d607 → 0001d5ed + {L'𝘢', L'𝘈'}, // 0001d622 → 0001d608 + {L'𝘣', L'𝘉'}, // 0001d623 → 0001d609 + {L'𝘤', L'𝘊'}, // 0001d624 → 0001d60a + {L'𝘥', L'𝘋'}, // 0001d625 → 0001d60b + {L'𝘦', L'𝘌'}, // 0001d626 → 0001d60c + {L'𝘧', L'𝘍'}, // 0001d627 → 0001d60d + {L'𝘨', L'𝘎'}, // 0001d628 → 0001d60e + {L'𝘩', L'𝘏'}, // 0001d629 → 0001d60f + {L'𝘪', L'𝘐'}, // 0001d62a → 0001d610 + {L'𝘫', L'𝘑'}, // 0001d62b → 0001d611 + {L'𝘬', L'𝘒'}, // 0001d62c → 0001d612 + {L'𝘭', L'𝘓'}, // 0001d62d → 0001d613 + {L'𝘮', L'𝘔'}, // 0001d62e → 0001d614 + {L'𝘯', L'𝘕'}, // 0001d62f → 0001d615 + {L'𝘰', L'𝘖'}, // 0001d630 → 0001d616 + {L'𝘱', L'𝘗'}, // 0001d631 → 0001d617 + {L'𝘲', L'𝘘'}, // 0001d632 → 0001d618 + {L'𝘳', L'𝘙'}, // 0001d633 → 0001d619 + {L'𝘴', L'𝘚'}, // 0001d634 → 0001d61a + {L'𝘵', L'𝘛'}, // 0001d635 → 0001d61b + {L'𝘶', L'𝘜'}, // 0001d636 → 0001d61c + {L'𝘷', L'𝘝'}, // 0001d637 → 0001d61d + {L'𝘸', L'𝘞'}, // 0001d638 → 0001d61e + {L'𝘹', L'𝘟'}, // 0001d639 → 0001d61f + {L'𝘺', L'𝘠'}, // 0001d63a → 0001d620 + {L'𝘻', L'𝘡'}, // 0001d63b → 0001d621 + {L'𝚊', L'𝙰'}, // 0001d68a → 0001d670 + {L'𝚋', L'𝙱'}, // 0001d68b → 0001d671 + {L'𝚌', L'𝙲'}, // 0001d68c → 0001d672 + {L'𝚍', L'𝙳'}, // 0001d68d → 0001d673 + {L'𝚎', L'𝙴'}, // 0001d68e → 0001d674 + {L'𝚏', L'𝙵'}, // 0001d68f → 0001d675 + {L'𝚐', L'𝙶'}, // 0001d690 → 0001d676 + {L'𝚑', L'𝙷'}, // 0001d691 → 0001d677 + {L'𝚒', L'𝙸'}, // 0001d692 → 0001d678 + {L'𝚓', L'𝙹'}, // 0001d693 → 0001d679 + {L'𝚔', L'𝙺'}, // 0001d694 → 0001d67a + {L'𝚕', L'𝙻'}, // 0001d695 → 0001d67b + {L'𝚖', L'𝙼'}, // 0001d696 → 0001d67c + {L'𝚗', L'𝙽'}, // 0001d697 → 0001d67d + {L'𝚘', L'𝙾'}, // 0001d698 → 0001d67e + {L'𝚙', L'𝙿'}, // 0001d699 → 0001d67f + {L'𝚚', L'𝚀'}, // 0001d69a → 0001d680 + {L'𝚛', L'𝚁'}, // 0001d69b → 0001d681 + {L'𝚜', L'𝚂'}, // 0001d69c → 0001d682 + {L'𝚝', L'𝚃'}, // 0001d69d → 0001d683 + {L'𝚞', L'𝚄'}, // 0001d69e → 0001d684 + {L'𝚟', L'𝚅'}, // 0001d69f → 0001d685 + {L'𝚠', L'𝚆'}, // 0001d6a0 → 0001d686 + {L'𝚡', L'𝚇'}, // 0001d6a1 → 0001d687 + {L'𝚢', L'𝚈'}, // 0001d6a2 → 0001d688 + {L'𝚣', L'𝚉'}, // 0001d6a3 → 0001d689 + {L'𝛂', L'𝚨'}, // 0001d6c2 → 0001d6a8 + {L'𝛃', L'𝚩'}, // 0001d6c3 → 0001d6a9 + {L'𝛄', L'𝚪'}, // 0001d6c4 → 0001d6aa + {L'𝛅', L'𝚫'}, // 0001d6c5 → 0001d6ab + {L'𝛆', L'𝚬'}, // 0001d6c6 → 0001d6ac + {L'𝛇', L'𝚭'}, // 0001d6c7 → 0001d6ad + {L'𝛈', L'𝚮'}, // 0001d6c8 → 0001d6ae + {L'𝛉', L'𝚯'}, // 0001d6c9 → 0001d6af + {L'𝛊', L'𝚰'}, // 0001d6ca → 0001d6b0 + {L'𝛋', L'𝚱'}, // 0001d6cb → 0001d6b1 + {L'𝛌', L'𝚲'}, // 0001d6cc → 0001d6b2 + {L'𝛍', L'𝚳'}, // 0001d6cd → 0001d6b3 + {L'𝛎', L'𝚴'}, // 0001d6ce → 0001d6b4 + {L'𝛏', L'𝚵'}, // 0001d6cf → 0001d6b5 + {L'𝛐', L'𝚶'}, // 0001d6d0 → 0001d6b6 + {L'𝛑', L'𝚷'}, // 0001d6d1 → 0001d6b7 + {L'𝛒', L'𝚸'}, // 0001d6d2 → 0001d6b8 + {L'𝛔', L'𝚺'}, // 0001d6d4 → 0001d6ba + {L'𝛕', L'𝚻'}, // 0001d6d5 → 0001d6bb + {L'𝛖', L'𝚼'}, // 0001d6d6 → 0001d6bc + {L'𝛗', L'𝚽'}, // 0001d6d7 → 0001d6bd + {L'𝛘', L'𝚾'}, // 0001d6d8 → 0001d6be + {L'𝛙', L'𝚿'}, // 0001d6d9 → 0001d6bf + {L'𝛚', L'𝛀'}, // 0001d6da → 0001d6c0 + {L'𝛼', L'𝛢'}, // 0001d6fc → 0001d6e2 + {L'𝛽', L'𝛣'}, // 0001d6fd → 0001d6e3 + {L'𝛾', L'𝛤'}, // 0001d6fe → 0001d6e4 + {L'𝛿', L'𝛥'}, // 0001d6ff → 0001d6e5 + {L'𝜀', L'𝛦'}, // 0001d700 → 0001d6e6 + {L'𝜁', L'𝛧'}, // 0001d701 → 0001d6e7 + {L'𝜂', L'𝛨'}, // 0001d702 → 0001d6e8 + {L'𝜃', L'𝛩'}, // 0001d703 → 0001d6e9 + {L'𝜄', L'𝛪'}, // 0001d704 → 0001d6ea + {L'𝜅', L'𝛫'}, // 0001d705 → 0001d6eb + {L'𝜆', L'𝛬'}, // 0001d706 → 0001d6ec + {L'𝜇', L'𝛭'}, // 0001d707 → 0001d6ed + {L'𝜈', L'𝛮'}, // 0001d708 → 0001d6ee + {L'𝜉', L'𝛯'}, // 0001d709 → 0001d6ef + {L'𝜊', L'𝛰'}, // 0001d70a → 0001d6f0 + {L'𝜋', L'𝛱'}, // 0001d70b → 0001d6f1 + {L'𝜌', L'𝛲'}, // 0001d70c → 0001d6f2 + {L'𝜎', L'𝛴'}, // 0001d70e → 0001d6f4 + {L'𝜏', L'𝛵'}, // 0001d70f → 0001d6f5 + {L'𝜐', L'𝛶'}, // 0001d710 → 0001d6f6 + {L'𝜑', L'𝛷'}, // 0001d711 → 0001d6f7 + {L'𝜒', L'𝛸'}, // 0001d712 → 0001d6f8 + {L'𝜓', L'𝛹'}, // 0001d713 → 0001d6f9 + {L'𝜔', L'𝛺'}, // 0001d714 → 0001d6fa + {L'𝜗', L'𝛳'}, // 0001d717 → 0001d6f3 + {L'𝜶', L'𝜜'}, // 0001d736 → 0001d71c + {L'𝜶', L'𝞐'}, // 0001d736 → 0001d790 + {L'𝜷', L'𝜝'}, // 0001d737 → 0001d71d + {L'𝜷', L'𝞑'}, // 0001d737 → 0001d791 + {L'𝜸', L'𝞒'}, // 0001d738 → 0001d792 + {L'𝜸', L'𝜞'}, // 0001d738 → 0001d71e + {L'𝜹', L'𝜟'}, // 0001d739 → 0001d71f + {L'𝜹', L'𝞓'}, // 0001d739 → 0001d793 + {L'𝜺', L'𝜠'}, // 0001d73a → 0001d720 + {L'𝜺', L'𝞔'}, // 0001d73a → 0001d794 + {L'𝜻', L'𝜡'}, // 0001d73b → 0001d721 + {L'𝜻', L'𝞕'}, // 0001d73b → 0001d795 + {L'𝜼', L'𝜢'}, // 0001d73c → 0001d722 + {L'𝜼', L'𝞖'}, // 0001d73c → 0001d796 + {L'𝜽', L'𝜣'}, // 0001d73d → 0001d723 + {L'𝜽', L'𝞗'}, // 0001d73d → 0001d797 + {L'𝜾', L'𝜤'}, // 0001d73e → 0001d724 + {L'𝜾', L'𝞘'}, // 0001d73e → 0001d798 + {L'𝜿', L'𝜥'}, // 0001d73f → 0001d725 + {L'𝜿', L'𝞙'}, // 0001d73f → 0001d799 + {L'𝝀', L'𝜦'}, // 0001d740 → 0001d726 + {L'𝝀', L'𝞚'}, // 0001d740 → 0001d79a + {L'𝝁', L'𝞛'}, // 0001d741 → 0001d79b + {L'𝝁', L'𝜧'}, // 0001d741 → 0001d727 + {L'𝝂', L'𝞜'}, // 0001d742 → 0001d79c + {L'𝝂', L'𝜨'}, // 0001d742 → 0001d728 + {L'𝝃', L'𝞝'}, // 0001d743 → 0001d79d + {L'𝝃', L'𝜩'}, // 0001d743 → 0001d729 + {L'𝝄', L'𝞞'}, // 0001d744 → 0001d79e + {L'𝝄', L'𝜪'}, // 0001d744 → 0001d72a + {L'𝝅', L'𝞟'}, // 0001d745 → 0001d79f + {L'𝝅', L'𝜫'}, // 0001d745 → 0001d72b + {L'𝝆', L'𝞠'}, // 0001d746 → 0001d7a0 + {L'𝝆', L'𝜬'}, // 0001d746 → 0001d72c + {L'𝝈', L'𝞢'}, // 0001d748 → 0001d7a2 + {L'𝝈', L'𝜮'}, // 0001d748 → 0001d72e + {L'𝝉', L'𝜯'}, // 0001d749 → 0001d72f + {L'𝝉', L'𝞣'}, // 0001d749 → 0001d7a3 + {L'𝝊', L'𝞤'}, // 0001d74a → 0001d7a4 + {L'𝝊', L'𝜰'}, // 0001d74a → 0001d730 + {L'𝝋', L'𝜱'}, // 0001d74b → 0001d731 + {L'𝝋', L'𝞥'}, // 0001d74b → 0001d7a5 + {L'𝝌', L'𝜲'}, // 0001d74c → 0001d732 + {L'𝝌', L'𝞦'}, // 0001d74c → 0001d7a6 + {L'𝝍', L'𝜳'}, // 0001d74d → 0001d733 + {L'𝝍', L'𝞧'}, // 0001d74d → 0001d7a7 + {L'𝝎', L'𝜴'}, // 0001d74e → 0001d734 + {L'𝝎', L'𝞨'}, // 0001d74e → 0001d7a8 + {L'𝝑', L'𝜭'}, // 0001d751 → 0001d72d + {L'𝝰', L'𝝖'}, // 0001d770 → 0001d756 + {L'𝝱', L'𝝗'}, // 0001d771 → 0001d757 + {L'𝝲', L'𝝘'}, // 0001d772 → 0001d758 + {L'𝝳', L'𝝙'}, // 0001d773 → 0001d759 + {L'𝝴', L'𝝚'}, // 0001d774 → 0001d75a + {L'𝝵', L'𝝛'}, // 0001d775 → 0001d75b + {L'𝝶', L'𝝜'}, // 0001d776 → 0001d75c + {L'𝝷', L'𝝝'}, // 0001d777 → 0001d75d + {L'𝝸', L'𝝞'}, // 0001d778 → 0001d75e + {L'𝝹', L'𝝟'}, // 0001d779 → 0001d75f + {L'𝝺', L'𝝠'}, // 0001d77a → 0001d760 + {L'𝝻', L'𝝡'}, // 0001d77b → 0001d761 + {L'𝝼', L'𝝢'}, // 0001d77c → 0001d762 + {L'𝝽', L'𝝣'}, // 0001d77d → 0001d763 + {L'𝝾', L'𝝤'}, // 0001d77e → 0001d764 + {L'𝝿', L'𝝥'}, // 0001d77f → 0001d765 + {L'𝞀', L'𝝦'}, // 0001d780 → 0001d766 + {L'𝞂', L'𝝨'}, // 0001d782 → 0001d768 + {L'𝞃', L'𝝩'}, // 0001d783 → 0001d769 + {L'𝞄', L'𝝪'}, // 0001d784 → 0001d76a + {L'𝞅', L'𝝫'}, // 0001d785 → 0001d76b + {L'𝞆', L'𝝬'}, // 0001d786 → 0001d76c + {L'𝞇', L'𝝭'}, // 0001d787 → 0001d76d + {L'𝞈', L'𝝮'}, // 0001d788 → 0001d76e + {L'𝟅', L'𝞡'}, // 0001d7c5 → 0001d7a1 + {L'𝟅', L'𝝧'}, // 0001d7c5 → 0001d767 + {L'𝟋', L'𝟊'}, // 0001d7cb → 0001d7ca + {L'𞤢', L'𞤀'}, // 0001e922 → 0001e900 + {L'𞤣', L'𞤁'}, // 0001e923 → 0001e901 + {L'𞤤', L'𞤂'}, // 0001e924 → 0001e902 + {L'𞤥', L'𞤃'}, // 0001e925 → 0001e903 + {L'𞤦', L'𞤄'}, // 0001e926 → 0001e904 + {L'𞤧', L'𞤅'}, // 0001e927 → 0001e905 + {L'𞤨', L'𞤆'}, // 0001e928 → 0001e906 + {L'𞤩', L'𞤇'}, // 0001e929 → 0001e907 + {L'𞤪', L'𞤈'}, // 0001e92a → 0001e908 + {L'𞤫', L'𞤉'}, // 0001e92b → 0001e909 + {L'𞤬', L'𞤊'}, // 0001e92c → 0001e90a + {L'𞤭', L'𞤋'}, // 0001e92d → 0001e90b + {L'𞤮', L'𞤌'}, // 0001e92e → 0001e90c + {L'𞤯', L'𞤍'}, // 0001e92f → 0001e90d + {L'𞤰', L'𞤎'}, // 0001e930 → 0001e90e + {L'𞤱', L'𞤏'}, // 0001e931 → 0001e90f + {L'𞤲', L'𞤐'}, // 0001e932 → 0001e910 + {L'𞤳', L'𞤑'}, // 0001e933 → 0001e911 + {L'𞤴', L'𞤒'}, // 0001e934 → 0001e912 + {L'𞤵', L'𞤓'}, // 0001e935 → 0001e913 + {L'𞤶', L'𞤔'}, // 0001e936 → 0001e914 + {L'𞤷', L'𞤕'}, // 0001e937 → 0001e915 + {L'𞤸', L'𞤖'}, // 0001e938 → 0001e916 + {L'𞤹', L'𞤗'}, // 0001e939 → 0001e917 + {L'𞤺', L'𞤘'}, // 0001e93a → 0001e918 + {L'𞤻', L'𞤙'}, // 0001e93b → 0001e919 + {L'𞤼', L'𞤚'}, // 0001e93c → 0001e91a + {L'𞤽', L'𞤛'}, // 0001e93d → 0001e91b + {L'𞤾', L'𞤜'}, // 0001e93e → 0001e91c + {L'𞤿', L'𞤝'}, // 0001e93f → 0001e91d + {L'𞥀', L'𞤞'}, // 0001e940 → 0001e91e + {L'𞥁', L'𞤟'}, // 0001e941 → 0001e91f + {L'𞥂', L'𞤠'}, // 0001e942 → 0001e920 + {L'𞥃', L'𞤡'}, // 0001e943 → 0001e921 +}; /** * Converts wide character to upper case. */ -wint_t towupper(wint_t wc) { - return toupper(wc); +wint_t towupper(wint_t c) { + int m, l, r; + if (isascii(c)) return toupper(c); + if (IsTiny() || c < 0x00010000) { + l = 0; + r = ARRAYLEN(kLowerToUpper) - 1; + while (l <= r) { + m = (l + r) >> 1; + if (kLowerToUpper[m][0] < c) { + l = m + 1; + } else if (kLowerToUpper[m][0] > c) { + r = m - 1; + } else { + return kLowerToUpper[m][1]; + } + } + } else { + l = 0; + r = ARRAYLEN(kLowerToUpperAstral) - 1; + while (l <= r) { + m = (l + r) >> 1; + if (kLowerToUpperAstral[m][0] < c) { + l = m + 1; + } else if (kLowerToUpperAstral[m][0] > c) { + r = m - 1; + } else { + return kLowerToUpperAstral[m][1]; + } + } + } + return c; } diff --git a/libc/str/wcstok.c b/libc/str/wcstok.c new file mode 100644 index 000000000..4cbcf8a3a --- /dev/null +++ b/libc/str/wcstok.c @@ -0,0 +1,61 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ This is free and unencumbered software released into the public domain. │ +│ │ +│ Anyone is free to copy, modify, publish, use, compile, sell, or │ +│ distribute this software, either in source code form or as a compiled │ +│ binary, for any purpose, commercial or non-commercial, and by any │ +│ means. │ +│ │ +│ In jurisdictions that recognize copyright laws, the author or authors │ +│ of this software dedicate any and all copyright interest in the │ +│ software to the public domain. We make this dedication for the benefit │ +│ of the public at large and to the detriment of our heirs and │ +│ successors. We intend this dedication to be an overt act of │ +│ relinquishment in perpetuity of all present and future rights to this │ +│ software under copyright law. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │ +│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │ +│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │ +│ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR │ +│ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, │ +│ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR │ +│ OTHER DEALINGS IN THE SOFTWARE. │ +│ │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/str/str.h" + +/** + * Extracts non-empty tokens from string. + * + * @param s is mutated and should be NULL on subsequent calls + * @param sep is a NUL-terminated set of wchars to consider separators + * @param state tracks progress between calls + * @return pointer to next token or NULL for end + */ +wchar_t *wcstok(wchar_t *s, const wchar_t *sep, wchar_t **state) { + size_t leadingseps, tokenlen; + if (!s) { + s = *state; + if (!s) { + return NULL; + } + } + leadingseps = wcsspn(s, sep); + s += leadingseps; + if (*s) { + tokenlen = wcscspn(s, sep); + if (s[tokenlen]) { + s[tokenlen] = L'\0'; + *state = &s[tokenlen + 1]; + return s; + } else if (tokenlen) { + s[tokenlen] = L'\0'; + *state = NULL; + return s; + } + } + return (*state = NULL); +} diff --git a/libc/nexgen32e/tzcntl.S b/libc/str/wctype.c similarity index 59% rename from libc/nexgen32e/tzcntl.S rename to libc/str/wctype.c index 5d0993b35..c9af081b5 100644 --- a/libc/nexgen32e/tzcntl.S +++ b/libc/str/wctype.c @@ -1,7 +1,7 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,29 +16,44 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/bits/bits.h" #include "libc/macros.internal.h" +#include "libc/str/str.h" -// Finds lowest set bit in 𝑥. -// -// uint32 𝑥 bsf(𝑥) tzcnt(𝑥) ffs(𝑥) bsr(𝑥) lzcnt(𝑥) -// 0x00000000 wut 32 0 wut 32 -// 0x00000001 0 0 1 0 31 -// 0x80000001 0 0 1 31 0 -// 0x80000000 31 31 32 31 0 -// 0x00000010 4 4 5 4 27 -// 0x08000010 4 4 5 27 4 -// 0x08000000 27 27 28 27 4 -// 0xffffffff 0 0 1 31 0 -// -// @param rdi is 64-bit unsigned 𝑥 value -// @return rax number in range [0,64) or 64 if 𝑥 is 0 -// @see also treasure trove of nearly identical functions -tzcntl: .leafprologue - .profilable - mov $64,%esi - bsf %rdi,%rax - cmovz %esi,%eax - .leafepilogue - .endfn tzcntl,globl - .alias tzcntl,tzcntll - .source __FILE__ +static const char kWcTypeNames[][8] = { + "alnum", // + "alpha", // + "blank", // + "cntrl", // + "digit", // + "graph", // + "lower", // + "print", // + "punct", // + "space", // + "upper", // + "xdigit", // +}; + +/** + * Returns number representing character class name. + * + * @param s can be "alnum", "alpha", "blank", "cntrl", "digit", "graph", + * "lower", "print", "punct", "space", "upper", "xdigit" + * @return nonzero id or 0 if not found + */ +wctype_t wctype(const char *s) { + int i; + char b[8]; + for (i = 0; i < 8; ++i) { + b[i] = *s ? *s++ : 0; + } + if (!*s) { + for (i = 0; i < ARRAYLEN(kWcTypeNames); ++i) { + if (READ64LE(b) == READ64LE(kWcTypeNames[i])) { + return i + 1; + } + } + } + return 0; +} diff --git a/libc/str/wmemcmp.c b/libc/str/wmemcmp.c new file mode 100644 index 000000000..cae978ac7 --- /dev/null +++ b/libc/str/wmemcmp.c @@ -0,0 +1,40 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ This is free and unencumbered software released into the public domain. │ +│ │ +│ Anyone is free to copy, modify, publish, use, compile, sell, or │ +│ distribute this software, either in source code form or as a compiled │ +│ binary, for any purpose, commercial or non-commercial, and by any │ +│ means. │ +│ │ +│ In jurisdictions that recognize copyright laws, the author or authors │ +│ of this software dedicate any and all copyright interest in the │ +│ software to the public domain. We make this dedication for the benefit │ +│ of the public at large and to the detriment of our heirs and │ +│ successors. We intend this dedication to be an overt act of │ +│ relinquishment in perpetuity of all present and future rights to this │ +│ software under copyright law. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │ +│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │ +│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │ +│ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR │ +│ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, │ +│ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR │ +│ OTHER DEALINGS IN THE SOFTWARE. │ +│ │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/str/str.h" + +/** + * Compares NUL-terminated wide strings w/ limit. + * + * @param a is first non-null NUL-terminated string pointer + * @param b is second non-null NUL-terminated string pointer + * @return is <0, 0, or >0 based on uint8_t comparison + * @asyncsignalsafe + */ +int wmemcmp(const wchar_t *a, const wchar_t *b, size_t n) { + return wcsncmp(a, b, n); +} diff --git a/libc/stubs/ubsan.S b/libc/stubs/ubsan.S deleted file mode 100644 index d441e81d6..000000000 --- a/libc/stubs/ubsan.S +++ /dev/null @@ -1,311 +0,0 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "ape/macros.internal.h" -.real -.code16 # ∩ .code32 ∩ .code64 -.source __FILE__ - -/** - * @fileoverview Overridable stubs for synthetic sanitizer calls. - */ - -__ubsan_default_options: - ret - .endfn __ubsan_default_options,weak - -__ubsan_get_current_report_data: - ret - .endfn __ubsan_get_current_report_data,weak - -__ubsan_handle_add_overflow: - nop -// fallthrough - .endfn __ubsan_handle_add_overflow,weak - -__ubsan_handle_add_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_add_overflow_abort,weak - -__ubsan_handle_alignment_assumption: - nop -// fallthrough - .endfn __ubsan_handle_alignment_assumption,weak - -__ubsan_handle_alignment_assumption_abort: - nop -// fallthrough - .endfn __ubsan_handle_alignment_assumption_abort,weak - -__ubsan_handle_builtin_unreachable: - nop -// fallthrough - .endfn __ubsan_handle_builtin_unreachable,weak - -__ubsan_handle_builtin_unreachable_abort: - nop -// fallthrough - .endfn __ubsan_handle_builtin_unreachable_abort,weak - -__ubsan_handle_cfi_bad_type: - nop -// fallthrough - .endfn __ubsan_handle_cfi_bad_type,weak - -__ubsan_handle_cfi_bad_type_abort: - nop -// fallthrough - .endfn __ubsan_handle_cfi_bad_type_abort,weak - -__ubsan_handle_cfi_check_fail: - nop -// fallthrough - .endfn __ubsan_handle_cfi_check_fail,weak - -__ubsan_handle_cfi_check_fail_abort: - nop -// fallthrough - .endfn __ubsan_handle_cfi_check_fail_abort,weak - -__ubsan_handle_divrem_overflow: - nop -// fallthrough - .endfn __ubsan_handle_divrem_overflow,weak - -__ubsan_handle_divrem_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_divrem_overflow_abort,weak - -__ubsan_handle_dynamic_type_cache_miss: - nop -// fallthrough - .endfn __ubsan_handle_dynamic_type_cache_miss,weak - -__ubsan_handle_dynamic_type_cache_miss_abort: - nop -// fallthrough - .endfn __ubsan_handle_dynamic_type_cache_miss_abort,weak - -__ubsan_handle_float_cast_overflow: - nop -// fallthrough - .endfn __ubsan_handle_float_cast_overflow,weak - -__ubsan_handle_float_cast_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_float_cast_overflow_abort,weak - -__ubsan_handle_function_type_mismatch: - nop -// fallthrough - .endfn __ubsan_handle_function_type_mismatch,weak - -__ubsan_handle_function_type_mismatch_abort: - nop -// fallthrough - .endfn __ubsan_handle_function_type_mismatch_abort,weak - -__ubsan_handle_implicit_conversion: - nop -// fallthrough - .endfn __ubsan_handle_implicit_conversion,weak - -__ubsan_handle_implicit_conversion_abort: - nop -// fallthrough - .endfn __ubsan_handle_implicit_conversion_abort,weak - -__ubsan_handle_invalid_builtin: - nop -// fallthrough - .endfn __ubsan_handle_invalid_builtin,weak - -__ubsan_handle_invalid_builtin_abort: - nop -// fallthrough - .endfn __ubsan_handle_invalid_builtin_abort,weak - -__ubsan_handle_load_invalid_value: - nop -// fallthrough - .endfn __ubsan_handle_load_invalid_value,weak - -__ubsan_handle_load_invalid_value_abort: - nop -// fallthrough - .endfn __ubsan_handle_load_invalid_value_abort,weak - -__ubsan_handle_missing_return: - nop -// fallthrough - .endfn __ubsan_handle_missing_return,weak - -__ubsan_handle_missing_return_abort: - nop -// fallthrough - .endfn __ubsan_handle_missing_return_abort,weak - -__ubsan_handle_mul_overflow: - nop -// fallthrough - .endfn __ubsan_handle_mul_overflow,weak - -__ubsan_handle_mul_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_mul_overflow_abort,weak - -__ubsan_handle_negate_overflow: - nop -// fallthrough - .endfn __ubsan_handle_negate_overflow,weak - -__ubsan_handle_negate_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_negate_overflow_abort,weak - -__ubsan_handle_nonnull_arg: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_arg,weak - -__ubsan_handle_nonnull_arg_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_arg_abort,weak - -__ubsan_handle_nonnull_return: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return,weak - -__ubsan_handle_nonnull_return_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return_abort,weak - -__ubsan_handle_nonnull_return_v1: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return_v1,weak - -__ubsan_handle_nonnull_return_v1_abort: - nop -// fallthrough - .endfn __ubsan_handle_nonnull_return_v1_abort,weak - -__ubsan_handle_nullability_arg: - nop -// fallthrough - .endfn __ubsan_handle_nullability_arg,weak - -__ubsan_handle_nullability_arg_abort: - nop -// fallthrough - .endfn __ubsan_handle_nullability_arg_abort,weak - -__ubsan_handle_nullability_return_v1: - nop -// fallthrough - .endfn __ubsan_handle_nullability_return_v1,weak - -__ubsan_handle_nullability_return_v1_abort: - nop -// fallthrough - .endfn __ubsan_handle_nullability_return_v1_abort,weak - -__ubsan_handle_out_of_bounds: - nop -// fallthrough - .endfn __ubsan_handle_out_of_bounds,weak - -__ubsan_handle_out_of_bounds_abort: - nop -// fallthrough - .endfn __ubsan_handle_out_of_bounds_abort,weak - -__ubsan_handle_pointer_overflow: - nop -// fallthrough - .endfn __ubsan_handle_pointer_overflow,weak - -__ubsan_handle_pointer_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_pointer_overflow_abort,weak - -__ubsan_handle_shift_out_of_bounds: - nop -// fallthrough - .endfn __ubsan_handle_shift_out_of_bounds,weak - -__ubsan_handle_shift_out_of_bounds_abort: - nop -// fallthrough - .endfn __ubsan_handle_shift_out_of_bounds_abort,weak - -__ubsan_handle_sub_overflow: - nop -// fallthrough - .endfn __ubsan_handle_sub_overflow,weak - -__ubsan_handle_sub_overflow_abort: - nop -// fallthrough - .endfn __ubsan_handle_sub_overflow_abort,weak - -__ubsan_handle_type_mismatch: - nop -// fallthrough - .endfn __ubsan_handle_type_mismatch,weak - -__ubsan_handle_type_mismatch_abort: - nop -// fallthrough - .endfn __ubsan_handle_type_mismatch_abort,weak - -__ubsan_handle_type_mismatch_v1: - nop -// fallthrough - .endfn __ubsan_handle_type_mismatch_v1,weak - -__ubsan_handle_type_mismatch_v1_abort: - nop -// fallthrough - .endfn __ubsan_handle_type_mismatch_v1_abort,weak - -__ubsan_handle_vla_bound_not_positive: - nop -// fallthrough - .endfn __ubsan_handle_vla_bound_not_positive,weak - -__ubsan_handle_vla_bound_not_positive_abort: - nop -// fallthrough - .endfn __ubsan_handle_vla_bound_not_positive_abort,weak - -__ubsan_abort_stub: - push %bp - mov %sp,%bp -1: int3 - jmp 1b - .endfn __ubsan_abort_stub diff --git a/libc/sysv/calls/__sys_fcntl.s b/libc/sysv/calls/__sys_fcntl.s new file mode 100644 index 000000000..672d1118a --- /dev/null +++ b/libc/sysv/calls/__sys_fcntl.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall __sys_fcntl,0x05c05c05c205c048,globl,hidden diff --git a/libc/sysv/calls/sys_fcntl.s b/libc/sysv/calls/sys_fcntl.s deleted file mode 100644 index a33374cd1..000000000 --- a/libc/sysv/calls/sys_fcntl.s +++ /dev/null @@ -1,2 +0,0 @@ -.include "o/libc/sysv/macros.internal.inc" -.scall sys_fcntl,0x05c05c05c205c048,globl,hidden diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 0ae6131af..140445c56 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -22,7 +22,7 @@ dir=libc/sysv/consts # The Fifth Bell System, Community Edition # » catalogue of carnage # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD Windows Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon errno ENOSYS 38 78 78 78 78 1 # system call unavailable; bsd consensus; kNtErrorInvalidFunction syscon errno EPERM 1 1 1 1 1 12 # operation not permitted; unix consensus; kNtErrorInvalidAccess (should be kNtErrorNotOwner but is that mutex only??); raised by accept(2), acct(2), add_key(2), adjtimex(2), arch_prctl(2), bdflush(2), bpf(2), capget(2), chmod(2), chown(2), chroot(2), clock_getres(2), clone(2), copy_file_range(2), create_module(2), delete_module(2), epoll_ctl(2), execve(2), fallocate(2), fanotify_init(2), fcntl(2), futex(2), get_robust_list(2), getdomainname(2), getgroups(2), gethostname(2), getpriority(2), getrlimit(2), getsid(2), gettimeofday(2), idle(2), init_module(2), io_submit(2), ioctl_console(2), ioctl_ficlonerange(2), ioctl_fideduperange(2), ioctl_ns(2), ioctl_tty(2), ioperm(2), iopl(2), ioprio_set(2), kcmp(2), kexec_load(2), keyctl(2), kill(2), link(2), lookup_dcookie(2), madvise(2), mbind(2), membarrier(2), migrate_pages(2), mkdir(2), mknod(2), mlock(2), mmap(2), mount(2), move_pages(2), msgctl(2), nice(2), open(2), open_by_handle_at(2), pciconfig_read(2), perf_event_open(2), pidfd_getfd(2), pidfd_send_signal(2), pivot_root(2), prctl(2), process_vm_readv(2), ptrace(2), quotactl(2), reboot(2), rename(2), request_key(2), rmdir(2), rt_sigqueueinfo(2), sched_setaffinity(2), sched_setattr(2), sched_setparam(2), sched_setscheduler(2), semctl(2), seteuid(2), setfsgid(2), setfsuid(2), setgid(2), setns(2), setpgid(2), setresuid(2), setreuid(2), setsid(2), setuid(2), setup(2), setxattr(2), shmctl(2), shmget(2), sigaltstack(2), spu_create(2), stime(2), swapon(2), symlink(2), syslog(2), timer_create(2), timerfd_create(2), tkill(2), truncate(2), umount(2), unlink(2), unshare(2), utime(2), utimensat(2), vhangup(2), vm86(2), write(2), unix(7), ip(7) syscon errno ENOENT 2 2 2 2 2 2 # no such file or directory; unix consensus; kNtErrorFileNotFound; raised by access(2), acct(2), alloc_hugepages(2), bind(2), bpf(2), chdir(2), chmod(2), chown(2), chroot(2), clock_getres(2), delete_module(2), epoll_ctl(2), execve(2), execveat(2), fanotify_mark(2), getdents(2), inotify_add_watch(2), ioctl_fat(2), kcmp(2), keyctl(2), link(2), mkdir(2), mknod(2), mount(2), msgget(2), open(2), open_by_handle_at(2), perf_event_open(2), query_module(2), quotactl(2), readdir(2), readlink(2), rename(2), rmdir(2), semget(2), shmget(2), spu_create(2), stat(2), statfs(2), statx(2), swapon(2), symlink(2), truncate(2), umount(2), unlink(2), utime(2), utimensat(2), unix(7), ip(7) @@ -106,74 +106,35 @@ syscon errno EBADMSG 74 94 89 92 88 0 # raised by ioctl_getfsmap syscon errno ECANCELED 125 89 85 88 87 0 # raised by timerfd_create(2) syscon errno EOWNERDEAD 130 105 96 94 97 0 # raised by pthread_cond_timedwait(3), pthread_mutex_consistent(3), pthread_mutex_getprioceiling(3), pthread_mutex_lock(3), pthread_mutex_timedlock(3), pthread_mutexattr_getrobust(3), pthread_mutexattr_setrobust(3) syscon errno ENOTRECOVERABLE 131 104 95 93 98 0 # raised by pthread_cond_timedwait(3), pthread_mutex_consistent(3), pthread_mutex_getprioceiling(3), pthread_mutex_lock(3), pthread_mutex_timedlock(3), pthread_mutexattr_getrobust(3), pthread_mutexattr_setrobust(3) -syscon errno ENONET 64 0 0 0 0 0 # bsd consensus; raised by accept(2) -syscon errno ERESTART 85 0 0 0 -3 0 # bsd consensus; should only be seen in ptrace() -syscon junkerr ECHRNG 44 0 0 0 0 0 # bsd consensus -syscon junkerr EL2NSYNC 45 0 0 0 0 0 # bsd consensus -syscon junkerr EL3HLT 46 0 0 0 0 0 # bsd consensus -syscon junkerr EL3RST 47 0 0 0 0 0 # bsd consensus -syscon junkerr ELNRNG 48 0 0 0 0 0 # bsd consensus -syscon junkerr EUNATCH 49 0 0 0 0 0 # bsd consensus -syscon junkerr ENOCSI 50 0 0 0 0 0 # bsd consensus -syscon junkerr EL2HLT 51 0 0 0 0 0 # bsd consensus -syscon junkerr EBADE 52 0 0 0 0 0 # bsd consensus -syscon junkerr EBADR 53 0 0 0 0 0 # bsd consensus -syscon junkerr EXFULL 54 0 0 0 0 0 # bsd consensus -syscon junkerr ENOANO 55 0 0 0 0 0 # bsd consensus -syscon junkerr EBADRQC 56 0 0 0 0 0 # bsd consensus -syscon junkerr EBADSLT 57 0 0 0 0 0 # bsd consensus -syscon junkerr ENOSTR 60 99 0 0 91 0 # -syscon junkerr ENODATA 61 96 0 0 89 0 # raised by getxattr(2), removexattr(2), setxattr(2) -syscon junkerr ENOSR 63 98 0 0 90 0 # -syscon junkerr ENOPKG 65 0 0 0 0 0 # bsd consensus, ip(7) -syscon junkerr ENOLINK 67 97 91 0 95 0 # -syscon junkerr EADV 68 0 0 0 0 0 # bsd consensus -syscon junkerr ESRMNT 69 0 0 0 0 0 # bsd consensus -syscon junkerr ECOMM 70 0 0 0 0 0 # bsd consensus -syscon junkerr EMULTIHOP 72 95 90 0 94 0 # -syscon junkerr EDOTDOT 73 0 0 0 0 0 # bsd consensus -syscon junkerr ENOTUNIQ 76 0 0 0 0 0 # bsd consensus -syscon junkerr EREMCHG 78 0 0 0 0 0 # bsd consensus -syscon junkerr ELIBACC 79 0 0 0 0 0 # bsd consensus -syscon junkerr ELIBBAD 80 0 0 0 0 0 # bsd consensus -syscon junkerr ELIBSCN 81 0 0 0 0 0 # bsd consensus -syscon junkerr ELIBMAX 82 0 0 0 0 0 # bsd consensus -syscon junkerr ELIBEXEC 83 0 0 0 0 0 # bsd consensus -syscon junkerr ESTRPIPE 86 0 0 0 0 0 # bsd consensus -syscon junkerr EUCLEAN 117 0 0 0 0 0 # bsd consensus -syscon junkerr ENOTNAM 118 0 0 0 0 0 # bsd consensus -syscon junkerr ENAVAIL 119 0 0 0 0 0 # bsd consensus -syscon junkerr EISNAM 120 0 0 0 0 0 # bsd consensus -syscon junkerr EREMOTEIO 121 0 0 0 0 0 # bsd consensus -syscon junkerr ENOMEDIUM 123 0 0 85 85 0 # -syscon junkerr EMEDIUMTYPE 124 0 0 86 86 0 # -syscon junkerr ENOKEY 126 0 0 0 0 0 # bsd consensus -syscon junkerr EKEYEXPIRED 127 0 0 0 0 0 # bsd consensus -syscon junkerr EKEYREVOKED 128 0 0 0 0 0 # bsd consensus -syscon junkerr EKEYREJECTED 129 0 0 0 0 0 # bsd consensus -syscon junkerr ERFKILL 132 0 0 0 0 0 # bsd consensus -syscon junkerr EHWPOISON 133 0 0 0 0 0 # bsd consensus -syscon junkerr EBADFD 77 9 9 9 9 6 # file descriptor in bad state; cf. EBADF; fudged on non-Linux -syscon compat EWOULDBLOCK 11 35 35 35 35 10035 # same as EWOULDBLOCK +syscon errno ENONET 64 0 0 0 0 0 # unilateral; raised by accept(2) +syscon errno ERESTART 85 -1 -1 -1 -3 0 # should only be seen in ptrace() +syscon errno ENOSR 63 98 0 90 90 0 # out of streams resources; something like EAGAIN; it's in POSIX; maybe some commercial UNIX returns it with openat, putmsg, putpmsg, posix_openpt, ioctl, open +syscon errno ENOSTR 60 99 0 0 91 0 # not a stream; returned by getmsg, putmsg, putpmsg, getpmsg +syscon errno ENODATA 61 96 0 0 89 0 # no data available; barely in posix; returned by ioctl +syscon errno EMULTIHOP 72 95 90 0 94 0 # barely in posix +syscon errno ENOLINK 67 97 91 0 95 0 # barely in posix +syscon errno ENOMEDIUM 123 0 0 85 0 0 # not posix; not documented +syscon errno EMEDIUMTYPE 124 0 0 86 0 0 # not posix; not documented +syscon compat EWOULDBLOCK 11 35 35 35 35 10035 # same as EAGAIN on every platform we've seen # signals # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon sig SIGHUP 1 1 1 1 1 1 # terminal hangup or daemon reload; resumable; auto-broadcasted to process group; unix consensus & faked on nt -syscon sig SIGINT 2 2 2 2 2 2 # terminal ctrl-c keystroke; resumable; auto-broadcasted to process group; unix consensus & faked on nt +syscon sig SIGINT 2 2 2 2 2 2 # terminal ctrl-c keystroke; resumable; auto-broadcasted to process group; unix consensus & faked on nt; X3.159-1988 syscon sig SIGQUIT 3 3 3 3 3 3 # terminal ctrl-\ keystroke; resumable; unix consensus & faked on nt -syscon sig SIGILL 4 4 4 4 4 4 # illegal instruction; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt +syscon sig SIGILL 4 4 4 4 4 4 # illegal instruction; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt; X3.159-1988 syscon sig SIGTRAP 5 5 5 5 5 5 # int3 instruction; resumable; unix consensus & faked on nt -syscon sig SIGABRT 6 6 6 6 6 6 # process aborted; resumable; unix consensus & faked on nt +syscon sig SIGABRT 6 6 6 6 6 6 # process aborted; resumable; unix consensus & faked on nt; X3.159-1988 syscon sig SIGBUS 7 10 10 10 10 7 # valid memory access that went beyond underlying end of file; bsd consensus -syscon sig SIGFPE 8 8 8 8 8 8 # illegal math; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt +syscon sig SIGFPE 8 8 8 8 8 8 # illegal math; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt; X3.159-1988 syscon sig SIGKILL 9 9 9 9 9 9 # terminate with extreme prejudice; unreceivable; unix consensus & faked on nt syscon sig SIGUSR1 10 30 30 30 30 10 # do whatever you want; bsd consensus -syscon sig SIGSEGV 11 11 11 11 11 11 # invalid memory access; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt +syscon sig SIGSEGV 11 11 11 11 11 11 # invalid memory access; unresumable (unless you longjmp() or edit ucontex->rip+=ild(ucontex->rip)); unix consensus & faked on nt; X3.159-1988 syscon sig SIGUSR2 12 31 31 31 31 12 # do whatever you want; bsd consensus syscon sig SIGPIPE 13 13 13 13 13 13 # write to closed file descriptor; unix consensus & faked on nt syscon sig SIGALRM 14 14 14 14 14 14 # sent by setitimer(2) or timer_settime(2); unix consensus & faked on nt -syscon sig SIGTERM 15 15 15 15 15 15 # terminate; resumable; unix consensus & faked on nt +syscon sig SIGTERM 15 15 15 15 15 15 # terminate; resumable; unix consensus & faked on nt; X3.159-1988 syscon sig SIGCHLD 17 20 20 20 20 17 # child process exited or terminated and is now a zombie (unless this is SIG_IGN or SA_NOCLDWAIT) or child process stopped due to terminal i/o or profiling/debugging (unless you used SA_NOCLDSTOP); bsd consensus syscon sig SIGCONT 18 19 19 19 19 18 # child process resumed from profiling/debugging; bsd consensus syscon sig SIGSTOP 19 17 17 17 17 19 # child process stopped due to profiling/debugging; bsd consensus @@ -205,36 +166,37 @@ syscon open O_RDONLY 0 0 0 0 0 0xA0000000 # unix consensus & kNtGe syscon open O_WRONLY 1 1 1 1 1 0x40000000 # unix consensus & kNtGenericWrite syscon open O_RDWR 2 2 2 2 2 0xE0000000 # unix consensus & kNtGenericRead|kNtGenericWrite|kNtGenericExecute syscon open O_ACCMODE 3 3 3 3 3 0xE0000000 # O_RDONLY|O_WRONLY|O_RDWR -syscon open O_APPEND 0x0400 8 8 8 8 0x00000004 # bsd consensus & kNtFileAppendData; won't pose issues w/ mknod(S_IFIFO) -syscon open O_CREAT 0x40 0x0200 0x0200 0x0200 0x0200 0x00000040 # bsd consensus & NT faked as Linux -syscon open O_EXCL 0x80 0x0800 0x0800 0x0800 0x0800 0x00000080 # bsd consensus & NT faked as Linux -syscon open O_TRUNC 0x0200 0x0400 0x0400 0x0400 0x0400 0x00000200 # bsd consensus & NT faked as Linux -syscon open O_DIRECTORY 0x010000 0x100000 0x020000 0x020000 0x200000 0x02000000 # kNtFileFlagBackupSemantics -syscon open O_DIRECT 0x4000 0 0x010000 0 0x80000 0x00200000 # kNtFileFlagNoBuffering>>8 -syscon open O_CLOEXEC 0x080000 0x01000000 0x100000 0x010000 0x400000 0x00080000 # NT faked as Linux -syscon open O_TMPFILE 0x410000 0 0 0 0 0x04000100 # Linux 3.11+ (c. 2013) & kNtFileAttributeTemporary|kNtFileFlagDeleteOnClose +syscon open O_APPEND 0x00000400 8 8 8 8 0x00000004 # bsd consensus & kNtFileAppendData; won't pose issues w/ mknod(S_IFIFO) +syscon open O_CREAT 0x00000040 0x00000200 0x00000200 0x00000200 0x00000200 0x00000040 # bsd consensus & NT faked as Linux +syscon open O_EXCL 0x00000080 0x00000800 0x00000800 0x00000800 0x00000800 0x00000080 # bsd consensus & NT faked as Linux +syscon open O_TRUNC 0x00000200 0x00000400 0x00000400 0x00000400 0x00000400 0x00000200 # bsd consensus & NT faked as Linux +syscon open O_DIRECTORY 0x00010000 0x00100000 0x00020000 0x00020000 0x00200000 0x02000000 # kNtFileFlagBackupSemantics +syscon open O_DIRECT 0x00004000 0 0x00010000 0 0x00080000 0x00200000 # kNtFileFlagNoBuffering>>8 +syscon open O_CLOEXEC 0x00080000 0x01000000 0x00100000 0x00010000 0x00400000 0x00080000 # NT faked as Linux +syscon open O_TMPFILE 0x00410000 0 0 0 0 0x04000100 # Linux 3.11+ (c. 2013) & kNtFileAttributeTemporary|kNtFileFlagDeleteOnClose syscon open O_SPARSE 0 0 0 0 0 0x00040000 # we invented it -syscon open O_NDELAY 0x0800 4 4 4 4 0 # bsd consensus & kNtFileFlagWriteThrough>>8 → 0x00800000 (???) -syscon open O_NONBLOCK 0x0800 4 4 4 4 0x00000800 # bsd consensus & faked on nt to be same as linux -syscon open O_ASYNC 0x2000 0x40 0x40 0x40 0x40 0 # bsd consensus -syscon open O_NOFOLLOW 0x020000 0x0100 0x0100 0x0100 0x0100 0 # bsd consensus -syscon open O_SYNC 0x101000 0x80 0x80 0x80 0x80 0 # bsd consensus -syscon open O_NOCTTY 0x0100 0x020000 0x8000 0x8000 0x8000 0 # used for remote viewing (default behavior on freebsd) -syscon open O_DSYNC 0x1000 0x400000 0 0x80 0x10000 0 -syscon open O_RSYNC 0x101000 0 0 0x80 0x20000 0 -syscon open O_NOATIME 0x040000 0 0 0 0 0 -syscon open O_PATH 0x200000 0 0 0 0 0 -syscon open O_EXEC 0 0 0x040000 0 0x4000000 0 -syscon open O_TTY_INIT 0 0 0x080000 0 0 0 +syscon open O_NDELAY 0x00000800 0x00000004 0x00000004 0x00000004 0x00000004 0x00000800 # bsd consensus & kNtFileFlagWriteThrough>>8 → 0x00800000 (???) +syscon open O_NONBLOCK 0x00000800 0x00000004 0x00000004 0x00000004 0x00000004 0x00000800 # bsd consensus & faked on nt to be same as linux +syscon open O_ASYNC 0x00002000 0x00000040 0x00000040 0x00000040 0x00000040 0 # bsd consensus +syscon open O_NOFOLLOW 0x00020000 0x00000100 0x00000100 0x00000100 0x00000100 0 # bsd consensus +syscon open O_SYNC 0x00101000 0x00000080 0x00000080 0x00000080 0x00000080 0 # bsd consensus +syscon open O_NOCTTY 0x00000100 0x00020000 0x00008000 0x00008000 0x00008000 0 # used for remote viewing (default behavior on freebsd) +syscon open O_NOATIME 0x00040000 0 0 0 0 0 # optimize away access time update +syscon open O_EXEC 0 0 0x00040000 0 0x04000000 0 # it's specified by posix what does it mean +syscon open O_DSYNC 0x00001000 0x00400000 0 0x00000080 0x00010000 0 +syscon open O_RSYNC 0x00101000 0 0 0x00000080 0x00020000 0 +syscon open O_PATH 0x00200000 0 0 0 0 0 +syscon open O_TTY_INIT 0 0 0x00080000 0 0 0 syscon compat O_LARGEFILE 0 0 0 0 0 0 # mmap() flags # the revolutionary praxis of malloc() # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon compat MAP_FILE 0 0 0 0 0 0 # consensus syscon mmap MAP_SHARED 1 1 1 1 1 1 # forced consensus & faked nt syscon mmap MAP_PRIVATE 2 2 2 2 2 2 # forced consensus & faked nt +syscon mmap MAP_TYPE 15 15 15 15 15 15 # mask for type of mapping syscon mmap MAP_FIXED 0x10 0x10 0x10 0x10 0x10 0x10 # unix consensus; openbsd appears to forbid; faked nt syscon mmap MAP_ANONYMOUS 0x20 0x1000 0x1000 0x1000 0x1000 0x20 # bsd consensus; faked nt syscon mmap MAP_GROWSDOWN 0x0100 0 0x0400 0x4000 0x4000 0x100000 # mandatory for OpenBSD stacks; MAP_STACK on Free/OpenBSD; MEM_TOP_DOWN on NT @@ -246,7 +208,7 @@ syscon mmap MAP_HUGE_SHIFT 26 0 0 0 0 0 syscon mmap MAP_LOCKED 0x2000 0 0 0 0 0 syscon mmap MAP_NONBLOCK 0x10000 0 0 0 0 0 syscon mmap MAP_POPULATE 0x8000 0 0 0 0 0 # can avoid madvise(MADV_WILLNEED) on private file mapping -syscon mmap MAP_TYPE 15 0 0 0 0 0 # what is it +syscon mmap MAP_CONCEAL 0 0 0 0x8000 0 0 # omit from dumps syscon compat MAP_STACK 0x0100 0 0x0400 0x4000 0x2000 0x100000 # use MAP_GROWSDOWN syscon compat MAP_NOCORE 0 0 0x20000 0x8000 0x8000 0 # use MAP_CONCEAL syscon compat MAP_ANON 0x20 0x1000 0x1000 0x1000 0x1000 0x20 # bsd consensus; faked nt @@ -257,7 +219,7 @@ syscon compat MAP_32BIT 0x40 0 0x080000 0 0 0 # iffy # madvise() flags # beneath the iceberg memory management # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon madv MADV_NORMAL 0 0 0 0 0 0x00000080 # consensus & kNtFileAttributeNormal syscon compat POSIX_FADV_NORMAL 0 0 0 0 0 0x00000080 # consensus & kNtFileAttributeNormal syscon compat POSIX_MADV_NORMAL 0 0 0 0 0 0x00000080 # consensus & kNtFileAttributeNormal @@ -280,7 +242,7 @@ syscon madv MADV_HUGEPAGE 14 0 0 0 0 0 # TODO(jart): why would we syscon madv MADV_NOHUGEPAGE 15 0 0 0 0 0 # TODO(jart): why would we need it? syscon madv MADV_DODUMP 17 0 0 0 0 0 # TODO(jart): what is it? syscon madv MADV_DOFORK 11 0 0 0 0 0 # TODO(jart): what is it? -syscon madv MADV_DONTDUMP 16 0 0 0 0 0 # TODO(jart): what is it? +syscon madv MADV_DONTDUMP 16 0 0 0 0 0 # see MAP_CONCEAL in OpenBSD; TODO(jart): what is it? syscon madv MADV_DONTFORK 10 0 0 0 0 0 # TODO(jart): what is it? syscon madv MADV_HWPOISON 100 0 0 0 0 0 # TODO(jart): what is it? syscon madv MADV_REMOVE 9 0 0 0 0 0 # TODO(jart): what is it? @@ -289,7 +251,7 @@ syscon fadv POSIX_FADV_NOREUSE 5 0 5 0 5 0 # wut # mmap(), mprotect(), etc. # digital restrictions management for the people # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon mprot PROT_NONE 0 0 0 0 0 0 # mmap, mprotect, unix consensus (nt needs special business logic here) syscon mprot PROT_READ 1 1 1 1 1 1 # mmap, mprotect, unix consensus syscon mprot PROT_WRITE 2 2 2 2 2 2 # mmap, mprotect, unix consensus @@ -300,13 +262,21 @@ syscon mprot PROT_GROWSUP 0x02000000 0 0 0 0 0 # intended for mpro # mremap() flags # the revolutionary praxis of realloc() # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon mremap MREMAP_MAYMOVE 1 1 1 1 1 1 # faked non-linux (b/c linux only) syscon mremap MREMAP_FIXED 2 2 2 2 2 2 # faked non-linux (b/c linux only) +# sigprocmask() flags +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon misc SIG_BLOCK 0 1 1 1 1 0 # bsd consensus; faked nt +syscon misc SIG_UNBLOCK 1 2 2 2 2 1 # bsd consensus; faked nt +syscon misc SIG_SETMASK 2 3 3 3 3 2 # bsd consensus; faked nt +syscon misc SIG_ATOMIC_MIN -2147483648 -2147483648 -9223372036854775808 -2147483648 -2147483648 0 + # splice() flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon splice SPLICE_F_MOVE 1 0 0 0 0 0 # can be safely ignored by polyfill; it's a hint syscon splice SPLICE_F_NONBLOCK 2 0 0 0 0 0 # can be safely ignored by polyfill, since linux says it doesn't apply to underlying FDs syscon splice SPLICE_F_MORE 4 0 0 0 0 0 # can be safely ignored by polyfill; it's a hint @@ -314,7 +284,7 @@ syscon splice SPLICE_F_GIFT 8 0 0 0 0 0 # can probably be ignored # access() flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon access F_OK 0 0 0 0 0 0 # consensus syscon access X_OK 1 1 1 1 1 0xa0000000 # unix consensus and kNtGenericExecute | kNtGenericRead syscon access W_OK 2 2 2 2 2 0x40000000 # unix consensus and kNtGenericWrite @@ -322,29 +292,30 @@ syscon access R_OK 4 4 4 4 4 0x80000000 # unix consensus and kNtG # flock() flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon lock LOCK_SH 1 1 1 1 1 0 # shared [unix consensus] -syscon lock LOCK_EX 2 2 2 2 2 2 # exclusive [consensus!] -syscon lock LOCK_NB 4 4 4 4 4 1 # non-blocking [unix consensus] +syscon lock LOCK_EX 2 2 2 2 2 2 # exclusive [consensus!] a.k.a. kNtLockfileExclusiveLock +syscon lock LOCK_NB 4 4 4 4 4 1 # non-blocking [unix consensus] a.k.a. kNtLockfileFailImmediately syscon lock LOCK_UN 8 8 8 8 8 8 # unlock [unix consensus & faked NT] # waitpid() / wait4() options # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon waitpid WNOHANG 1 1 1 1 1 0 # helps you reap zombies; unix consensus syscon waitpid WUNTRACED 2 2 2 2 2 0 # unix consensus syscon waitpid WCONTINUED 8 0x10 4 8 16 0 # waitid() options +# no dice on openbsd >:\ # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon waitid WEXITED 4 4 0x10 0 32 0 syscon waitid WSTOPPED 2 8 2 0 2 0 syscon waitid WNOWAIT 0x01000000 0x20 8 0 0x10000 0 # stat::st_mode constants # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon stat S_IFREG 0100000 0100000 0100000 0100000 0100000 0100000 # regular file (unix consensus; faked nt) syscon stat S_IFBLK 0060000 0060000 0060000 0060000 0060000 0060000 # block device (unix consensus; faked nt) syscon stat S_IFCHR 0020000 0020000 0020000 0020000 0020000 0020000 # character device (unix consensus; faked nt) @@ -374,7 +345,7 @@ syscon stat S_IRWXO 0000007 0000007 0000007 0000007 0000007 000000 # fcntl() # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon fcntl2 F_DUPFD 0 0 0 0 0 0 # consensus syscon fcntl2 F_GETFD 1 1 1 1 1 1 # unix consensus & faked nt @@ -393,19 +364,22 @@ syscon fcntl2 F_SETFL 4 4 4 4 4 4 # unix consensus & faked nt syscon fcntl2 F_SETOWN 8 6 6 6 6 0 # bsd consensus syscon fcntl2 F_GETOWN 9 5 5 5 5 0 # bsd consensus -syscon fcntl F_ULOCK 0 0 0 0 0 0 # consensus -syscon fcntl F_RDLCK 0 1 1 1 1 0 # bsd consensus +# fcntl() POSIX Advisory Locks +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon fcntl F_SETLK 6 8 12 8 8 6 # polyfilled nt +syscon compat F_SETLK64 6 8 12 8 8 6 # polyfilled nt +syscon fcntl F_SETLKW 7 9 13 9 9 7 +syscon compat F_SETLKW64 7 9 13 9 9 7 +syscon fcntl F_GETLK 5 7 11 7 7 5 # polyfilled nt +syscon compat F_GETLK64 5 7 11 7 7 5 # polyfilled nt +syscon fcntl F_RDLCK 0 1 1 1 1 0 # polyfilled nt; bsd consensus +syscon fcntl F_WRLCK 1 3 3 3 3 1 # polyfilled nt; bsd consensus +syscon fcntl F_UNLCK 2 2 2 2 2 2 # polyfilled nt; unix consensus + syscon fcntl F_LOCK 1 1 1 1 1 0 # unix consensus -syscon fcntl F_WRLCK 1 3 3 3 3 0 # bsd consensus syscon fcntl F_TLOCK 2 2 2 2 2 0 # unix consensus -syscon fcntl F_UNLCK 2 2 2 2 2 0 # unix consensus syscon fcntl F_TEST 3 3 3 3 3 0 # unix consensus -syscon fcntl F_GETLK 5 7 11 7 7 0 -syscon fcntl F_SETLK 6 8 12 8 8 0 -syscon fcntl F_SETLKW 7 9 13 9 9 0 -syscon fcntl F_GETLK64 5 0 0 0 0 0 -syscon fcntl F_SETLK64 6 0 0 0 0 0 -syscon fcntl F_SETLKW64 7 0 0 0 0 0 syscon fcntl F_SETSIG 10 0 0 0 0 0 syscon fcntl F_GETSIG 11 0 0 0 0 0 syscon fcntl F_SETOWN_EX 15 0 0 0 0 0 @@ -418,10 +392,19 @@ syscon fcntl F_GETLEASE 0x0401 0 0 0 0 0 syscon fcntl F_NOTIFY 0x0402 0 0 0 0 0 syscon fcntl F_SETPIPE_SZ 0x0407 0 0 0 0 0 syscon fcntl F_GETPIPE_SZ 0x0408 0 0 0 0 0 +syscon fcntl F_ULOCK 0 0 0 0 0 0 # TODO: specified by posix but not kernels? + +syscon ioctl FIONBIO 0x5421 0x8004667e 0x8004667e 0x8004667e 0x8004667e 0x8004667e # BSD-The New Technology consensus; FIONBIO is traditional O_NONBLOCK; see F_SETFL for re-imagined api +syscon ioctl FIOASYNC 0x5452 0x8004667d 0x8004667d 0x8004667d 0x8004667d 0x8004667d # BSD-The New Technology consensus +syscon ioctl FIONREAD 0x541b 0x4004667f 0x4004667f 0x4004667f 0x4004667f 0x4004667f # BSD-The New Technology consensus; bytes waiting in FD's input buffer +#syscon ioctl FIONWRITE 0x0 0x0 0x40046677 0x0 0x0 -1 # [FreeBSD Generalization] bytes queued in FD's output buffer (same as TIOCOUTQ for TTY FDs; see also SO_SNDBUF) +#syscon ioctl FIONSPACE 0x0 0x0 0x40046676 0x0 0x0 -1 # [FreeBSD Generalization] capacity of FD's output buffer, e.g. equivalent to TIOCGSERIAL w/ UART +syscon ioctl TIOCINQ 0x541b 0x4004667f 0x4004667f 0x4004667f 0x4004667f 0x4004667f # [Linuxism] same as FIONREAD +#syscon ioctl TIOCOUTQ 0x5411 0x40047473 0x40047473 0x40047473 0x40047473 -1 # bytes queued in TTY's output buffer # openat(), fstatat(), linkat(), etc. magnums # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon at AT_FDCWD -100 -2 -100 -100 -100 -100 # faked nt syscon at AT_SYMLINK_FOLLOW 0x0400 0x40 0x0400 4 0x400 0 syscon at AT_SYMLINK_NOFOLLOW 0x0100 0x20 0x0200 2 0x200 0 # TODO(jart): What should NT do? @@ -434,13 +417,13 @@ syscon at AT_EMPTY_PATH 0x1000 0 0 0 0 0 # linux 2.6.39+; see unl # # Unsupported flags are encoded as 0. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon memfd MFD_CLOEXEC 1 0 0 0 0 0 syscon memfd MFD_ALLOW_SEALING 2 0 0 0 0 0 # utimensat() special values # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon utime UTIME_NOW 0x3fffffff 0x3fffffff -1 -2 0x3fffffff -2 # timespec::tv_sec may be this; polyfilled xnu/nt syscon utime UTIME_OMIT 0x3ffffffe 0x3ffffffe -2 -1 0x3ffffffe -1 # timespec::tv_nsec may be this; polyfilled xnu/nt @@ -448,7 +431,7 @@ syscon utime UTIME_OMIT 0x3ffffffe 0x3ffffffe -2 -1 0x3ffffffe -1 # # # Unsupported values are encoded as 0. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon auxv AT_EXECFD 2 0 2 0 2 0 # file descriptor of program syscon auxv AT_PHDR 3 0 3 0 3 0 # address of program headers of executable syscon auxv AT_PHENT 4 0 4 0 4 0 @@ -481,7 +464,7 @@ syscon auxv AT_NO_AUTOMOUNT 0x0800 0 0 0 0 0 # # Unsupported values are encoded as 127. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon rlimit RLIMIT_CPU 0 0 0 0 0 127 # max cpu time in seconds; see SIGXCPU; unix consensus syscon rlimit RLIMIT_FSIZE 1 1 1 1 1 127 # max file size in bytes; unix consensus syscon rlimit RLIMIT_DATA 2 2 2 2 2 127 # max mmap() / brk() / sbrk() size in bytes; unix consensus @@ -501,7 +484,7 @@ syscon compat RLIMIT_VMEM 9 5 10 127 10 127 # same as RLIMIT_AS # resource limit special values # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon rlim RLIM_NLIMITS 16 9 15 9 12 0 # no clue why we need it syscon rlim RLIM_INFINITY 0xffffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0 syscon rlim RLIM_SAVED_CUR 0xffffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0x7fffffffffffffff 0 @@ -509,7 +492,7 @@ syscon rlim RLIM_SAVED_MAX 0xffffffffffffffff 0x7fffffffffffffff 0x7fffffffff # sigaction() codes # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon sigact SA_NOCLDSTOP 1 8 8 8 8 1 # lets you set SIGCHLD handler that's only notified on exit/termination and not notified on SIGSTOP/SIGTSTP/SIGTTIN/SIGTTOU/SIGCONT lool; bsd consensus syscon sigact SA_NOCLDWAIT 2 32 32 32 32 2 # changes SIGCHLD so the zombie is gone and you can't call wait(2) anymore; similar to SIGCHLD+SIG_IGN but may still deliver the SIGCHLD; bsd consensus syscon sigact SA_SIGINFO 4 64 64 64 64 4 # asks kernel to provide ucontext_t argument, which has mutable cpu/fpu state of signalled process; and it is polyfilled by cosmopolitan; bsd consensus @@ -522,10 +505,10 @@ syscon compat SA_ONESHOT 0x80000000 4 4 4 4 0x80000000 # same as SA # siginfo::si_code values # -# Windows NT is polyfilled as Linux. +# The New Technology NT is polyfilled as Linux. # Unsupported values are encoded as 0x80000000. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon sicode SI_USER 0 0x010001 0x010001 0 0 0 # sent by kill(2); openbsd defines si_code<=0 as originating from user syscon sicode SI_QUEUE -1 0x010002 0x010002 -2 -1 -1 # sent by sigqueue(2) syscon sicode SI_TIMER -2 0x010003 0x010003 -3 -2 -2 # sent by setitimer(2) or clock_settime(2) @@ -575,12 +558,12 @@ syscon sicode POLL_HUP 6 6 6 6 6 6 # SIGIO; device disconnected; # sigalstack() values # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon ss SIGSTKSZ 0x2000 0x020000 0x8800 0x7000 0x7000 0x2000 # clock_{gettime,settime} timers # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon clock CLOCK_REALTIME 0 0 0 0 0 0 # consensus syscon clock CLOCK_MONOTONIC 1 1 4 3 3 1 # XNU/NT faked; could move backwards if NTP introduces negative leap second syscon clock CLOCK_PROCESS_CPUTIME_ID 2 -1 15 2 0x40000000 -1 @@ -595,7 +578,7 @@ syscon clock CLOCK_TAI 11 -1 -1 -1 -1 -1 # bsd consensus # poll() # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon poll POLLIN 1 1 1 1 1 0x300 # unix consensus syscon poll POLLPRI 2 2 2 2 2 0x0400 # unix consensus syscon poll POLLOUT 4 4 4 4 4 0x10 # unix consensus @@ -610,7 +593,7 @@ syscon poll POLLRDHUP 0x2000 0x10 0x10 0x10 0x10 2 # bsd consensu # epoll # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon epoll EPOLL_CLOEXEC 0x080000 0x01000000 0x100000 0x010000 0x010000 0x80000 # O_CLOEXEC syscon epoll EPOLL_CTL_ADD 1 1 1 1 1 1 # forced consensus, linux only natively, polyfilled elsewhere syscon epoll EPOLL_CTL_DEL 2 2 2 2 2 2 # forced consensus, linux only natively, polyfilled elsewhere @@ -636,7 +619,7 @@ syscon epoll EPOLLET 0x80000000 0x80000000 0x80000000 0x80000000 0x80000 # * 0 we define as EINVAL # * -1 we define as no-op # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon so SO_REUSEPORT 15 0x0200 0x0200 0x0200 0x0200 4 # bsd consensus (NT calls it SO_REUSEADDR) syscon so SO_REUSEADDR 2 4 4 4 4 0 # bsd consensus (default behavior on NT) syscon so SO_KEEPALIVE 9 8 8 8 8 8 # bsd consensus @@ -732,7 +715,7 @@ syscon sol SOL_X25 262 0 0 0 0 0 # @see https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt # @see https://www.iana.org/assignments/tcp-parameters/tcp-parameters.txt # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon tcp TCP_NODELAY 1 1 1 1 1 1 # strong consensus for disabling nagle's algorithm; so be sure to disable it by turning this on syscon tcp TCP_CORK 3 4 4 16 0 0 # nagle's algorithm strikes again; TCP_NOPUSH on BSD; be sure to turn it off; protip: mmap+writev vs. write+sendfile; see also /proc/sys/net/ipv4/tcp_autocorking; netbsd is 4 but not implemented syscon tcp TCP_MAXSEG 2 2 2 2 2 0 # reduces tcp segment size; see also tcp offloading @@ -763,7 +746,7 @@ syscon tcp TCP_REPAIR_OPTIONS 22 0 0 0 0 0 # what is it syscon tcp TCP_REPAIR_QUEUE 20 0 0 0 0 0 # what is it syscon tcp TCP_THIN_LINEAR_TIMEOUTS 16 0 0 0 0 0 # what is it -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon ip IP_DEFAULT_MULTICAST_LOOP 1 1 1 1 1 1 # consensus syscon ip IP_DEFAULT_MULTICAST_TTL 1 1 1 1 1 1 # consensus syscon ip IP_PMTUDISC_DONT 0 0 0 0 0 0 # consensus @@ -816,7 +799,7 @@ syscon ip INET_ADDRSTRLEN 0x10 0x10 0x10 0x10 0x10 22 # unix cons # ptrace() codes # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon ptrace PTRACE_TRACEME 0 0 0 0 -1 -1 # unix consensus a.k.a. PT_TRACE_ME syscon ptrace PTRACE_PEEKTEXT 1 1 1 1 -1 -1 # unix consensus a.k.a. PT_READ_I syscon ptrace PTRACE_PEEKDATA 2 2 2 2 -1 -1 # unix consensus a.k.a. PT_READ_D @@ -897,11 +880,6 @@ syscon iproto IPPROTO_BEETPH 94 0 0 0 0 0 syscon iproto IPPROTO_COMP 108 0 0 0 0 0 syscon iproto IPPROTO_DCCP 33 0 0 0 0 0 -syscon misc EXTA 14 0x4b00 0x4b00 0x4b00 0x4b00 0 # bsd consensus -syscon misc EXTB 15 0x9600 0x9600 0x9600 0x9600 0 # bsd consensus -syscon misc ERA 0x02002c 45 45 0 0 0 -syscon misc EMPTY 0 0 0 0 0 0 # consensus - syscon pr PR_SET_PTRACER_ANY -1 0 0 0 0 0 syscon pr PR_ENDIAN_BIG 0 0 0 0 0 0 # consensus syscon pr PR_FP_EXC_DISABLED 0 0 0 0 0 0 # consensus @@ -1005,209 +983,6 @@ syscon pr PR_FP_EXC_RES 0x080000 0 0 0 0 0 syscon pr PR_FP_EXC_INV 0x100000 0 0 0 0 0 syscon pr PR_SET_PTRACER 0x59616d61 0 0 0 0 0 -syscon log LOG_EMERG 0 0 0 0 0 0 # consensus -syscon log LOG_KERN 0 0 0 0 0 0 # consensus -syscon log LOG_ALERT 1 1 1 1 1 1 # unix consensus -syscon log LOG_PID 1 1 1 1 1 1 # unix consensus -syscon log LOG_CONS 2 2 2 2 2 2 # unix consensus -syscon log LOG_CRIT 2 2 2 2 2 2 # unix consensus -syscon log LOG_ERR 3 3 3 3 3 3 # unix consensus -syscon log LOG_ODELAY 4 4 4 4 4 4 # unix consensus -syscon log LOG_WARNING 4 4 4 4 4 4 # unix consensus -syscon log LOG_NOTICE 5 5 5 5 5 5 # unix consensus -syscon log LOG_INFO 6 6 6 6 6 6 # unix consensus -syscon log LOG_DEBUG 7 7 7 7 7 7 # unix consensus -syscon log LOG_PRIMASK 7 7 7 7 7 7 # unix consensus -syscon log LOG_NDELAY 8 8 8 8 8 8 # unix consensus -syscon log LOG_USER 8 8 8 8 8 8 # unix consensus -syscon log LOG_MAIL 0x10 0x10 0x10 0x10 0x10 0x10 # unix consensus -syscon log LOG_NOWAIT 0x10 0x10 0x10 0x10 0x10 0x10 # unix consensus -syscon log LOG_DAEMON 24 24 24 24 24 24 # unix consensus -syscon log LOG_NFACILITIES 24 25 24 24 24 24 -syscon log LOG_AUTH 0x20 0x20 0x20 0x20 0x20 0x20 # unix consensus -syscon log LOG_PERROR 0x20 0x20 0x20 0x20 0x20 0x20 # unix consensus -syscon log LOG_SYSLOG 40 40 40 40 40 40 # unix consensus -syscon log LOG_LPR 48 48 48 48 48 48 # unix consensus -syscon log LOG_NEWS 56 56 56 56 56 56 # unix consensus -syscon log LOG_UUCP 0x40 0x40 0x40 0x40 0x40 40 # unix consensus -syscon log LOG_CRON 72 72 72 72 72 72 # unix consensus -syscon log LOG_SELECT 76 0 0 0 0 0 -syscon log LOG_SENSE 77 0 0 0 0 0 -syscon log LOG_LOCAL0 0x80 0x80 0x80 0x80 0x80 0x80 # unix consensus -syscon log LOG_LOCAL1 136 136 136 136 136 136 # unix consensus -syscon log LOG_LOCAL2 144 144 144 144 144 144 # unix consensus -syscon log LOG_LOCAL3 152 152 152 152 152 152 # unix consensus -syscon log LOG_LOCAL4 160 160 160 160 160 160 # unix consensus -syscon log LOG_LOCAL5 168 168 168 168 168 168 # unix consensus -syscon log LOG_LOCAL6 176 176 176 176 176 176 # unix consensus -syscon log LOG_LOCAL7 184 184 184 184 184 184 # unix consensus -syscon log LOG_FACMASK 0x03f8 0x03f8 0x03f8 0x03f8 0x03f8 0x03f8 # unix consensus - -syscon sg SG_DXFER_TO_FROM_DEV -4 0 0 0 0 0 -syscon sg SG_DXFER_FROM_DEV -3 0 0 0 0 0 -syscon sg SG_DXFER_TO_DEV -2 0 0 0 0 0 -syscon sg SG_DXFER_NONE -1 0 0 0 0 0 -syscon sg SG_DEF_COMMAND_Q 0 0 0 0 0 0 # consensus -syscon sg SG_DEF_FORCE_LOW_DMA 0 0 0 0 0 0 # consensus -syscon sg SG_DEF_FORCE_PACK_ID 0 0 0 0 0 0 # consensus -syscon sg SG_DEF_KEEP_ORPHAN 0 0 0 0 0 0 # consensus -syscon sg SG_DEF_UNDERRUN_FLAG 0 0 0 0 0 0 # consensus -syscon sg SG_INFO_INDIRECT_IO 0 0 0 0 0 0 # consensus -syscon sg SG_INFO_OK 0 0 0 0 0 0 # consensus -syscon sg SG_SCSI_RESET_NOTHING 0 0 0 0 0 0 # consensus -syscon sg SG_DEFAULT_RETRIES 1 0 0 0 0 0 -syscon sg SG_FLAG_DIRECT_IO 1 0 0 0 0 0 -syscon sg SG_INFO_CHECK 1 0 0 0 0 0 -syscon sg SG_INFO_OK_MASK 1 0 0 0 0 0 -syscon sg SG_SCSI_RESET_DEVICE 1 0 0 0 0 0 -syscon sg SG_FLAG_LUN_INHIBIT 2 0 0 0 0 0 -syscon sg SG_INFO_DIRECT_IO 2 0 0 0 0 0 -syscon sg SG_SCSI_RESET_BUS 2 0 0 0 0 0 -syscon sg SG_SCSI_RESET_HOST 3 0 0 0 0 0 -syscon sg SG_INFO_MIXED_IO 4 0 0 0 0 0 -syscon sg SG_INFO_DIRECT_IO_MASK 6 0 0 0 0 0 -syscon misc VOLUME_OVERFLOW 13 0 0 0 0 0 -syscon sg SG_MAX_QUEUE 0x10 0 0 0 0 0 -syscon sg SG_MAX_SENSE 0x10 0 0 0 0 0 -syscon sg SG_DEFAULT_TIMEOUT 0x1770 0 0 0 0 0 -syscon sg SG_SET_TIMEOUT 0x2201 0 0 0 0 0 -syscon sg SG_GET_TIMEOUT 0x2202 0 0 0 0 0 -syscon sg SG_EMULATED_HOST 0x2203 0 0 0 0 0 -syscon sg SG_SET_TRANSFORM 0x2204 0 0 0 0 0 -syscon sg SG_GET_TRANSFORM 0x2205 0 0 0 0 0 -syscon sg SG_GET_COMMAND_Q 0x2270 0 0 0 0 0 -syscon sg SG_SET_COMMAND_Q 0x2271 0 0 0 0 0 -syscon sg SG_GET_RESERVED_SIZE 0x2272 0 0 0 0 0 -syscon sg SG_SET_RESERVED_SIZE 0x2275 0 0 0 0 0 -syscon sg SG_GET_SCSI_ID 0x2276 0 0 0 0 0 -syscon sg SG_SET_FORCE_LOW_DMA 0x2279 0 0 0 0 0 -syscon sg SG_GET_LOW_DMA 0x227a 0 0 0 0 0 -syscon sg SG_SET_FORCE_PACK_ID 0x227b 0 0 0 0 0 -syscon sg SG_GET_PACK_ID 0x227c 0 0 0 0 0 -syscon sg SG_GET_NUM_WAITING 0x227d 0 0 0 0 0 -syscon sg SG_SET_DEBUG 0x227e 0 0 0 0 0 -syscon sg SG_GET_SG_TABLESIZE 0x227f 0 0 0 0 0 -syscon sg SG_GET_VERSION_NUM 0x2282 0 0 0 0 0 -syscon sg SG_NEXT_CMD_LEN 0x2283 0 0 0 0 0 -syscon sg SG_SCSI_RESET 0x2284 0 0 0 0 0 -syscon sg SG_IO 0x2285 0 0 0 0 0 -syscon sg SG_GET_REQUEST_TABLE 0x2286 0 0 0 0 0 -syscon sg SG_SET_KEEP_ORPHAN 0x2287 0 0 0 0 0 -syscon sg SG_GET_KEEP_ORPHAN 0x2288 0 0 0 0 0 -syscon sg SG_BIG_BUFF 0x8000 0 0 0 0 0 -syscon sg SG_DEF_RESERVED_SIZE 0x8000 0 0 0 0 0 -syscon sg SG_SCATTER_SZ 0x8000 0 0 0 0 0 -syscon sg SG_FLAG_NO_DXFER 0x010000 0 0 0 0 0 - -syscon posix _POSIX_ARG_MAX 0x1000 0x1000 0x1000 0x1000 0x1000 0 # unix consensus -syscon posix _POSIX_CHILD_MAX 25 25 25 25 25 0 # unix consensus -syscon posix _POSIX_HOST_NAME_MAX 255 255 255 255 255 0 # unix consensus -syscon posix _POSIX_LINK_MAX 8 8 8 8 8 0 # unix consensus -syscon posix _POSIX_LOGIN_NAME_MAX 9 9 9 9 9 0 # unix consensus -syscon posix _POSIX_MAX_CANON 255 255 255 255 255 0 # unix consensus -syscon posix _POSIX_MAX_INPUT 255 255 255 255 255 0 # unix consensus -syscon posix _POSIX_NAME_MAX 14 14 14 14 14 14 # forced consensus -syscon posix _POSIX_NGROUPS_MAX 8 8 8 8 8 0 # unix consensus -syscon posix _POSIX_OPEN_MAX 20 20 20 20 20 20 # forced consensus -syscon posix _POSIX_PATH_MAX 255 255 255 255 255 255 # forced consensus -syscon posix _POSIX_PIPE_BUF 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus -syscon posix _POSIX_RE_DUP_MAX 255 255 255 255 255 0 # unix consensus -syscon posix _POSIX_SEM_NSEMS_MAX 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus -syscon posix _POSIX_SEM_VALUE_MAX 0x7fff 0x7fff 0x7fff 0x7fff 0x7fff 0 # unix consensus -syscon posix _POSIX_SSIZE_MAX 0x7fff 0x7fff 0x7fff 0x7fff 0x7fff 0 # unix consensus -syscon posix _POSIX_STREAM_MAX 8 8 8 8 8 0 # unix consensus -syscon posix _POSIX_SYMLINK_MAX 255 255 255 255 255 0 # unix consensus -syscon posix _POSIX_SYMLOOP_MAX 8 8 8 8 8 0 # unix consensus -syscon posix _POSIX_THREAD_DESTRUCTOR_ITERATIONS 4 4 4 4 4 0 # unix consensus -syscon posix _POSIX_THREAD_KEYS_MAX 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon posix _POSIX_TTY_NAME_MAX 9 9 9 9 9 0 # unix consensus -syscon posix _POSIX_TZNAME_MAX 6 6 6 6 6 0 # unix consensus -syscon posix _POSIX_CLOCK_SELECTION 0x031069 -1 -1 -1 -1 0 # bsd consensus -syscon posix _POSIX_FSYNC 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_MAPPED_FILES 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_MEMORY_PROTECTION 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_READER_WRITER_LOCKS 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_THREADS 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_THREAD_ATTR_STACKADDR 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_THREAD_ATTR_STACKSIZE 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon posix _POSIX_ADVISORY_INFO 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_ASYNCHRONOUS_IO 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_BARRIERS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_JOB_CONTROL 1 0x030db0 1 1 1 0 -syscon posix _POSIX_MEMLOCK 0x031069 -1 -1 0x030db0 0x030db0 0 -syscon posix _POSIX_MEMLOCK_RANGE 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_MESSAGE_PASSING 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_NO_TRUNC 1 0x030db0 1 1 1 0 -syscon posix _POSIX_RAW_SOCKETS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_REALTIME_SIGNALS 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_REGEXP 1 0x030db0 1 1 1 0 -syscon posix _POSIX_SEMAPHORES 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_SHARED_MEMORY_OBJECTS 0x031069 -1 0x030db0 0x031069 0x031069 0 -syscon posix _POSIX_SHELL 1 0x030db0 1 1 1 0 -syscon posix _POSIX_SPAWN 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_SPIN_LOCKS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_THREAD_PRIORITY_SCHEDULING 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_THREAD_PROCESS_SHARED 0x031069 0x030db0 0x030db0 -1 -1 0 -syscon posix _POSIX_THREAD_SAFE_FUNCTIONS 0x031069 0x030db0 -1 0x030db0 0x030db0 0 -syscon posix _POSIX_THREAD_THREADS_MAX 0x40 0x40 0x40 4 4 0 -syscon posix _POSIX_TIMEOUTS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_TIMERS 0x031069 -1 0x030db0 -1 -1 0 -syscon posix _POSIX_VERSION 0x031069 0x030db0 0x030db0 0x031069 0x031069 0 -syscon posix _POSIX_VDISABLE 0 255 255 255 255 0 # bsd consensus -syscon posix _POSIX_AIO_LISTIO_MAX 2 2 2 0 0 0 -syscon posix _POSIX_AIO_MAX 1 1 1 0 0 0 -syscon posix _POSIX_CHOWN_RESTRICTED 0 0x030db0 1 1 1 0 -syscon posix _POSIX_CLOCKRES_MIN 0x01312d00 0 0x01312d00 0x01312d00 0x01312d00 0 -syscon posix _POSIX_CPUTIME 0 -1 0x030db0 0x031069 0x031069 0 -syscon posix _POSIX_DELAYTIMER_MAX 0x20 0x20 0x20 0 0 0 -syscon posix _POSIX_MONOTONIC_CLOCK 0 -1 0x030db0 0x030db0 0x030db0 0 -syscon posix _POSIX_MQ_OPEN_MAX 8 8 8 0 0 0 -syscon posix _POSIX_MQ_PRIO_MAX 0x20 0x20 0x20 0 0 0 -syscon posix _POSIX_RTSIG_MAX 8 8 8 0 0 0 -syscon posix _POSIX_SAVED_IDS 1 0x030db0 0 1 1 0 -syscon posix _POSIX_SIGQUEUE_MAX 0x20 0x20 0x20 0 0 0 -syscon posix _POSIX_THREAD_CPUTIME 0 -1 0x030db0 0x031069 0x031069 0 -syscon posix _POSIX_TIMER_MAX 0x20 0x20 0x20 0 0 0 -syscon posix _POSIX_IPV6 0x031069 0x030db0 0 0 0 0 -syscon posix _POSIX_SS_REPL_MAX 0 4 4 0 0 0 -syscon posix _POSIX_TRACE_EVENT_NAME_MAX 0 30 30 0 0 0 -syscon posix _POSIX_TRACE_NAME_MAX 0 8 8 0 0 0 -syscon posix _POSIX_TRACE_SYS_MAX 0 8 8 0 0 0 -syscon posix _POSIX_TRACE_USER_EVENT_MAX 0 0x20 0x20 0 0 0 -syscon posix _POSIX_V6_LP64_OFF64 1 1 0 0 0 0 -syscon posix _POSIX_V7_LP64_OFF64 1 1 0 0 0 0 - -syscon icmp6 ICMP6_DST_UNREACH_NOROUTE 0 0 0 0 0 0 # consensus -syscon icmp6 ICMP6_PARAMPROB_HEADER 0 0 0 0 0 0 # consensus -syscon icmp6 ICMP6_TIME_EXCEED_TRANSIT 0 0 0 0 0 0 # consensus -syscon icmp6 ICMP6_DST_UNREACH_ADMIN 1 1 1 1 1 1 # consensus -syscon icmp6 ICMP6_PARAMPROB_NEXTHEADER 1 1 1 1 1 1 # consensus -syscon icmp6 ICMP6_TIME_EXCEED_REASSEMBLY 1 1 1 1 1 1 # consensus -syscon icmp6 ICMP6_DST_UNREACH 1 1 1 1 1 0 # unix consensus -syscon icmp6 ICMP6_FILTER 1 18 18 18 18 0 # bsd consensus -syscon icmp6 ICMP6_DST_UNREACH_BEYONDSCOPE 2 2 2 2 2 2 # consensus -syscon icmp6 ICMP6_PARAMPROB_OPTION 2 2 2 2 2 2 # consensus -syscon icmp6 ICMP6_PACKET_TOO_BIG 2 2 2 2 2 0 # unix consensus -syscon icmp6 ICMP6_DST_UNREACH_ADDR 3 3 3 3 3 3 # consensus -syscon icmp6 ICMP6_TIME_EXCEEDED 3 3 3 3 3 0 # unix consensus -syscon icmp6 ICMP6_DST_UNREACH_NOPORT 4 4 4 4 4 4 # consensus -syscon icmp6 ICMP6_PARAM_PROB 4 4 4 4 4 0 # unix consensus -syscon icmp6 ICMP6_RR_FLAGS_PREVDONE 8 8 8 8 8 0 # unix consensus -syscon icmp6 ICMP6_RR_FLAGS_SPECSITE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus -syscon icmp6 ICMP6_RR_PCOUSE_RAFLAGS_AUTO 0x10 0x40 0x40 0x40 0x40 0 # bsd consensus -syscon icmp6 ICMP6_RR_FLAGS_FORCEAPPLY 0x20 0x20 0x20 0x20 0x20 0 # unix consensus -syscon icmp6 ICMP6_RR_PCOUSE_RAFLAGS_ONLINK 0x20 0x80 0x80 0x80 0x80 0 # bsd consensus -syscon icmp6 ICMP6_RR_FLAGS_REQRESULT 0x40 0x40 0x40 0x40 0x40 0 # unix consensus -syscon icmp6 ICMP6_RR_PCOUSE_FLAGS_DECRPLTIME 0x40 0x40 0x40 0x40 0x40 0 # unix consensus -syscon icmp6 ICMP6_INFOMSG_MASK 0x80 0x80 0x80 0x80 0x80 0x80 # consensus -syscon icmp6 ICMP6_ECHO_REQUEST 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon icmp6 ICMP6_RR_FLAGS_TEST 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon icmp6 ICMP6_RR_PCOUSE_FLAGS_DECRVLTIME 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon icmp6 ICMP6_ECHO_REPLY 129 129 129 129 129 0 # unix consensus -syscon icmp6 ICMP6_ROUTER_RENUMBERING 138 138 138 138 138 0 # unix consensus -syscon icmp6 ICMP6_RR_RESULT_FLAGS_FORBIDDEN 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus -syscon icmp6 ICMP6_RR_RESULT_FLAGS_OOB 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus - syscon sio SIOCADDMULTI 0x8931 0x80206931 0x80206931 0x80206931 0x80206931 0 # bsd consensus syscon sio SIOCATMARK 0x8905 0x40047307 0x40047307 0x40047307 0x40047307 0 # bsd consensus syscon sio SIOCDELMULTI 0x8932 0x80206932 0x80206932 0x80206932 0x80206932 0 # bsd consensus @@ -1360,104 +1135,6 @@ syscon pf PF_VSOCK 40 0 0 0 0 0 syscon pf PF_WANPIPE 25 0 0 0 0 0 syscon pf PF_X25 9 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_DONT 0 0 0 0 0 0 # consensus -syscon ipv6 IPV6_RTHDR_LOOSE 0 0 0 0 0 0 # consensus -syscon ipv6 IPV6_RTHDR_TYPE_0 0 0 0 0 0 0 # consensus -syscon ipv6 IPV6_CHECKSUM 7 26 26 26 26 26 # bsd consensus -syscon ipv6 IPV6_JOIN_GROUP 20 12 12 12 12 12 # bsd consensus -syscon ipv6 IPV6_LEAVE_GROUP 21 13 13 13 13 13 # bsd consensus -syscon ipv6 IPV6_MULTICAST_HOPS 18 10 10 10 10 10 # bsd consensus -syscon ipv6 IPV6_MULTICAST_IF 17 9 9 9 9 9 # bsd consensus -syscon ipv6 IPV6_MULTICAST_LOOP 19 11 11 11 11 11 # bsd consensus -syscon ipv6 IPV6_UNICAST_HOPS 0x10 4 4 4 4 4 # bsd consensus -syscon ipv6 IPV6_V6ONLY 26 27 27 27 27 27 # bsd consensus -syscon ipv6 IPV6_RECVTCLASS 66 35 57 57 57 40 -syscon ipv6 IPV6_TCLASS 67 36 61 61 61 39 -syscon ipv6 IPV6_DONTFRAG 62 0 62 62 62 14 -syscon ipv6 IPV6_HOPLIMIT 52 0 47 47 47 21 -syscon ipv6 IPV6_HOPOPTS 54 0 49 49 49 1 -syscon ipv6 IPV6_PKTINFO 50 0 46 46 46 19 -syscon ipv6 IPV6_RECVRTHDR 56 0 38 38 38 38 -syscon ipv6 IPV6_RTHDR 57 0 51 51 51 0x20 -syscon ipv6 IPV6_DSTOPTS 59 0 50 50 50 0 -syscon ipv6 IPV6_IPSEC_POLICY 34 28 28 0 0 0 -syscon ipv6 IPV6_NEXTHOP 9 0 48 48 48 0 -syscon ipv6 IPV6_PATHMTU 61 0 44 44 44 0 -syscon ipv6 IPV6_RECVDSTOPTS 58 0 40 40 40 0 -syscon ipv6 IPV6_RECVHOPLIMIT 51 0 37 37 37 0 -syscon ipv6 IPV6_RECVHOPOPTS 53 0 39 39 39 0 -syscon ipv6 IPV6_RECVPATHMTU 60 0 43 43 43 0 -syscon ipv6 IPV6_RECVPKTINFO 49 0 36 36 36 0 -syscon ipv6 IPV6_RTHDRDSTOPTS 55 0 35 35 35 0 -syscon ipv6 IPV6_RTHDR_STRICT 1 1 1 0 0 0 -syscon ipv6 IPV6_ADD_MEMBERSHIP 20 0 0 0 0 12 # bsd consensus -syscon ipv6 IPV6_DROP_MEMBERSHIP 21 0 0 0 0 13 # bsd consensus -syscon ipv6 IPV6_HDRINCL 36 0 0 0 0 2 # bsd consensus -syscon ipv6 IPV6_MTU 24 0 0 0 0 72 # bsd consensus -syscon ipv6 IPV6_MTU_DISCOVER 23 0 0 0 0 71 # bsd consensus -syscon ipv6 IPV6_RECVERR 25 0 0 0 0 75 # bsd consensus -syscon ipv6 IPV6_2292DSTOPTS 4 23 0 0 0 0 -syscon ipv6 IPV6_2292HOPLIMIT 8 20 0 0 0 0 -syscon ipv6 IPV6_2292HOPOPTS 3 22 0 0 0 0 -syscon ipv6 IPV6_2292PKTINFO 2 19 0 0 0 0 -syscon ipv6 IPV6_2292PKTOPTIONS 6 25 0 0 0 0 -syscon ipv6 IPV6_2292RTHDR 5 24 0 0 0 0 -syscon ipv6 IPV6_AUTOFLOWLABEL 0 0 59 59 59 0 -syscon ipv6 IPV6_ADDRFORM 1 0 0 0 0 0 -syscon ipv6 IPV6_AUTHHDR 10 0 0 0 0 0 -syscon ipv6 IPV6_JOIN_ANYCAST 27 0 0 0 0 0 -syscon ipv6 IPV6_LEAVE_ANYCAST 28 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_DO 2 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_INTERFACE 4 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_OMIT 5 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_PROBE 3 0 0 0 0 0 -syscon ipv6 IPV6_PMTUDISC_WANT 1 0 0 0 0 0 -syscon ipv6 IPV6_ROUTER_ALERT 22 0 0 0 0 0 -syscon ipv6 IPV6_RXDSTOPTS 59 0 0 0 0 0 -syscon ipv6 IPV6_RXHOPOPTS 54 0 0 0 0 0 -syscon ipv6 IPV6_XFRM_POLICY 35 0 0 0 0 0 -syscon ipv6 IPV6_MINHOPCOUNT 0 0 0 65 65 0 -syscon ipv6 IPV6_ORIGDSTADDR 0 0 72 0 0 0 -syscon ipv6 IPV6_RECVORIGDSTADDR 0 0 72 0 0 0 -syscon ipv6 INET6_ADDRSTRLEN 46 46 46 46 46 65 # unix consensus - -syscon fan FAN_CLASS_NOTIF 0 0 0 0 0 0 # consensus -syscon fan FAN_ACCESS 1 0 0 0 0 0 -syscon fan FAN_ACCESS_PERM 0x020000 0 0 0 0 0 -syscon fan FAN_ALLOW 1 0 0 0 0 0 -syscon fan FAN_ALL_CLASS_BITS 12 0 0 0 0 0 -syscon fan FAN_ALL_EVENTS 59 0 0 0 0 0 -syscon fan FAN_ALL_INIT_FLAGS 63 0 0 0 0 0 -syscon fan FAN_ALL_MARK_FLAGS 255 0 0 0 0 0 -syscon fan FAN_ALL_OUTGOING_EVENTS 0x03403b 0 0 0 0 0 -syscon fan FAN_ALL_PERM_EVENTS 0x030000 0 0 0 0 0 -syscon fan FAN_CLASS_CONTENT 4 0 0 0 0 0 -syscon fan FAN_CLASS_PRE_CONTENT 8 0 0 0 0 0 -syscon fan FAN_CLOEXEC 1 0 0 0 0 0 -syscon fan FAN_CLOSE 24 0 0 0 0 0 -syscon fan FAN_CLOSE_NOWRITE 0x10 0 0 0 0 0 -syscon fan FAN_CLOSE_WRITE 8 0 0 0 0 0 -syscon fan FAN_DENY 2 0 0 0 0 0 -syscon fan FAN_EVENT_METADATA_LEN 24 0 0 0 0 0 -syscon fan FAN_EVENT_ON_CHILD 0x08000000 0 0 0 0 0 -syscon fan FAN_MARK_ADD 1 0 0 0 0 0 -syscon fan FAN_MARK_DONT_FOLLOW 4 0 0 0 0 0 -syscon fan FAN_MARK_FLUSH 0x80 0 0 0 0 0 -syscon fan FAN_MARK_IGNORED_MASK 0x20 0 0 0 0 0 -syscon fan FAN_MARK_IGNORED_SURV_MODIFY 0x40 0 0 0 0 0 -syscon fan FAN_MARK_MOUNT 0x10 0 0 0 0 0 -syscon fan FAN_MARK_ONLYDIR 8 0 0 0 0 0 -syscon fan FAN_MARK_REMOVE 2 0 0 0 0 0 -syscon fan FAN_MODIFY 2 0 0 0 0 0 -syscon fan FAN_NOFD -1 0 0 0 0 0 -syscon fan FAN_NONBLOCK 2 0 0 0 0 0 -syscon fan FAN_ONDIR 0x40000000 0 0 0 0 0 -syscon fan FAN_OPEN 0x20 0 0 0 0 0 -syscon fan FAN_OPEN_PERM 0x010000 0 0 0 0 0 -syscon fan FAN_Q_OVERFLOW 0x4000 0 0 0 0 0 -syscon fan FAN_UNLIMITED_MARKS 0x20 0 0 0 0 0 -syscon fan FAN_UNLIMITED_QUEUE 0x10 0 0 0 0 0 - syscon exit EXIT_SUCCESS 0 0 0 0 0 0 # consensus syscon exit EXIT_FAILURE 1 1 1 1 1 1 # consensus @@ -1467,7 +1144,7 @@ syscon exit EXIT_FAILURE 1 1 1 1 1 1 # consensus # - Dating back to 1980 in 4.0BSD; # - That won't be standardized. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon ex EX_OK 0 0 0 0 0 0 # consensus syscon ex EX_USAGE 64 64 64 64 64 64 # unix consensus & force NT syscon ex EX_DATAERR 65 65 65 65 65 65 # unix consensus & force NT @@ -1489,7 +1166,7 @@ syscon ex EX__MAX 78 78 78 78 78 78 # unix consensus & force NT # getdents() constants # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon dt DT_UNKNOWN 0 0 0 0 0 0 # consensus syscon dt DT_FIFO 1 1 1 1 1 1 # unix consensus & faked nt syscon dt DT_CHR 2 2 2 2 2 2 # unix consensus & faked nt @@ -1501,14 +1178,30 @@ syscon dt DT_SOCK 12 12 12 12 12 12 # unix consensus & faked nt # msync() flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon ms MS_SYNC 4 16 0 2 4 4 # faked nt syscon ms MS_ASYNC 1 1 1 1 1 1 # consensus (faked nt) syscon ms MS_INVALIDATE 2 2 2 4 2 0 +# statvfs() flags +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon statvfs ST_NOSUID 2 2 2 2 2 0 # unix consensus +syscon statvfs ST_RDONLY 1 1 1 1 1 0 # unix consensus +syscon statvfs ST_APPEND 0x0100 0 0 0 0 0 +syscon statvfs ST_IMMUTABLE 0x0200 0 0 0 0 0 +syscon statvfs ST_MANDLOCK 0x40 0 0 0 0 0 +syscon statvfs ST_NOATIME 0x0400 0 0 0 0 0 +syscon statvfs ST_NODEV 4 0 0 0 0 0 +syscon statvfs ST_NODIRATIME 0x0800 0 0 0 0 0 +syscon statvfs ST_NOEXEC 8 0 0 0 0 0 +syscon statvfs ST_RELATIME 0x1000 0 0 0 0 0 +syscon statvfs ST_SYNCHRONOUS 0x10 0 0 0 0 0 +syscon statvfs ST_WRITE 0x80 0 0 0 0 0 + # mount flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon mount MS_ACTIVE 0x40000000 0 0 0 0 0 syscon mount MS_BIND 0x1000 0 0 0 0 0 syscon mount MS_DIRSYNC 0x80 0 0 0 0 0 @@ -1539,7 +1232,9 @@ syscon mount MS_SYNCHRONOUS 0x10 0 0 0 0 0 syscon mount MS_UNBINDABLE 0x020000 0 0 0 0 0 syscon mount MS_MGC_MSK 0xffff0000 0 0 0 0 0 -# TODO(jart): MSG_ZEROCOPY +# send() / recv() flags +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon msg MSG_OOB 1 1 1 1 1 1 # consensus syscon msg MSG_PEEK 2 2 2 2 2 2 # consensus syscon msg MSG_DONTROUTE 4 4 4 4 4 4 # consensus @@ -1566,33 +1261,240 @@ syscon msg MSG_RST 0x1000 0 0 0 0 0 syscon msg MSG_STAT 11 0 0 0 0 0 syscon msg MSG_SYN 0x0400 0 0 0 0 0 -syscon in IN_LOOPBACKNET 127 127 127 127 127 0 # unix consensus -syscon in IN_ACCESS 1 0 0 0 0 0 -syscon in IN_ALL_EVENTS 0x0fff 0 0 0 0 0 -syscon in IN_ATTRIB 4 0 0 0 0 0 -syscon in IN_CLOEXEC 0x080000 0 0 0 0 0 -syscon in IN_CLOSE 24 0 0 0 0 0 -syscon in IN_CLOSE_NOWRITE 0x10 0 0 0 0 0 -syscon in IN_CLOSE_WRITE 8 0 0 0 0 0 -syscon in IN_CREATE 0x0100 0 0 0 0 0 -syscon in IN_DELETE 0x0200 0 0 0 0 0 -syscon in IN_DELETE_SELF 0x0400 0 0 0 0 0 -syscon in IN_DONT_FOLLOW 0x02000000 0 0 0 0 0 -syscon in IN_EXCL_UNLINK 0x04000000 0 0 0 0 0 -syscon in IN_IGNORED 0x8000 0 0 0 0 0 -syscon in IN_ISDIR 0x40000000 0 0 0 0 0 -syscon in IN_MASK_ADD 0x20000000 0 0 0 0 0 -syscon in IN_MODIFY 2 0 0 0 0 0 -syscon in IN_MOVE 192 0 0 0 0 0 -syscon in IN_MOVED_FROM 0x40 0 0 0 0 0 -syscon in IN_MOVED_TO 0x80 0 0 0 0 0 -syscon in IN_MOVE_SELF 0x0800 0 0 0 0 0 -syscon in IN_NONBLOCK 0x0800 0 0 0 0 0 -syscon in IN_ONESHOT 0x80000000 0 0 0 0 0 -syscon in IN_ONLYDIR 0x01000000 0 0 0 0 0 -syscon in IN_OPEN 0x20 0 0 0 0 0 -syscon in IN_Q_OVERFLOW 0x4000 0 0 0 0 0 -syscon in IN_UNMOUNT 0x2000 0 0 0 0 0 +# getpriority() / setpriority() magnums (a.k.a. nice) +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon prio PRIO_PROCESS 0 0 0 0 0 0 # consensus / poly nt +syscon prio PRIO_PGRP 1 1 1 1 1 1 # unix consensus / poly nt +syscon prio PRIO_USER 2 2 2 2 2 2 # unix consensus / poly nt +syscon prio PRIO_MIN -20 -20 -20 -20 -20 -20 # unix consensus / poly nt +syscon prio PRIO_MAX 20 20 20 20 20 20 # unix consensus / poly nt +syscon prio NZERO 20 20 20 20 20 20 # unix consensus / polyfilled nt + +# Teletypewriter Control, e.g. +# +# TCSETS → About 70,800 results (0.31 seconds) +# = TCSETNOW → About 47,700 results (0.31 seconds) +# ≈ TCSETA → About 12,600 results (0.32 seconds) +# = TIOCSETA → About 3,110 results (0.41 seconds) +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon termios TCGETS 0x5401 0x40487413 0x402c7413 0x402c7413 0x402c7413 0 # Gets console settings; tcgetattr(tty, argp) → ioctl(tty, TCGETS, struct termios *argp); polyfilled NT +syscon compat TIOCGETA 0x5401 0x40487413 0x402c7413 0x402c7413 0x402c7413 0 # Gets console settings; = tcgetattr(tty, struct termios *argp) +#syscon compat TCGETA 0x5405 0 0 0 0 0 # Gets console settings; ≈ ioctl(fd, TCGETA, struct termio *argp) +syscon termios TCSANOW 0 0 0 0 0 0 # Sets console settings; tcsetattr(fd, TCSANOW, argp); polyfilled NT +syscon termios TCSETS 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; = ioctl(tty, TCSETS, const struct termios *argp); polyfilled NT +syscon compat TIOCSETA 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; = ioctl(tty, TIOCSETA, const struct termios *argp); polyfilled NT +#syscon compat TCSETA 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; ≈ ioctl(tty, TCSETA, const struct termio *argp); polyfilled NT +syscon termios TCSADRAIN 1 1 1 1 1 1 # Drains output & sets console settings; tcsetawttr(fd, TCSADRAIN, argp); polyfilled NT +syscon termios TCSETSW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; = ioctl(tty, TCSETSW, const struct termios *argp); polyfilled NT +syscon compat TIOCSETAW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; = ioctl(tty, TIOCSETAW, const struct termios *argp); polyfilled NT +#syscon compat TCSETAW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; ≈ ioctl(tty, TCSETAW, const struct termio *argp); polyfilled NT +syscon termios TCSAFLUSH 2 2 2 2 2 2 # Drops input & drains output & sets console settings; tcsetafttr(fd, TCSAFLUSH, argp); polyfilled NT +syscon termios TCSETSF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5404 # Drops input & drains output & sets console settings; = ioctl(tty, TCSETSF, const struct termios *argp); polyfilled NT +syscon compat TIOCSETAF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5402 # Drops input & drains output & sets console settings; = ioctl(tty, TIOCSETAF, const struct termios *argp); polyfilled NT +#syscon compat TCSETAF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5402 # Drops input & drains output & sets console settings; ≈ ioctl(tty, TCSETAF, const struct termio *argp); polyfilled NT +syscon termios TIOCGWINSZ 0x5413 1074295912 1074295912 1074295912 1074295912 0x5413 # ioctl(tty, TIOCGWINSZ, struct winsize *argp); polyfilled NT +syscon termios TIOCSWINSZ 0x5414 0x80087467 0x80087467 0x80087467 0x80087467 0x5414 # ioctl(tty, TIOCSWINSZ, const struct winsize *argp) (faked NT) +syscon termios TIOCOUTQ 0x5411 0x40047473 0x40047473 0x40047473 0x40047473 0 # get # bytes queued in TTY's output buffer ioctl(tty, TIOCSWINSZ, const struct winsize *argp) +syscon termios TIOCCBRK 0x5428 0x2000747a 0x2000747a 0x2000747a 0x2000747a 0 # boop +syscon termios TIOCCONS 0x541d 0x80047462 0x80047462 0x80047462 0x80047462 0 # boop +syscon termios TIOCGETD 0x5424 0x4004741a 0x4004741a 0x4004741a 0x4004741a 0 # boop +syscon termios TIOCGPGRP 0x540f 0x40047477 0x40047477 0x40047477 0x40047477 0 # boop +syscon termios TIOCNOTTY 0x5422 0x20007471 0x20007471 0x20007471 0x20007471 0 # boop +syscon termios TIOCNXCL 0x540d 0x2000740e 0x2000740e 0x2000740e 0x2000740e 0 # boop +syscon termios TIOCSBRK 0x5427 0x2000747b 0x2000747b 0x2000747b 0x2000747b 0 # boop +syscon termios TIOCSCTTY 0x540e 0x20007461 0x20007461 0x20007461 0x20007461 0 # boop +syscon termios TIOCSETD 0x5423 0x8004741b 0x8004741b 0x8004741b 0x8004741b 0 # boop +syscon termios TIOCSIG 0x40045436 0x2000745f 0x2004745f 0x8004745f 0x8004745f 0 # boop +syscon termios TIOCSPGRP 0x5410 0x80047476 0x80047476 0x80047476 0x80047476 0 # boop +syscon termios TIOCSTI 0x5412 0x80017472 0x80017472 0 0 0 # boop +syscon termios TIOCGPTN 0x80045430 0 0x4004740f 0 0 0 # boop +syscon termios TIOCGSID 0x5429 0 0x40047463 0x40047463 0x40047463 0 # boop +syscon termios TABLDISC 0 0x3 0 0x3 0x3 0 # boop +syscon termios SLIPDISC 0 0x4 0x4 0x4 0x4 0 # boop +syscon termios PPPDISC 0 0x5 0x5 0x5 0x5 0 # boop +syscon termios TIOCDRAIN 0 0x2000745e 0x2000745e 0x2000745e 0x2000745e 0 # boop +syscon termios TIOCSTAT 0 0x20007465 0x20007465 0x20007465 0x20007465 0 # boop +syscon termios TIOCSTART 0 0x2000746e 0x2000746e 0x2000746e 0x2000746e 0 # boop +syscon termios TIOCCDTR 0 0x20007478 0x20007478 0x20007478 0x20007478 0 # boop +syscon termios TIOCSDTR 0 0x20007479 0x20007479 0x20007479 0x20007479 0 # boop +syscon termios TIOCFLUSH 0 0x80047410 0x80047410 0x80047410 0x80047410 0 # boop +syscon termios TIOCEXT 0 0x80047460 0x80047460 0x80047460 0x80047460 0 # boop +syscon termios TIOCGDRAINWAIT 0 0x40047456 0x40047456 0 0 0 # boop +syscon termios TIOCTIMESTAMP 0 0x40107459 0x40107459 0 0 0 # boop +syscon termios TIOCSDRAINWAIT 0 0x80047457 0x80047457 0 0 0 # boop +syscon termios TIOCREMOTE 0 0x80047469 0 0x80047469 0x80047469 0 # boop +syscon termios TTYDISC 0 0 0 0 0 0 # boop +syscon termios TIOCFLAG_SOFTCAR 0 0 0 0x1 0x1 0 # boop +syscon termios TIOCFLAG_PPS 0 0 0 0x10 0x10 0 # boop +syscon termios TIOCFLAG_CLOCAL 0 0 0 0x2 0x2 0 # boop +syscon termios TIOCCHKVERAUTH 0 0 0 0x2000741e 0x2000741e 0 # boop +syscon termios TIOCGFLAGS 0 0 0 0x4004745d 0x4004745d 0 # boop +syscon termios TIOCGTSTAMP 0 0 0 0x4010745b 0x4010745b 0 # boop +syscon termios STRIPDISC 0 0 0 0x6 0x6 0 # boop +syscon termios NMEADISC 0 0 0 0x7 0x7 0 # boop +syscon termios TIOCUCNTL_CBRK 0 0 0 0x7a 0x7a 0 # boop +syscon termios TIOCFLAG_MDMBUF 0 0 0 0x8 0x8 0 # boop +syscon termios TIOCSETVERAUTH 0 0 0 0x8004741c 0x8004741c 0 # boop +syscon termios TIOCSFLAGS 0 0 0 0x8004745c 0x8004745c 0 # boop +syscon termios TIOCSTSTAMP 0 0 0 0x8008745a 0x8008745a 0 # boop +syscon termios ENDRUNDISC 0 0 0 0x9 0x9 0 # boop +syscon termios TIOCPTMASTER 0 0 0x2000741c 0 0 0 # boop +syscon termios NETGRAPHDISC 0 0 0x6 0 0 0 # boop +syscon termios H4DISC 0 0 0x7 0 0 0 # boop +syscon termios ISIG 0b0000000000000001 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000000000001 # termios.c_lflag|=ISIG makes Ctrl-C, Ctrl-\, etc. generate signals +syscon termios ICANON 0b0000000000000010 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000000000010 # termios.c_lflag&=~ICANON disables 1960's version of gnu readline (see also VMIN) +syscon termios XCASE 0b0000000000000100 0 0 16777216 0 0b0000000000000100 # termios.c_lflag +syscon termios ECHO 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # termios.c_lflag&=~ECHO is for passwords and raw mode +syscon termios ECHOE 0b0000000000010000 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000010000 # termios.c_lflag +syscon termios ECHOK 0b0000000000100000 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000100000 # termios.c_lflag +syscon termios ECHONL 0b0000000001000000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000001000000 # termios.c_lflag +syscon termios NOFLSH 0b0000000010000000 2147483648 2147483648 2147483648 2147483648 0b0000000010000000 # termios.c_lflag|=NOFLSH means don't flush on INT/QUIT/SUSP +syscon termios TOSTOP 0b0000000100000000 4194304 4194304 4194304 4194304 0b0000000100000000 # termios.c_lflag|=TOSTOP raises SIGTTOU to process group if background job tries to write to controlling terminal +syscon termios ECHOCTL 0b0000001000000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000001000000000 # termios.c_lflag|=ECHOCTL prints ^𝑥 codes for monotonic motion +syscon termios ECHOPRT 0b0000010000000000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000010000000000 # termios.c_lflag|=ECHOPRT includes the parity bit +syscon termios ECHOKE 0b0000100000000000 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000100000000000 # termios.c_lflag +syscon termios FLUSHO 0b0001000000000000 8388608 8388608 8388608 8388608 0b0001000000000000 # termios.c_lflag +syscon termios PENDIN 0b0100000000000000 536870912 536870912 536870912 536870912 0b0100000000000000 # termios.c_lflag +syscon termios IEXTEN 0b1000000000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b1000000000000000 # termios.c_lflag&=~IEXTEN disables platform input processing magic +syscon termios EXTPROC 65536 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 65536 # termios.c_lflag +syscon termios IGNBRK 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # termios.c_iflag it's complicated, uart only? UNIXCONSENSUS +syscon termios BRKINT 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 # termios.c_iflag it's complicated, uart only? UNIXCONSENSUS +syscon termios IGNPAR 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 # termios.c_iflag|=IGNPAR ignores parity and framing errors; see PARMRK UNIXCONSENSUS +syscon termios PARMRK 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # termios.c_iflag|=PARMRK passes-through parity bit UNIXCONSENSUS +syscon termios INPCK 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 # termios.c_iflag|=INPCK enables parity checking UNIXCONSENSUS +syscon termios ISTRIP 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 # termios.c_iflag|=ISTRIP automates read(1)&0x7f UNIXCONSENSUS +syscon termios INLCR 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # termios.c_iflag|=INLCR maps \n → \r input UNIXCONSENSUS +syscon termios IGNCR 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # termios.c_iflag|=IGNCR maps \r → ∅ input UNIXCONSENSUS +syscon termios ICRNL 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 # termios.c_iflag|=ICRNL maps \r → \n input UNIXCONSENSUS +syscon termios IUCLC 0b0000001000000000 0 0 0b0001000000000000 0 0b0000001000000000 # termios.c_iflag|=IUCLC maps A-Z → a-z input +syscon termios IXON 0b0000010000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000010000000000 # termios.c_iflag|=IXON enables flow rida +syscon termios IXANY 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 # termios.c_iflag|=IXANY tying will un-stuck teletype UNIXCONSENSUS +syscon termios IXOFF 0b0001000000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0001000000000000 # termios.c_iflag|=IXOFF disables annoying display freeze keys +syscon termios IMAXBEL 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 # termios.c_iflag|=IMAXBEL rings when queue full UNIXCONSENSUS +syscon termios IUTF8 0b0100000000000000 0b0100000000000000 0 0 0 0b0100000000000000 # termios.c_iflag|=IUTF8 helps w/ rubout on UTF-8 input +syscon termios OPOST 0b0000000000000001 0b000000000000000001 0b000000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # termios.c_oflag&=~OPOST disables output processing magic +syscon termios OLCUC 0b0000000000000010 0 0 0b0000000000100000 0 0b0000000000000010 # termios.c_oflag|=OLCUC maps a-z → A-Z output +syscon termios ONLCR 0b0000000000000100 0b000000000000000010 0b000000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000100 # termios.c_oflag|=ONLCR maps \n → \r\n output +syscon termios OCRNL 0b0000000000001000 0b000000000000010000 0b000000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000001000 # termios.c_oflag|=OCRNL maps \r → \n output +syscon termios ONOCR 0b0000000000010000 0b000000000000100000 0b000000000000100000 0b0000000001000000 0b0000000001000000 0b0000000000010000 # termios.c_oflag|=ONOCR maps \r → ∅ output iff column 0 +syscon termios ONLRET 0b0000000000100000 0b000000000001000000 0b000000000001000000 0b0000000010000000 0b0000000010000000 0b0000000000100000 # termios.c_oflag|=ONLRET maps \r → ∅ output +syscon termios OFILL 0b0000000001000000 0b000000000010000000 0 0 0 0b0000000001000000 # termios.c_oflag +syscon termios OFDEL 0b0000000010000000 0b100000000000000000 0 0 0 0b0000000010000000 # termios.c_oflag +syscon termios NLDLY 0b0000000100000000 0b000000001100000000 0b000000001100000000 0 0 0b0000000100000000 # (termios.c_oflag & NLDLY) ∈ {NL0,NL1,NL2,NL3} +syscon termios NL0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0b000000000000000000 0b0000000000000000 # (termios.c_oflag & NLDLY) == NL0 +syscon termios NL1 0b0000000100000000 0b000000000100000000 0b000000000100000000 0 0b000000000100000000 0b0000000100000000 # (termios.c_oflag & NLDLY) == NL1 +syscon termios NL2 0 0b000000001000000000 0b000000001000000000 0 0b000000001000000000 0 # (termios.c_oflag & NLDLY) == NL2 +syscon termios NL3 0 0b000000001100000000 0b000000001100000000 0 0b000000001100000000 0 # (termios.c_oflag & NLDLY) == NL3 +syscon termios CRDLY 0b0000011000000000 0b000011000000000000 0b000011000000000000 0 0 0b0000011000000000 # (termios.c_oflag & CRDLY) ∈ {CR0,CR1,CR2,CR3} +syscon termios CR0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0b000000000000000000 0b0000000000000000 # (termios.c_oflag & CRDLY) == CR0 +syscon termios CR1 0b0000001000000000 0b000001000000000000 0b000001000000000000 0 0b000001000000000000 0b0000001000000000 # (termios.c_oflag & CRDLY) == CR1 +syscon termios CR2 0b0000010000000000 0b000010000000000000 0b000010000000000000 0 0b000000010000000000 0b0000010000000000 # (termios.c_oflag & CRDLY) == CR2 +syscon termios CR3 0b0000011000000000 0b000011000000000000 0b000011000000000000 0 0b000000011000000000 0b0000011000000000 # (termios.c_oflag & CRDLY) == CR3 +syscon termios TABDLY 0b0001100000000000 0b000000110000000100 0b000000000000000100 0 0 0b0001100000000000 # (termios.c_oflag & TABDLY) ∈ {TAB0,TAB1,TAB2,TAB3,XTABS} +syscon termios TAB0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0b000000000000000000 0b0000000000000000 # (termios.c_oflag & TABDLY) == TAB0 +syscon termios TAB1 0b0000100000000000 0b000000010000000000 0b000000010000000000 0 0b000000010000000000 0b0000100000000000 # (termios.c_oflag & TABDLY) == TAB1 +syscon termios TAB2 0b0001000000000000 0b000000100000000000 0b000000100000000000 0 0b000000100000000000 0b0001000000000000 # (termios.c_oflag & TABDLY) == TAB2 +syscon termios TAB3 0b0001100000000000 0b000000000000000100 0b000000000000000100 0 0b000000000000000100 0b0001100000000000 # (termios.c_oflag & TABDLY) == TAB3 +syscon termios XTABS 0b0001100000000000 0b000000110000000000 0b000000110000000000 0 0b000000110000000000 0b0001100000000000 # (termios.c_oflag & TABDLY) == XTABS +syscon termios BSDLY 0b0010000000000000 0b001000000000000000 0b001000000000000000 0 0 0b0010000000000000 # termios.c_oflag +syscon termios BS0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag +syscon termios BS1 0b0010000000000000 0b001000000000000000 0b001000000000000000 0 0 0b0010000000000000 # termios.c_oflag +syscon termios VTDLY 0b0100000000000000 0b010000000000000000 0b010000000000000000 0 0 0b0100000000000000 # termios.c_oflag +syscon termios VT0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag +syscon termios VT1 0b0100000000000000 0b010000000000000000 0b010000000000000000 0 0 0b0100000000000000 # termios.c_oflag +syscon termios FFDLY 0b1000000000000000 0b000100000000000000 0b000100000000000000 0 0 0b1000000000000000 # termios.c_oflag +syscon termios FF0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag +syscon termios FF1 0b1000000000000000 0b000100000000000000 0b000100000000000000 0 0 0b1000000000000000 # termios.c_oflag +syscon termios CS6 0b0000000000010000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000000010000 # termios.c_cflag flag for 6-bit characters +syscon termios CS7 0b0000000000100000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000000000100000 # termios.c_cflag flag for 7-bit characters +syscon termios CS8 0b0000000000110000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000000000110000 # termios.c_cflag flag for 8-bit characters +syscon termios CSIZE 0b0000000000110000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000000000110000 # mask for CS𝑥 flags +syscon termios NCCS 32 32 32 32 20 32 # ARRAYLEN(termios.c_cc); faked xnu/freebsd/openbsd (originally 20) and faked nt +syscon termios VINTR 0 8 8 8 8 0 # termios.c_cc[VINTR]=𝑥 +syscon termios VQUIT 1 9 9 9 9 0 # termios.c_cc[VQUIT]=𝑥 +syscon termios VERASE 2 3 3 3 3 0 # termios.c_cc[VERASE]=𝑥 +syscon termios VKILL 3 5 5 5 5 0 # termios.c_cc[VKILL]=𝑥 +syscon termios VEOF 4 0 0 0 0 0 # termios.c_cc[VEOF]=𝑥 +syscon termios VTIME 5 17 17 17 17 0 # termios.c_cc[VTIME]=𝑥 sets non-canonical read timeout to 𝑥×𝟷𝟶𝟶ms which is needed when entering escape sequences manually with the escape key +syscon termios VMIN 6 16 16 16 16 0 # termios.c_cc[VMIN]=𝑥 in non-canonical mode can be set to 0 for non-blocking reads, 1 for single character raw mode reads, or higher to buffer +syscon termios VSWTC 7 0 0 0 0 0 # termios.c_cc[VSWTC]=𝑥 +syscon termios VSTART 8 12 12 12 12 0 # termios.c_cc[VSTART]=𝑥 +syscon termios VSTOP 9 13 13 13 13 0 # termios.c_cc[VSTOP]=𝑥 +syscon termios VSUSP 10 10 10 10 10 0 # termios.c_cc[VSUSP]=𝑥 defines suspend, i.e. Ctrl-Z (a.k.a. →, ^Z, SUB, 26, 032, 0x1A, ord('Z')^0b01000000); unix consensus +syscon termios VEOL 11 1 1 1 1 0 # termios.c_cc[VEOL]=𝑥 +syscon termios VEOL2 16 2 2 2 2 0 # termios.c_cc[VEOL2]=𝑥 +syscon termios VREPRINT 12 6 6 6 6 0 # termios.c_cc[VREPRINT]=𝑥 +syscon termios VDISCARD 13 15 15 15 15 0 # termios.c_cc[VDISCARD]=𝑥 +syscon termios VWERASE 14 4 4 4 4 0 # termios.c_cc[VWERASE]=𝑥 +syscon termios VLNEXT 15 14 14 14 14 0 # termios.c_cc[VLNEXT]=𝑥 +syscon termios TIOCSERGETLSR 0x5459 0 0 0 0 0 # +syscon termios TIOCSERGETMULTI 0x545a 0 0 0 0 0 # +syscon termios TIOCSERSETMULTI 0x545b 0 0 0 0 0 # +syscon termios TIOCSER_TEMT 1 0 0 0 0 0 # +syscon termios VERIFY 47 0 0 0 0 0 +syscon termios PARENB 0x0100 0x1000 0x1000 0x1000 0x1000 0 # +syscon termios PARODD 0x0200 0x2000 0x2000 0x2000 0x2000 0 # +syscon termios CIBAUD 0x100f0000 0 0 0 0 0 +syscon termios CLOCAL 0x0800 0x8000 0x8000 0x8000 0x8000 0 # +syscon termios CMSPAR 0x40000000 0 0 0 0 0 +syscon termios BUSY 4 0 0 0 0 0 +syscon termios CANBSIZ 255 0 0 0 0 0 +syscon termios CBAUD 0x100f 0 0 0 0 0 +syscon termios CBAUDEX 0x1000 0 0 0 0 0 +syscon termios CBRK 0 255 255 255 255 0 # +syscon termios CEOL 0 255 255 255 255 0 # +syscon termios EXTA 14 0x4b00 0x4b00 0x4b00 0x4b00 0 # bsd consensus +syscon termios EXTB 15 0x9600 0x9600 0x9600 0x9600 0 # bsd consensus +syscon termios ERA 0x02002c 45 45 0 0 0 +syscon termios EMPTY 0 0 0 0 0 0 # consensus +syscon termios TCFLSH 0x540b 0 0 0 0 0 +syscon termios TCIFLUSH 0 1 1 1 1 0 # bsd consensus +syscon termios TCIOFF 2 3 3 3 3 0 # bsd consensus +syscon termios TCIOFLUSH 2 3 3 3 3 0 # bsd consensus +syscon termios TCION 3 4 4 4 4 0 # bsd consensus +syscon termios TCOFLUSH 1 2 2 2 2 0 # bsd consensus +syscon termios TCOOFF 0 1 1 1 1 0 # bsd consensus +syscon termios TCOON 1 2 2 2 2 0 # bsd consensus + +# Pseudoteletypewriter Control +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon pty TIOCPKT 0x5420 0x80047470 0x80047470 0x80047470 0x80047470 -1 # boop +syscon pty TIOCPKT_DATA 0 0 0 0 0 0 # consensus +syscon pty TIOCPKT_FLUSHREAD 1 1 1 1 1 1 # unix consensus +syscon pty TIOCPKT_FLUSHWRITE 2 2 2 2 2 2 # unix consensus +syscon pty TIOCPKT_STOP 4 4 4 4 4 4 # unix consensus +syscon pty TIOCPKT_START 8 8 8 8 8 8 # unix consensus +syscon pty TIOCPKT_NOSTOP 16 16 16 16 16 16 # unix consensus +syscon pty TIOCPKT_DOSTOP 32 32 32 32 32 32 # unix consensus +syscon pty TIOCPKT_IOCTL 64 64 64 64 64 64 # unix consensus +syscon pty TIOCSPTLCK 0x40045431 0 0 0 0 -1 # boop +syscon pty PTMGET 0 0 0 0x40287401 0x40287401 -1 # for /dev/ptm + +# Modem Control +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon modem TIOCMGET 0x5415 0x4004746a 0x4004746a 0x4004746a 0x4004746a -1 # get status of modem bits; ioctl(fd, TIOCMGET, int *argp) +syscon modem TIOCMSET 0x5418 0x8004746d 0x8004746d 0x8004746d 0x8004746d -1 # set status of modem bits; ioctl(fd, TIOCMSET, const int *argp) +syscon modem TIOCMBIC 0x5417 0x8004746b 0x8004746b 0x8004746b 0x8004746b -1 # clear indicated modem bits; ioctl(fd, TIOCMBIC, int *argp) +syscon modem TIOCMBIS 0x5416 0x8004746c 0x8004746c 0x8004746c 0x8004746c -1 # set indicated modem bits; ioctl(fd, TIOCMBIS, int *argp) +syscon modem TIOCM_LE 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # consensus +syscon modem TIOCM_DTR 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 # consensus +syscon modem TIOCM_RTS 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 # consensus +syscon modem TIOCM_ST 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # consensus +syscon modem TIOCM_SR 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 # consensus +syscon modem TIOCM_CTS 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 # consensus +syscon modem TIOCM_CAR 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # consensus +syscon modem TIOCM_CD 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # boop +syscon modem TIOCM_RI 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # boop +syscon modem TIOCM_RNG 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # boop +syscon modem TIOCM_DSR 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 # consensus +syscon modem TIOCM_DCD 0 0 0x40 0 0 -1 # wut +syscon modem TIOCMODG 0 0x40047403 0 0x4004746a 0x4004746a -1 # wut +syscon modem TIOCMODS 0 0x80047404 0 0x8004746d 0x8004746d -1 # wut +syscon modem TIOCMSDTRWAIT 0 0x8004745b 0x8004745b 0 0 -1 # wut syscon iff IFF_BROADCAST 2 2 2 2 2 2 # consensus syscon iff IFF_LOOPBACK 8 8 8 8 8 4 # unix consensus @@ -1610,97 +1512,6 @@ syscon iff IFF_MASTER 0x0400 0 0 0 0 0 syscon iff IFF_PORTSEL 0x2000 0 0 0 0 0 syscon iff IFF_SLAVE 0x0800 0 0 0 0 0 -syscon nd ND_RA_FLAG_MANAGED 0x80 0x80 0x80 0x80 0x80 0x80 # consensus -syscon nd ND_RA_FLAG_OTHER 0x40 0x40 0x40 0x40 0x40 0x40 # consensus -syscon nd ND_NA_FLAG_OVERRIDE 0x20 0x20 0x20 0x20 0x20 0x20000000 # unix consensus -syscon nd ND_NA_FLAG_ROUTER 0x80 0x80 0x80 0x80 0x80 0x80000000 # unix consensus -syscon nd ND_NA_FLAG_SOLICITED 0x40 0x40 0x40 0x40 0x40 0x40000000 # unix consensus -syscon nd ND_NEIGHBOR_ADVERT 136 136 136 136 136 0 # unix consensus -syscon nd ND_NEIGHBOR_SOLICIT 135 135 135 135 135 0 # unix consensus -syscon nd ND_REDIRECT 137 137 137 137 137 0 # unix consensus -syscon nd ND_ROUTER_ADVERT 134 134 134 134 134 0 # unix consensus -syscon nd ND_ROUTER_SOLICIT 133 133 133 133 133 0 # unix consensus -syscon nd ND_RA_FLAG_HOME_AGENT 0x20 0 0 0 0 0x20 # bsd consensus - -syscon misc TCFLSH 0x540b 0 0 0 0 0 -syscon misc TCIFLUSH 0 1 1 1 1 0 # bsd consensus -syscon misc TCIOFF 2 3 3 3 3 0 # bsd consensus -syscon misc TCIOFLUSH 2 3 3 3 3 0 # bsd consensus -syscon misc TCION 3 4 4 4 4 0 # bsd consensus -syscon misc TCOFLUSH 1 2 2 2 2 0 # bsd consensus -syscon misc TCOOFF 0 1 1 1 1 0 # bsd consensus -syscon misc TCOON 1 2 2 2 2 0 # bsd consensus - -syscon misc TYPE_DISK 0 0 0 0 0 0 # consensus -syscon misc TYPE_A 1 1 1 1 1 0 # unix consensus -syscon misc TYPE_E 2 2 2 2 2 0 # unix consensus -syscon misc TYPE_I 3 3 3 3 3 0 # unix consensus -syscon misc TYPE_L 4 4 4 4 4 0 # unix consensus -syscon misc TYPE_ENCLOSURE 13 0 0 0 0 0 -syscon misc TYPE_MEDIUM_CHANGER 8 0 0 0 0 0 -syscon misc TYPE_MOD 7 0 0 0 0 0 -syscon misc TYPE_NO_LUN 127 0 0 0 0 0 -syscon misc TYPE_PROCESSOR 3 0 0 0 0 0 -syscon misc TYPE_ROM 5 0 0 0 0 0 -syscon misc TYPE_SCANNER 6 0 0 0 0 0 -syscon misc TYPE_TAPE 1 0 0 0 0 0 -syscon misc TYPE_WORM 4 0 0 0 0 0 - -syscon misc _POSIX2_BC_BASE_MAX 99 99 99 99 99 0 # unix consensus -syscon misc _POSIX2_BC_DIM_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus -syscon misc _POSIX2_BC_SCALE_MAX 99 99 99 99 99 0 # unix consensus -syscon misc _POSIX2_BC_STRING_MAX 0x03e8 0x03e8 0x03e8 0x03e8 0x03e8 0 # unix consensus -syscon misc _POSIX2_CHARCLASS_NAME_MAX 14 14 14 14 14 0 # unix consensus -syscon misc _POSIX2_COLL_WEIGHTS_MAX 2 2 2 2 2 0 # unix consensus -syscon misc _POSIX2_EXPR_NEST_MAX 0x20 0x20 0x20 0x20 0x20 0 # unix consensus -syscon misc _POSIX2_LINE_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus -syscon misc _POSIX2_RE_DUP_MAX 255 255 255 255 255 0 # unix consensus -syscon misc _POSIX2_C_BIND 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus -syscon misc _POSIX2_VERSION 0x031069 0x030db0 0x030a2c 0x031069 0x031069 0 - -syscon misc PTHREAD_MUTEX_STALLED 0 0 0 0 0 0 # consensus -syscon misc PTHREAD_PRIO_NONE 0 0 0 0 0 0 # consensus -syscon misc PTHREAD_PRIO_INHERIT 0 1 1 1 1 0 # bsd consensus -syscon misc PTHREAD_PRIO_PROTECT 0 2 2 2 2 0 # bsd consensus -syscon misc PTHREAD_DESTRUCTOR_ITERATIONS 4 4 4 4 4 0 # unix consensus -syscon misc PTHREAD_PROCESS_SHARED 1 1 1 1 1 0 # unix consensus -syscon misc PTHREAD_CREATE_DETACHED 1 2 1 1 1 0 -syscon misc PTHREAD_KEYS_MAX 0x0400 0x0200 0x0100 0x0100 0x0100 0 -syscon misc PTHREAD_STACK_MIN 0x4000 0x2000 0x0800 0x1000 0x1000 0 -syscon misc PTHREAD_BARRIER_SERIAL_THREAD -1 0 -1 -1 -1 0 -syscon misc PTHREAD_CANCEL_ASYNCHRONOUS 1 0 2 2 2 0 -syscon misc PTHREAD_CANCEL_DISABLE 1 0 1 1 1 0 -syscon misc PTHREAD_INHERIT_SCHED 0 1 4 4 4 0 -syscon misc PTHREAD_SCOPE_SYSTEM 0 1 2 2 2 0 -syscon misc PTHREAD_EXPLICIT_SCHED 1 2 0 0 0 0 -syscon misc PTHREAD_MUTEX_DEFAULT 0 0 1 4 4 0 -syscon misc PTHREAD_MUTEX_ERRORCHECK 0 1 0 1 1 0 -syscon misc PTHREAD_MUTEX_RECURSIVE 0 2 0 2 2 0 -syscon misc PTHREAD_SCOPE_PROCESS 1 2 0 0 0 0 -syscon misc PTHREAD_CANCEL_DEFERRED 0 2 0 0 0 0 -syscon misc PTHREAD_CANCEL_ENABLE 0 1 0 0 0 0 -syscon misc PTHREAD_CREATE_JOINABLE 0 1 0 0 0 0 -syscon misc PTHREAD_MUTEX_NORMAL 0 0 0 3 3 0 -syscon misc PTHREAD_MUTEX_ROBUST 0 0 1 0 0 0 -syscon misc PTHREAD_PROCESS_PRIVATE 0 2 0 0 0 0 - -syscon misc N_TTY 0 0 0 0 0 0 # consensus -syscon misc N_6PACK 7 0 0 0 0 0 -syscon misc N_AX25 5 0 0 0 0 0 -syscon misc N_HCI 15 0 0 0 0 0 -syscon misc N_HDLC 13 0 0 0 0 0 -syscon misc N_IRDA 11 0 0 0 0 0 -syscon misc N_MASC 8 0 0 0 0 0 -syscon misc N_MOUSE 2 0 0 0 0 0 -syscon misc N_PPP 3 0 0 0 0 0 -syscon misc N_PROFIBUS_FDL 10 0 0 0 0 0 -syscon misc N_R3964 9 0 0 0 0 0 -syscon misc N_SLIP 1 0 0 0 0 0 -syscon misc N_SMSBLOCK 12 0 0 0 0 0 -syscon misc N_STRIP 4 0 0 0 0 0 -syscon misc N_SYNC_PPP 14 0 0 0 0 0 -syscon misc N_X25 6 0 0 0 0 0 - syscon sock SOCK_STREAM 1 1 1 1 1 1 # consensus syscon sock SOCK_DGRAM 2 2 2 2 2 2 # consensus syscon sock SOCK_RAW 3 3 3 3 3 3 # consensus @@ -1723,32 +1534,6 @@ syscon prsnlty ADDR_NO_RANDOMIZE 0x0040000 0 0 0 0 0 # linux only syscon prsnlty SHORT_INODE 0x1000000 0 0 0 0 0 # linux only syscon prsnlty UNAME26 0x0020000 0 0 0 0 0 # linux only -syscon misc INADDR_ANY 0 0 0 0 0 0 # consensus -syscon misc INADDR_BROADCAST 0xffffffff 0xffffffff 0xffffffff 0xffffffff 0xffffffff 0xffffffff # consensus -syscon misc INADDR_NONE 0xffffffff 0xffffffff 0xffffffff 0xffffffff 0xffffffff 0xffffffff # consensus -syscon misc INADDR_ALLHOSTS_GROUP 0xe0000001 0xe0000001 0xe0000001 0xe0000001 0xe0000001 0 # unix consensus -syscon misc INADDR_LOOPBACK 0x7f000001 0x7f000001 0x7f000001 0x7f000001 0x7f000001 0x7f000001 # consensus -syscon misc INADDR_MAX_LOCAL_GROUP 0xe00000ff 0xe00000ff 0xe00000ff 0xe00000ff 0xe00000ff 0 # unix consensus -syscon misc INADDR_UNSPEC_GROUP 0xe0000000 0xe0000000 0xe0000000 0xe0000000 0xe0000000 0 # unix consensus -syscon misc INADDR_ALLRTRS_GROUP 0xe0000002 0xe0000002 0xe0000002 0 0 0 - -syscon misc BLKTYPE 52 52 52 52 52 0 # unix consensus -syscon misc BLKBSZGET 0x80081270 0 0 0 0 0 -syscon misc BLKBSZSET 0x40081271 0 0 0 0 0 -syscon misc BLKFLSBUF 0x1261 0 0 0 0 0 -syscon misc BLKFRAGET 0x1265 0 0 0 0 0 -syscon misc BLKFRASET 0x1264 0 0 0 0 0 -syscon misc BLKGETSIZE 0x1260 0 0 0 0 0 -syscon misc BLKGETSIZE64 0x80081272 0 0 0 0 0 -syscon misc BLKRAGET 0x1263 0 0 0 0 0 -syscon misc BLKRASET 0x1262 0 0 0 0 0 -syscon misc BLKROGET 0x125e 0 0 0 0 0 -syscon misc BLKROSET 0x125d 0 0 0 0 0 -syscon misc BLKRRPART 0x125f 0 0 0 0 0 -syscon misc BLKSECTGET 0x1267 0 0 0 0 0 -syscon misc BLKSECTSET 0x1266 0 0 0 0 0 -syscon misc BLKSSZGET 0x1268 0 0 0 0 0 - syscon misc TH_FIN 1 1 1 1 1 1 # consensus syscon misc TH_SYN 2 2 2 2 2 2 # consensus syscon misc TH_RST 4 4 4 4 4 4 # consensus @@ -1788,108 +1573,6 @@ syscon misc TCPOPT_SACK_PERMITTED 4 4 4 4 4 0 # unix consensus syscon misc TCPOPT_TIMESTAMP 8 8 8 8 8 0 # unix consensus syscon misc TCPOPT_WINDOW 3 3 3 3 3 0 # unix consensus -syscon misc ETH_P_CUST 0x6006 0 0 0 0 0 -syscon misc ETH_P_DDCMP 6 0 0 0 0 0 -syscon misc ETH_P_DEC 0x6000 0 0 0 0 0 -syscon misc ETH_P_DIAG 0x6005 0 0 0 0 0 -syscon misc ETH_P_DNA_DL 0x6001 0 0 0 0 0 -syscon misc ETH_P_DNA_RC 0x6002 0 0 0 0 0 -syscon misc ETH_P_DNA_RT 0x6003 0 0 0 0 0 -syscon misc ETH_P_IEEE802154 246 0 0 0 0 0 -syscon misc ETH_P_LAT 0x6004 0 0 0 0 0 -syscon misc ETH_P_LOCALTALK 9 0 0 0 0 0 -syscon misc ETH_P_PPP_MP 8 0 0 0 0 0 -syscon misc ETH_P_RARP 0x8035 0 0 0 0 0 -syscon misc ETH_P_SCA 0x6007 0 0 0 0 0 -syscon misc ETH_P_WAN_PPP 7 0 0 0 0 0 - -syscon misc ST_NOSUID 2 2 2 2 2 0 # unix consensus -syscon misc ST_RDONLY 1 1 1 1 1 0 # unix consensus -syscon misc ST_APPEND 0x0100 0 0 0 0 0 -syscon misc ST_IMMUTABLE 0x0200 0 0 0 0 0 -syscon misc ST_MANDLOCK 0x40 0 0 0 0 0 -syscon misc ST_NOATIME 0x0400 0 0 0 0 0 -syscon misc ST_NODEV 4 0 0 0 0 0 -syscon misc ST_NODIRATIME 0x0800 0 0 0 0 0 -syscon misc ST_NOEXEC 8 0 0 0 0 0 -syscon misc ST_RELATIME 0x1000 0 0 0 0 0 -syscon misc ST_SYNCHRONOUS 0x10 0 0 0 0 0 -syscon misc ST_WRITE 0x80 0 0 0 0 0 - -syscon misc SCSI_IOCTL_BENCHMARK_COMMAND 3 0 0 0 0 0 -syscon misc SCSI_IOCTL_DOORLOCK 0x5380 0 0 0 0 0 -syscon misc SCSI_IOCTL_DOORUNLOCK 0x5381 0 0 0 0 0 -syscon misc SCSI_IOCTL_GET_BUS_NUMBER 0x5386 0 0 0 0 0 -syscon misc SCSI_IOCTL_GET_IDLUN 0x5382 0 0 0 0 0 -syscon misc SCSI_IOCTL_PROBE_HOST 0x5385 0 0 0 0 0 -syscon misc SCSI_IOCTL_SEND_COMMAND 1 0 0 0 0 0 -syscon misc SCSI_IOCTL_START_UNIT 5 0 0 0 0 0 -syscon misc SCSI_IOCTL_STOP_UNIT 6 0 0 0 0 0 -syscon misc SCSI_IOCTL_SYNC 4 0 0 0 0 0 -syscon misc SCSI_IOCTL_TAGGED_DISABLE 0x5384 0 0 0 0 0 -syscon misc SCSI_IOCTL_TAGGED_ENABLE 0x5383 0 0 0 0 0 -syscon misc SCSI_IOCTL_TEST_UNIT_READY 2 0 0 0 0 0 -syscon misc BUS_DEVICE_RESET 12 0 0 0 0 0 # SIGBUS; - -syscon misc READ_10 40 0 0 0 0 0 -syscon misc READ_12 168 0 0 0 0 0 -syscon misc READ_6 8 0 0 0 0 0 -syscon misc READ_BLOCK_LIMITS 5 0 0 0 0 0 -syscon misc READ_BUFFER 60 0 0 0 0 0 -syscon misc READ_CAPACITY 37 0 0 0 0 0 -syscon misc READ_DEFECT_DATA 55 0 0 0 0 0 -syscon misc READ_ELEMENT_STATUS 184 0 0 0 0 0 -syscon misc READ_LONG 62 0 0 0 0 0 -syscon misc READ_POSITION 52 0 0 0 0 0 -syscon misc READ_REVERSE 15 0 0 0 0 0 -syscon misc READ_TOC 67 0 0 0 0 0 - -# getpriority() / setpriority() magnums (a.k.a. nice) -# -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary -syscon prio PRIO_PROCESS 0 0 0 0 0 0 # consensus / poly nt -syscon prio PRIO_PGRP 1 1 1 1 1 1 # unix consensus / poly nt -syscon prio PRIO_USER 2 2 2 2 2 2 # unix consensus / poly nt -syscon prio PRIO_MIN -20 -20 -20 -20 -20 -20 # unix consensus / poly nt -syscon prio PRIO_MAX 20 20 20 20 20 20 # unix consensus / poly nt -syscon prio NZERO 20 20 20 20 20 20 # unix consensus / polyfilled nt - -# getaddrinfo() flags -# -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary -syscon gai AI_PASSIVE 1 1 1 1 1 1 # consensus -syscon gai AI_CANONNAME 2 2 2 2 2 2 # consensus -syscon gai AI_NUMERICHOST 4 4 4 4 4 4 # consensus -syscon gai AI_ADDRCONFIG 0x20 0x0400 0x0400 0x40 0x40 0x0400 -syscon gai AI_NUMERICSERV 0x0400 0x1000 8 0x10 0x10 8 -syscon gai AI_ALL 0x10 0x0100 0x0100 0 0 0x0100 -syscon gai AI_V4MAPPED 8 0x0800 0x0800 0 0 0x0800 - -syscon misc BLK_BYTECOUNT 2 2 2 2 2 0 # unix consensus -syscon misc BLK_EOF 0x40 0x40 0x40 0x40 0x40 0 # unix consensus -syscon misc BLK_EOR 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon misc BLK_ERRORS 0x20 0x20 0x20 0x20 0x20 0 # unix consensus -syscon misc BLK_RESTART 0x10 0x10 0x10 0x10 0x10 0 # unix consensus - -syscon misc MODE_B 2 2 2 2 2 0 # unix consensus -syscon misc MODE_C 3 3 3 3 3 0 # unix consensus -syscon misc MODE_S 1 1 1 1 1 0 # unix consensus -syscon misc MODE_SELECT 21 0 0 0 0 0 -syscon misc MODE_SELECT_10 85 0 0 0 0 0 -syscon misc MODE_SENSE 26 0 0 0 0 0 -syscon misc MODE_SENSE_10 90 0 0 0 0 0 - -syscon misc WRITE_10 42 0 0 0 0 0 -syscon misc WRITE_12 170 0 0 0 0 0 -syscon misc WRITE_6 10 0 0 0 0 0 -syscon misc WRITE_BUFFER 59 0 0 0 0 0 -syscon misc WRITE_FILEMARKS 0x10 0 0 0 0 0 -syscon misc WRITE_LONG 63 0 0 0 0 0 -syscon misc WRITE_LONG_2 234 0 0 0 0 0 -syscon misc WRITE_SAME 65 0 0 0 0 0 -syscon misc WRITE_VERIFY 46 0 0 0 0 0 -syscon misc WRITE_VERIFY_12 174 0 0 0 0 0 - syscon lock LOCK_UNLOCK_CACHE 54 0 0 0 0 0 # wut syscon misc ARPHRD_ETHER 1 1 1 1 1 0 # unix consensus @@ -2098,11 +1781,6 @@ syscon misc MCAST_INCLUDE 1 1 1 0 0 0 syscon misc MCAST_EXCLUDE 0 2 2 0 0 0 syscon misc MCAST_MSFILTER 48 0 0 0 0 0 -syscon misc SIG_BLOCK 0 1 1 1 1 0 # bsd consensus; faked nt -syscon misc SIG_UNBLOCK 1 2 2 2 2 1 # bsd consensus; faked nt -syscon misc SIG_SETMASK 2 3 3 3 3 2 # bsd consensus; faked nt -syscon misc SIG_ATOMIC_MIN -2147483648 -2147483648 -9223372036854775808 -2147483648 -2147483648 0 - syscon misc AREGTYPE 0 0 0 0 0 0 # consensus syscon misc B0 0 0 0 0 0 0 # consensus syscon misc CS5 0 0 0 0 0 0 # consensus @@ -2174,7 +1852,6 @@ syscon misc UDP_NO_CHECK6_RX 102 0 0 0 0 0 syscon misc UDP_NO_CHECK6_TX 101 0 0 0 0 0 syscon misc ACK 4 4 4 4 4 0 # unix consensus -syscon misc BIG_ENDIAN 0x10e1 0x10e1 0x10e1 0x10e1 0x10e1 0 # unix consensus syscon misc CDISCARD 15 15 15 15 15 0 # unix consensus syscon misc CDSUSP 25 25 25 25 25 0 # unix consensus syscon misc CEOF 4 4 4 4 4 0 # unix consensus @@ -2212,7 +1889,7 @@ syscon misc EXPR_NEST_MAX 0x20 0x20 0x20 0x20 0x20 0 # unix conse # linux fallocate() flags # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon misc FALLOC_FL_KEEP_SIZE 0x01 -1 -1 -1 -1 -1 # bsd consensus syscon misc FALLOC_FL_PUNCH_HOLE 0x02 -1 -1 -1 -1 -1 # bsd consensus syscon misc FALLOC_FL_NO_HIDE_STALE 0x04 -1 -1 -1 -1 -1 # bsd consensus @@ -2221,513 +1898,9 @@ syscon misc FALLOC_FL_ZERO_RANGE 0x10 -1 -1 -1 -1 0x000980C8 # bsd syscon misc FALLOC_FL_INSERT_RANGE 0x20 -1 -1 -1 -1 -1 # bsd consensus syscon misc FALLOC_FL_UNSHARE_RANGE 0x40 -1 -1 -1 -1 -1 # bsd consensus -syscon misc FIFOTYPE 54 54 54 54 54 0 # unix consensus -syscon misc GRPQUOTA 1 1 1 1 1 0 # unix consensus -syscon misc IF_NAMESIZE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus -syscon misc INTERMEDIATE_C_GOOD 10 0 0 0 0 0 -syscon misc INTERMEDIATE_GOOD 8 0 0 0 0 0 - -syscon misc IOV_MAX 0x0400 0x0400 0x0400 0x0400 0x0400 16 # unix consensus & MSG_MAXIOVLEN -syscon misc LINE_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus -syscon misc LINKED_CMD_COMPLETE 10 0 0 0 0 0 -syscon misc LINKED_FLG_CMD_COMPLETE 11 0 0 0 0 0 - -syscon misc LITTLE_ENDIAN 0x04d2 0x04d2 0x04d2 0x04d2 0x04d2 0 # unix consensus -syscon misc LNKTYPE 49 49 49 49 49 0 # unix consensus -syscon misc MAXNAMLEN 255 255 255 255 255 0 # unix consensus -syscon misc MAXQUOTAS 2 2 2 2 2 0 # unix consensus -syscon misc MEDIUM_ERROR 3 0 0 0 0 0 -syscon misc MEDIUM_SCAN 56 0 0 0 0 0 - -syscon misc NBBY 8 8 8 8 8 0 # unix consensus -syscon misc NR_DQHASH 43 0 0 0 0 0 -syscon misc NR_DQUOTS 0x0100 0 0 0 0 0 - -syscon misc PERSISTENT_RESERVE_IN 94 0 0 0 0 0 -syscon misc PERSISTENT_RESERVE_OUT 95 0 0 0 0 0 - -syscon misc PRELIM 1 1 1 1 1 0 # unix consensus -syscon misc REGTYPE 48 48 48 48 48 0 # unix consensus -syscon misc RES_PRF_CLASS 4 4 4 4 4 0 # unix consensus -syscon misc RHF_GUARANTEE_START_INIT 0x80 0 0 0 0 0 -syscon misc RHF_NO_LIBRARY_REPLACEMENT 4 0 0 0 0 0 - -syscon misc RRQ 1 1 1 1 1 0 # unix consensus -syscon misc RTF_NOFORWARD 0x1000 0 0 0 0 0 -syscon misc RTF_NOPMTUDISC 0x4000 0 0 0 0 0 - -syscon misc SARMAG 8 8 8 8 8 0 # unix consensus -syscon misc SEGSIZE 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus -syscon misc SEND_DIAGNOSTIC 29 0 0 0 0 0 -syscon misc SEND_VOLUME_TAG 182 0 0 0 0 0 - -syscon misc SET_LIMITS 51 0 0 0 0 0 -syscon misc SET_WINDOW 36 0 0 0 0 0 - -syscon misc SFD_CLOEXEC 0x080000 0 0 0 0 0 -syscon misc SFD_NONBLOCK 0x0800 0 0 0 0 0 - -syscon misc SOMAXCONN 0x80 0x80 0x80 0x80 0x80 0x7fffffff # unix consensus -syscon misc SUBCMDMASK 255 255 255 255 255 0 # unix consensus -syscon misc SUBCMDSHIFT 8 8 8 8 8 0 # unix consensus -syscon misc SYMTYPE 50 50 50 50 50 0 # unix consensus -syscon misc TGEXEC 8 8 8 8 8 0 # unix consensus -syscon misc TGREAD 0x20 0x20 0x20 0x20 0x20 0 # unix consensus -syscon misc TGWRITE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus -syscon misc TMAGLEN 6 6 6 6 6 0 # unix consensus -syscon misc TOEXEC 1 1 1 1 1 0 # unix consensus -syscon misc TOREAD 4 4 4 4 4 0 # unix consensus -syscon misc TOWRITE 2 2 2 2 2 0 # unix consensus -syscon misc TRANSIENT 4 4 4 4 4 0 # unix consensus -syscon misc TRY_AGAIN 2 2 2 2 2 0x2afa # unix consensus -syscon misc TSGID 0x0400 0x0400 0x0400 0x0400 0x0400 0 # unix consensus -syscon misc TSUID 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus -syscon misc TSVTX 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus -syscon misc TUEXEC 0x40 0x40 0x40 0x40 0x40 0 # unix consensus -syscon misc TUREAD 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus -syscon misc TUWRITE 0x80 0x80 0x80 0x80 0x80 0 # unix consensus -syscon misc TVERSLEN 2 2 2 2 2 0 # unix consensus -syscon misc WORD_BIT 0x20 0x20 0x20 0x20 0x20 0 # unix consensus -syscon misc WRQ 2 2 2 2 2 0 # unix consensus -syscon misc SIGEV_THREAD 2 3 2 0 2 0 -syscon misc SIGEV_SIGNAL 0 1 1 0 1 0 -syscon misc SIGEV_NONE 1 0 0 0 0 0 - -syscon misc BC_BASE_MAX 99 99 99 0x7fffffff 0x7fffffff 0 -syscon misc BC_DIM_MAX 0x0800 0x0800 0x0800 0xffff 0xffff 0 -syscon misc BC_SCALE_MAX 99 99 99 0x7fffffff 0x7fffffff 0 -syscon misc BC_STRING_MAX 0x03e8 0x03e8 0x03e8 0x7fffffff 0x7fffffff 0 - -syscon misc ABORTED_COMMAND 11 0 0 0 0 0 -syscon misc ACORE 0 8 8 8 8 0 # bsd consensus -syscon misc AFORK 0 1 1 1 1 0 # bsd consensus -syscon misc AIO_ALLDONE 2 1 3 0 0 0 -syscon misc AIO_NOTCANCELED 1 4 2 0 0 0 -syscon misc AIO_CANCELED 0 2 1 0 0 0 - -syscon misc ALLOW_MEDIUM_REMOVAL 30 0 0 0 0 0 -syscon misc ASU 0 2 2 2 2 0 # bsd consensus -syscon misc ATF_NETMASK 0x20 0 0 0 0 0 -syscon misc AXSIG 0 0x10 0x10 0x10 0x10 0 # bsd consensus -syscon misc B1000000 0x1008 0 0 0 0 0 -syscon misc B110 3 110 110 110 110 0 # bsd consensus -syscon misc B115200 0x1002 0x01c200 0x01c200 0x01c200 0x01c200 0 # bsd consensus -syscon misc B1152000 0x1009 0 0 0 0 0 -syscon misc B1200 9 0x04b0 0x04b0 0x04b0 0x04b0 0 # bsd consensus -syscon misc B134 4 134 134 134 134 0 # bsd consensus -syscon misc B150 5 150 150 150 150 0 # bsd consensus -syscon misc B1500000 0x100a 0 0 0 0 0 -syscon misc B1800 10 0x0708 0x0708 0x0708 0x0708 0 # bsd consensus -syscon misc B19200 14 0x4b00 0x4b00 0x4b00 0x4b00 0 # bsd consensus -syscon misc B200 6 200 200 200 200 0 # bsd consensus -syscon misc B2000000 0x100b 0 0 0 0 0 -syscon misc B230400 0x1003 0x038400 0x038400 0x038400 0x038400 0 # bsd consensus -syscon misc B2400 11 0x0960 0x0960 0x0960 0x0960 0 # bsd consensus -syscon misc B2500000 0x100c 0 0 0 0 0 -syscon misc B300 7 300 300 300 300 0 # bsd consensus -syscon misc B3000000 0x100d 0 0 0 0 0 -syscon misc B3500000 0x100e 0 0 0 0 0 -syscon misc B38400 15 0x9600 0x9600 0x9600 0x9600 0 # bsd consensus -syscon misc B4000000 0x100f 0 0 0 0 0 -syscon misc B4800 12 0x12c0 0x12c0 0x12c0 0x12c0 0 # bsd consensus -syscon misc B50 1 50 50 50 50 0 # bsd consensus -syscon misc B500000 0x1005 0 0 0 0 0 -syscon misc B57600 0x1001 0xe100 0xe100 0xe100 0xe100 0 # bsd consensus -syscon misc B576000 0x1006 0 0 0 0 0 -syscon misc B600 8 600 600 600 600 0 # bsd consensus -syscon misc B75 2 75 75 75 75 0 # bsd consensus -syscon misc B9600 13 0x2580 0x2580 0x2580 0x2580 0 # bsd consensus -syscon misc BITSPERBYTE 8 0 0 0 0 0 -syscon misc BLANK_CHECK 8 0 0 0 0 0 -syscon misc CHANGE_DEFINITION 0x40 0 0 0 0 0 -syscon misc CHARBITS 8 0 0 0 0 0 -syscon misc CHECK_CONDITION 1 0 0 0 0 0 -syscon misc CONDITION_GOOD 2 0 0 0 0 0 -syscon misc CREAD 0x80 0x0800 0x0800 0x0800 0x0800 0 # bsd consensus -syscon misc CSTOPB 0x40 0x0400 0x0400 0x0400 0x0400 0 # bsd consensus -syscon misc DATA_PROTECT 7 0 0 0 0 0 -syscon misc DELAYTIMER_MAX 0x7fffffff 0 0 0 0 0 -syscon misc DMAXEXP 0x0400 0 0 0 0 0 -syscon misc DMINEXP -1021 0 0 0 0 0 -syscon misc DOUBLEBITS 0x40 0 0 0 0 0 -syscon misc ERA_D_FMT 0x02002e 46 46 0 0 0 -syscon misc ERA_D_T_FMT 0x020030 47 47 0 0 0 -syscon misc ERA_T_FMT 0x020031 48 48 0 0 0 - -# Teletypewriter Control, e.g. -# -# TCSETS → About 70,800 results (0.31 seconds) -# = TCSETNOW → About 47,700 results (0.31 seconds) -# ≈ TCSETA → About 12,600 results (0.32 seconds) -# = TIOCSETA → About 3,110 results (0.41 seconds) -# -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary -syscon termios TCGETS 0x5401 0x40487413 0x402c7413 0x402c7413 0x402c7413 0 # Gets console settings; tcgetattr(tty, argp) → ioctl(tty, TCGETS, struct termios *argp); polyfilled NT -syscon compat TIOCGETA 0x5401 0x40487413 0x402c7413 0x402c7413 0x402c7413 0 # Gets console settings; = tcgetattr(tty, struct termios *argp) -#syscon compat TCGETA 0x5405 0 0 0 0 0 # Gets console settings; ≈ ioctl(fd, TCGETA, struct termio *argp) -syscon termios TCSANOW 0 0 0 0 0 0 # Sets console settings; tcsetattr(fd, TCSANOW, argp); polyfilled NT -syscon termios TCSETS 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; = ioctl(tty, TCSETS, const struct termios *argp); polyfilled NT -syscon compat TIOCSETA 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; = ioctl(tty, TIOCSETA, const struct termios *argp); polyfilled NT -#syscon compat TCSETA 0x5402 0x80487414 0x802c7414 0x802c7414 0x802c7414 0x5402 # Sets console settings; ≈ ioctl(tty, TCSETA, const struct termio *argp); polyfilled NT -syscon termios TCSADRAIN 1 1 1 1 1 1 # Drains output & sets console settings; tcsetawttr(fd, TCSADRAIN, argp); polyfilled NT -syscon termios TCSETSW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; = ioctl(tty, TCSETSW, const struct termios *argp); polyfilled NT -syscon compat TIOCSETAW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; = ioctl(tty, TIOCSETAW, const struct termios *argp); polyfilled NT -#syscon compat TCSETAW 0x5403 0x80487415 0x802c7415 0x802c7415 0x802c7415 0x5403 # Drains output & sets console settings; ≈ ioctl(tty, TCSETAW, const struct termio *argp); polyfilled NT -syscon termios TCSAFLUSH 2 2 2 2 2 2 # Drops input & drains output & sets console settings; tcsetafttr(fd, TCSAFLUSH, argp); polyfilled NT -syscon termios TCSETSF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5404 # Drops input & drains output & sets console settings; = ioctl(tty, TCSETSF, const struct termios *argp); polyfilled NT -syscon compat TIOCSETAF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5402 # Drops input & drains output & sets console settings; = ioctl(tty, TIOCSETAF, const struct termios *argp); polyfilled NT -#syscon compat TCSETAF 0x5404 0x80487416 0x802c7416 0x802c7416 0x802c7416 0x5402 # Drops input & drains output & sets console settings; ≈ ioctl(tty, TCSETAF, const struct termio *argp); polyfilled NT -syscon termios TIOCGWINSZ 0x5413 1074295912 1074295912 1074295912 1074295912 0x5413 # ioctl(tty, TIOCGWINSZ, struct winsize *argp); polyfilled NT -syscon termios TIOCSWINSZ 0x5414 0x80087467 0x80087467 0x80087467 0x80087467 0x5414 # ioctl(tty, TIOCSWINSZ, const struct winsize *argp) (faked NT) -syscon termios TIOCOUTQ 0x5411 0x40047473 0x40047473 0x40047473 0x40047473 0 # get # bytes queued in TTY's output buffer ioctl(tty, TIOCSWINSZ, const struct winsize *argp) -syscon termios TIOCCBRK 0x5428 0x2000747a 0x2000747a 0x2000747a 0x2000747a 0 # boop -syscon termios TIOCCONS 0x541d 0x80047462 0x80047462 0x80047462 0x80047462 0 # boop -syscon termios TIOCGETD 0x5424 0x4004741a 0x4004741a 0x4004741a 0x4004741a 0 # boop -syscon termios TIOCGPGRP 0x540f 0x40047477 0x40047477 0x40047477 0x40047477 0 # boop -syscon termios TIOCNOTTY 0x5422 0x20007471 0x20007471 0x20007471 0x20007471 0 # boop -syscon termios TIOCNXCL 0x540d 0x2000740e 0x2000740e 0x2000740e 0x2000740e 0 # boop -syscon termios TIOCSBRK 0x5427 0x2000747b 0x2000747b 0x2000747b 0x2000747b 0 # boop -syscon termios TIOCSCTTY 0x540e 0x20007461 0x20007461 0x20007461 0x20007461 0 # boop -syscon termios TIOCSETD 0x5423 0x8004741b 0x8004741b 0x8004741b 0x8004741b 0 # boop -syscon termios TIOCSIG 0x40045436 0x2000745f 0x2004745f 0x8004745f 0x8004745f 0 # boop -syscon termios TIOCSPGRP 0x5410 0x80047476 0x80047476 0x80047476 0x80047476 0 # boop -syscon termios TIOCSTI 0x5412 0x80017472 0x80017472 0 0 0 # boop -syscon termios TIOCGPTN 0x80045430 0 0x4004740f 0 0 0 # boop -syscon termios TIOCGSID 0x5429 0 0x40047463 0x40047463 0x40047463 0 # boop -syscon termios TABLDISC 0 0x3 0 0x3 0x3 0 # boop -syscon termios SLIPDISC 0 0x4 0x4 0x4 0x4 0 # boop -syscon termios PPPDISC 0 0x5 0x5 0x5 0x5 0 # boop -syscon termios TIOCDRAIN 0 0x2000745e 0x2000745e 0x2000745e 0x2000745e 0 # boop -syscon termios TIOCSTAT 0 0x20007465 0x20007465 0x20007465 0x20007465 0 # boop -syscon termios TIOCSTART 0 0x2000746e 0x2000746e 0x2000746e 0x2000746e 0 # boop -syscon termios TIOCCDTR 0 0x20007478 0x20007478 0x20007478 0x20007478 0 # boop -syscon termios TIOCSDTR 0 0x20007479 0x20007479 0x20007479 0x20007479 0 # boop -syscon termios TIOCFLUSH 0 0x80047410 0x80047410 0x80047410 0x80047410 0 # boop -syscon termios TIOCEXT 0 0x80047460 0x80047460 0x80047460 0x80047460 0 # boop -syscon termios TIOCGDRAINWAIT 0 0x40047456 0x40047456 0 0 0 # boop -syscon termios TIOCTIMESTAMP 0 0x40107459 0x40107459 0 0 0 # boop -syscon termios TIOCSDRAINWAIT 0 0x80047457 0x80047457 0 0 0 # boop -syscon termios TIOCREMOTE 0 0x80047469 0 0x80047469 0x80047469 0 # boop -syscon termios TTYDISC 0 0 0 0 0 0 # boop -syscon termios TIOCFLAG_SOFTCAR 0 0 0 0x1 0x1 0 # boop -syscon termios TIOCFLAG_PPS 0 0 0 0x10 0x10 0 # boop -syscon termios TIOCFLAG_CLOCAL 0 0 0 0x2 0x2 0 # boop -syscon termios TIOCCHKVERAUTH 0 0 0 0x2000741e 0x2000741e 0 # boop -syscon termios TIOCGFLAGS 0 0 0 0x4004745d 0x4004745d 0 # boop -syscon termios TIOCGTSTAMP 0 0 0 0x4010745b 0x4010745b 0 # boop -syscon termios STRIPDISC 0 0 0 0x6 0x6 0 # boop -syscon termios NMEADISC 0 0 0 0x7 0x7 0 # boop -syscon termios TIOCUCNTL_CBRK 0 0 0 0x7a 0x7a 0 # boop -syscon termios TIOCFLAG_MDMBUF 0 0 0 0x8 0x8 0 # boop -syscon termios TIOCSETVERAUTH 0 0 0 0x8004741c 0x8004741c 0 # boop -syscon termios TIOCSFLAGS 0 0 0 0x8004745c 0x8004745c 0 # boop -syscon termios TIOCSTSTAMP 0 0 0 0x8008745a 0x8008745a 0 # boop -syscon termios ENDRUNDISC 0 0 0 0x9 0x9 0 # boop -syscon termios TIOCPTMASTER 0 0 0x2000741c 0 0 0 # boop -syscon termios NETGRAPHDISC 0 0 0x6 0 0 0 # boop -syscon termios H4DISC 0 0 0x7 0 0 0 # boop - -syscon termios ISIG 0b0000000000000001 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000000000001 # termios.c_lflag|=ISIG makes Ctrl-C, Ctrl-\, etc. generate signals -syscon termios ICANON 0b0000000000000010 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000000000010 # termios.c_lflag&=~ICANON disables 1960's version of gnu readline (see also VMIN) -syscon termios XCASE 0b0000000000000100 0 0 16777216 16777216 0b0000000000000100 # termios.c_lflag -syscon termios ECHO 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # termios.c_lflag&=~ECHO is for passwords and raw mode -syscon termios ECHOE 0b0000000000010000 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000010000 # termios.c_lflag -syscon termios ECHOK 0b0000000000100000 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000100000 # termios.c_lflag -syscon termios ECHONL 0b0000000001000000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000001000000 # termios.c_lflag -syscon termios NOFLSH 0b0000000010000000 2147483648 2147483648 2147483648 2147483648 0b0000000010000000 # termios.c_lflag|=NOFLSH means don't flush on INT/QUIT/SUSP -syscon termios TOSTOP 0b0000000100000000 4194304 4194304 4194304 4194304 0b0000000100000000 # termios.c_lflag|=TOSTOP raises SIGTTOU to process group if background job tries to write to controlling terminal -syscon termios ECHOCTL 0b0000001000000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000001000000000 # termios.c_lflag|=ECHOCTL prints ^𝑥 codes for monotonic motion -syscon termios ECHOPRT 0b0000010000000000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000010000000000 # termios.c_lflag|=ECHOPRT includes the parity bit -syscon termios ECHOKE 0b0000100000000000 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000100000000000 # termios.c_lflag -syscon termios FLUSHO 0b0001000000000000 8388608 8388608 8388608 8388608 0b0001000000000000 # termios.c_lflag -syscon termios PENDIN 0b0100000000000000 536870912 536870912 536870912 536870912 0b0100000000000000 # termios.c_lflag -syscon termios IEXTEN 0b1000000000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b1000000000000000 # termios.c_lflag&=~IEXTEN disables platform input processing magic -syscon termios EXTPROC 65536 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 65536 # termios.c_lflag - -syscon termios IGNBRK 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # termios.c_iflag it's complicated, uart only? UNIXCONSENSUS -syscon termios BRKINT 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 # termios.c_iflag it's complicated, uart only? UNIXCONSENSUS -syscon termios IGNPAR 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 # termios.c_iflag|=IGNPAR ignores parity and framing errors; see PARMRK UNIXCONSENSUS -syscon termios PARMRK 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # termios.c_iflag|=PARMRK passes-through parity bit UNIXCONSENSUS -syscon termios INPCK 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 # termios.c_iflag|=INPCK enables parity checking UNIXCONSENSUS -syscon termios ISTRIP 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 # termios.c_iflag|=ISTRIP automates read(1)&0x7f UNIXCONSENSUS -syscon termios INLCR 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # termios.c_iflag|=INLCR maps \n → \r input UNIXCONSENSUS -syscon termios IGNCR 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # termios.c_iflag|=IGNCR maps \r → ∅ input UNIXCONSENSUS -syscon termios ICRNL 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 # termios.c_iflag|=ICRNL maps \r → \n input UNIXCONSENSUS -syscon termios IUCLC 0b0000001000000000 0 0 0b0001000000000000 0b0001000000000000 0b0000001000000000 # termios.c_iflag|=IUCLC maps A-Z → a-z input -syscon termios IXON 0b0000010000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000010000000000 # termios.c_iflag|=IXON enables flow rida -syscon termios IXANY 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 0b0000100000000000 # termios.c_iflag|=IXANY tying will un-stuck teletype UNIXCONSENSUS -syscon termios IXOFF 0b0001000000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0000010000000000 0b0001000000000000 # termios.c_iflag|=IXOFF disables annoying display freeze keys -syscon termios IMAXBEL 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 0b0010000000000000 # termios.c_iflag|=IMAXBEL rings when queue full UNIXCONSENSUS -syscon termios IUTF8 0b0100000000000000 0b0100000000000000 0 0 0 0b0100000000000000 # termios.c_iflag|=IUTF8 helps w/ rubout on UTF-8 input - -syscon termios OPOST 0b0000000000000001 0b000000000000000001 0b000000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # termios.c_oflag&=~OPOST disables output processing magic -syscon termios OLCUC 0b0000000000000010 0b000000000000000000 0 0b0000000000100000 0b0000000000100000 0b0000000000000010 # termios.c_oflag|=OLCUC maps a-z → A-Z output -syscon termios ONLCR 0b0000000000000100 0b000000000000000010 0b000000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000100 # termios.c_oflag|=ONLCR maps \n → \r\n output -syscon termios OCRNL 0b0000000000001000 0b000000000000010000 0b000000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000001000 # termios.c_oflag|=OCRNL maps \r → \n output -syscon termios ONOCR 0b0000000000010000 0b000000000000100000 0b000000000000100000 0b0000000001000000 0b0000000001000000 0b0000000000010000 # termios.c_oflag|=ONOCR maps \r → ∅ output iff column 0 -syscon termios ONLRET 0b0000000000100000 0b000000000001000000 0b000000000001000000 0b0000000010000000 0b0000000010000000 0b0000000000100000 # termios.c_oflag|=ONLRET maps \r → ∅ output -syscon termios OFILL 0b0000000001000000 0b000000000010000000 0 0 0 0b0000000001000000 # termios.c_oflag -syscon termios OFDEL 0b0000000010000000 0b100000000000000000 0 0 0 0b0000000010000000 # termios.c_oflag -syscon termios NLDLY 0b0000000100000000 0b000000001100000000 0b000000001100000000 0 0 0b0000000100000000 # (termios.c_oflag & NLDLY) ∈ {NL0,NL1,NL2,NL3} -syscon termios NL0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # (termios.c_oflag & NLDLY) == NL0 -syscon termios NL1 0b0000000100000000 0b000000000100000000 0b000000000100000000 0 0 0b0000000100000000 # (termios.c_oflag & NLDLY) == NL1 -syscon termios NL2 0 0b000000001000000000 0b000000001000000000 0 0 0 # (termios.c_oflag & NLDLY) == NL2 -syscon termios NL3 0 0b000000001100000000 0b000000001100000000 0 0 0 # (termios.c_oflag & NLDLY) == NL3 -syscon termios CRDLY 0b0000011000000000 0b000011000000000000 0b000011000000000000 0 0 0b0000011000000000 # (termios.c_oflag & CRDLY) ∈ {CR0,CR1,CR2,CR3} -syscon termios CR0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0x0 0x0 0b0000000000000000 # (termios.c_oflag & CRDLY) == CR0 -syscon termios CR1 0b0000001000000000 0b000001000000000000 0b000001000000000000 0x0 0x0 0b0000001000000000 # (termios.c_oflag & CRDLY) == CR1 -syscon termios CR2 0b0000010000000000 0b000010000000000000 0b000010000000000000 0x0 0x0 0b0000010000000000 # (termios.c_oflag & CRDLY) == CR2 -syscon termios CR3 0b0000011000000000 0b000011000000000000 0b000011000000000000 0x0 0x0 0b0000011000000000 # (termios.c_oflag & CRDLY) == CR3 -syscon termios TABDLY 0b0001100000000000 0b000000110000000100 0b000000000000000100 0 0 0b0001100000000000 # (termios.c_oflag & TABDLY) ∈ {TAB0,TAB1,TAB2,TAB3,XTABS} -syscon termios TAB0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # (termios.c_oflag & TABDLY) == TAB0 -syscon termios TAB1 0b0000100000000000 0b000000010000000000 0b000000010000000000 0 0 0b0000100000000000 # (termios.c_oflag & TABDLY) == TAB1 -syscon termios TAB2 0b0001000000000000 0b000000100000000000 0b000000100000000000 0 0 0b0001000000000000 # (termios.c_oflag & TABDLY) == TAB2 -syscon termios TAB3 0b0001100000000000 0b000000000000000100 0b000000000000000100 0 0 0b0001100000000000 # (termios.c_oflag & TABDLY) == TAB3 -syscon termios XTABS 0b0001100000000000 0b000000000000000000 0b000000110000000000 0 0 0b0001100000000000 # (termios.c_oflag & TABDLY) == XTABS -syscon termios BSDLY 0b0010000000000000 0b001000000000000000 0b001000000000000000 0 0 0b0010000000000000 # termios.c_oflag -syscon termios BS0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag -syscon termios BS1 0b0010000000000000 0b001000000000000000 0b001000000000000000 0 0 0b0010000000000000 # termios.c_oflag -syscon termios VTDLY 0b0100000000000000 0b010000000000000000 0b010000000000000000 0 0 0b0100000000000000 # termios.c_oflag -syscon termios VT0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag -syscon termios VT1 0b0100000000000000 0b010000000000000000 0b010000000000000000 0 0 0b0100000000000000 # termios.c_oflag -syscon termios FFDLY 0b1000000000000000 0b000100000000000000 0b000100000000000000 0 0 0b1000000000000000 # termios.c_oflag -syscon termios FF0 0b0000000000000000 0b000000000000000000 0b000000000000000000 0 0 0b0000000000000000 # termios.c_oflag -syscon termios FF1 0b1000000000000000 0b000100000000000000 0b000100000000000000 0 0 0b1000000000000000 # termios.c_oflag - -syscon termios CS6 0b0000000000010000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000000010000 # termios.c_cflag flag for 6-bit characters -syscon termios CS7 0b0000000000100000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000001000000000 0b0000000000100000 # termios.c_cflag flag for 7-bit characters -syscon termios CS8 0b0000000000110000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000000000110000 # termios.c_cflag flag for 8-bit characters -syscon termios CSIZE 0b0000000000110000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000001100000000 0b0000000000110000 # mask for CS𝑥 flags - -syscon termios NCCS 32 32 32 32 20 32 # ARRAYLEN(termios.c_cc); faked xnu/freebsd/openbsd (originally 20) and faked nt -syscon termios VINTR 0 8 8 8 8 0 # termios.c_cc[VINTR]=𝑥 -syscon termios VQUIT 1 9 9 9 9 0 # termios.c_cc[VQUIT]=𝑥 -syscon termios VERASE 2 3 3 3 3 0 # termios.c_cc[VERASE]=𝑥 -syscon termios VKILL 3 5 5 5 5 0 # termios.c_cc[VKILL]=𝑥 -syscon termios VEOF 4 0 0 0 0 0 # termios.c_cc[VEOF]=𝑥 -syscon termios VTIME 5 17 17 17 17 0 # termios.c_cc[VTIME]=𝑥 sets non-canonical read timeout to 𝑥×𝟷𝟶𝟶ms which is needed when entering escape sequences manually with the escape key -syscon termios VMIN 6 16 16 16 16 0 # termios.c_cc[VMIN]=𝑥 in non-canonical mode can be set to 0 for non-blocking reads, 1 for single character raw mode reads, or higher to buffer -syscon termios VSWTC 7 0 0 0 0 0 # termios.c_cc[VSWTC]=𝑥 -syscon termios VSTART 8 12 12 12 12 0 # termios.c_cc[VSTART]=𝑥 -syscon termios VSTOP 9 13 13 13 13 0 # termios.c_cc[VSTOP]=𝑥 -syscon termios VSUSP 10 10 10 10 10 0 # termios.c_cc[VSUSP]=𝑥 defines suspend, i.e. Ctrl-Z (a.k.a. →, ^Z, SUB, 26, 032, 0x1A, ord('Z')^0b01000000); unix consensus -syscon termios VEOL 11 1 1 1 1 0 # termios.c_cc[VEOL]=𝑥 -syscon termios VEOL2 16 2 2 2 2 0 # termios.c_cc[VEOL2]=𝑥 -syscon termios VREPRINT 12 6 6 6 6 0 # termios.c_cc[VREPRINT]=𝑥 -syscon termios VDISCARD 13 15 15 15 15 0 # termios.c_cc[VDISCARD]=𝑥 -syscon termios VWERASE 14 4 4 4 4 0 # termios.c_cc[VWERASE]=𝑥 -syscon termios VLNEXT 15 14 14 14 14 0 # termios.c_cc[VLNEXT]=𝑥 - -syscon termios TIOCSERGETLSR 0x5459 0 0 0 0 0 # -syscon termios TIOCSERGETMULTI 0x545a 0 0 0 0 0 # -syscon termios TIOCSERSETMULTI 0x545b 0 0 0 0 0 # -syscon termios TIOCSER_TEMT 1 0 0 0 0 0 # -syscon termios VERIFY 47 0 0 0 0 0 -syscon termios PARENB 0x0100 0x1000 0x1000 0x1000 0x1000 0 # -syscon termios PARODD 0x0200 0x2000 0x2000 0x2000 0x2000 0 # -syscon termios CIBAUD 0x100f0000 0 0 0 0 0 -syscon termios CLOCAL 0x0800 0x8000 0x8000 0x8000 0x8000 0 # -syscon termios CMSPAR 0x40000000 0 0 0 0 0 -syscon termios BUSY 4 0 0 0 0 0 -syscon termios CANBSIZ 255 0 0 0 0 0 -syscon termios CBAUD 0x100f 0 0 0 0 0 -syscon termios CBAUDEX 0x1000 0 0 0 0 0 -syscon termios CBRK 0 255 255 255 255 0 # -syscon termios CEOL 0 255 255 255 255 0 # - -# Pseudoteletypewriter Control -# -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary -syscon pty TIOCPKT 0x5420 0x80047470 0x80047470 0x80047470 0x80047470 -1 # boop -syscon pty TIOCPKT_DATA 0 0 0 0 0 0 # consensus -syscon pty TIOCPKT_FLUSHREAD 1 1 1 1 1 1 # unix consensus -syscon pty TIOCPKT_FLUSHWRITE 2 2 2 2 2 2 # unix consensus -syscon pty TIOCPKT_STOP 4 4 4 4 4 4 # unix consensus -syscon pty TIOCPKT_START 8 8 8 8 8 8 # unix consensus -syscon pty TIOCPKT_NOSTOP 16 16 16 16 16 16 # unix consensus -syscon pty TIOCPKT_DOSTOP 32 32 32 32 32 32 # unix consensus -syscon pty TIOCPKT_IOCTL 64 64 64 64 64 64 # unix consensus -syscon pty TIOCSPTLCK 0x40045431 0 0 0 0 -1 # boop -syscon pty PTMGET 0 0 0 0x40287401 0x40287401 -1 # for /dev/ptm - -# Modem Control -# -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX Commentary -syscon modem TIOCMGET 0x5415 0x4004746a 0x4004746a 0x4004746a 0x4004746a -1 # get status of modem bits; ioctl(fd, TIOCMGET, int *argp) -syscon modem TIOCMSET 0x5418 0x8004746d 0x8004746d 0x8004746d 0x8004746d -1 # set status of modem bits; ioctl(fd, TIOCMSET, const int *argp) -syscon modem TIOCMBIC 0x5417 0x8004746b 0x8004746b 0x8004746b 0x8004746b -1 # clear indicated modem bits; ioctl(fd, TIOCMBIC, int *argp) -syscon modem TIOCMBIS 0x5416 0x8004746c 0x8004746c 0x8004746c 0x8004746c -1 # set indicated modem bits; ioctl(fd, TIOCMBIS, int *argp) -syscon modem TIOCM_LE 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 0b0000000000000001 # consensus -syscon modem TIOCM_DTR 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 0b0000000000000010 # consensus -syscon modem TIOCM_RTS 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 0b0000000000000100 # consensus -syscon modem TIOCM_ST 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 0b0000000000001000 # consensus -syscon modem TIOCM_SR 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 0b0000000000010000 # consensus -syscon modem TIOCM_CTS 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 0b0000000000100000 # consensus -syscon modem TIOCM_CAR 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # consensus -syscon modem TIOCM_CD 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 0b0000000001000000 # boop -syscon modem TIOCM_RI 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # boop -syscon modem TIOCM_RNG 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 0b0000000010000000 # boop -syscon modem TIOCM_DSR 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 0b0000000100000000 # consensus -syscon modem TIOCM_DCD 0 0 0x40 0 0 -1 # wut -syscon modem TIOCMODG 0 0x40047403 0 0x4004746a 0x4004746a -1 # wut -syscon modem TIOCMODS 0 0x80047404 0 0x8004746d 0x8004746d -1 # wut -syscon modem TIOCMSDTRWAIT 0 0x8004745b 0x8004745b 0 0 -1 # wut - -syscon ioctl FIONBIO 0x5421 0x8004667e 0x8004667e 0x8004667e 0x8004667e 0x8004667e # BSD-Windows consensus; FIONBIO is traditional O_NONBLOCK; see F_SETFL for re-imagined api -syscon ioctl FIOASYNC 0x5452 0x8004667d 0x8004667d 0x8004667d 0x8004667d 0x8004667d # BSD-Windows consensus -syscon ioctl FIONREAD 0x541b 0x4004667f 0x4004667f 0x4004667f 0x4004667f 0x4004667f # BSD-Windows consensus; bytes waiting in FD's input buffer -#syscon ioctl FIONWRITE 0x0 0x0 0x40046677 0x0 0x0 -1 # [FreeBSD Generalization] bytes queued in FD's output buffer (same as TIOCOUTQ for TTY FDs; see also SO_SNDBUF) -#syscon ioctl FIONSPACE 0x0 0x0 0x40046676 0x0 0x0 -1 # [FreeBSD Generalization] capacity of FD's output buffer, e.g. equivalent to TIOCGSERIAL w/ UART -syscon ioctl TIOCINQ 0x541b 0x4004667f 0x4004667f 0x4004667f 0x4004667f 0x4004667f # [Linuxism] same as FIONREAD -#syscon ioctl TIOCOUTQ 0x5411 0x40047473 0x40047473 0x40047473 0x40047473 -1 # bytes queued in TTY's output buffer - -syscon misc FANOTIFY_METADATA_VERSION 3 0 0 0 0 0 -syscon misc FAPPEND 0x0400 8 8 8 8 0 # bsd consensus -syscon misc FASYNC 0x2000 0x40 0x40 0x40 0x40 0 # bsd consensus -syscon misc FFSYNC 0x101000 0x80 0x80 0x80 0x80 0 # bsd consensus -syscon misc FILENAME_MAX 0x1000 0x0400 0x0400 0x0400 0x0400 0 # bsd consensus -syscon misc FIOGETOWN 0x8903 0x4004667b 0x4004667b 0x4004667b 0x4004667b 0 # bsd consensus -syscon misc FIOSETOWN 0x8901 0x8004667c 0x8004667c 0x8004667c 0x8004667c 0 # bsd consensus -syscon misc FMAXEXP 0x80 0 0 0 0 0 -syscon misc FMINEXP -125 0 0 0 0 0 -syscon misc FNDELAY 0x0800 4 4 4 4 0 # bsd consensus -syscon misc FNONBLOCK 0x0800 4 4 4 4 0 # bsd consensus -syscon misc FOPEN_MAX 0x10 20 20 20 20 0 # bsd consensus -syscon misc FORMAT_UNIT 4 0 0 0 0 0 -syscon misc HARDWARE_ERROR 4 0 0 0 0 0 -syscon misc HEAD_OF_QUEUE_TAG 33 0 0 0 0 0 -syscon misc HUPCL 0x0400 0x4000 0x4000 0x4000 0x4000 0 # bsd consensus -syscon misc IGMP_MEMBERSHIP_QUERY 17 0 0 0 0 0 -syscon misc ILLEGAL_REQUEST 5 0 0 0 0 0 -syscon misc INITIATE_RECOVERY 15 0 0 0 0 0 -syscon misc INITIATOR_ERROR 5 0 0 0 0 0 -syscon misc INQUIRY 18 0 0 0 0 0 -syscon misc MAXHOSTNAMELEN 0x40 0x0100 0x0100 0x0100 0x0100 0 # bsd consensus -syscon misc MAXPATHLEN 255 255 255 255 255 255 # forced consensus -syscon misc MAXSYMLINKS 20 0x20 0x20 0x20 0x20 0 # bsd consensus -syscon misc MESSAGE_REJECT 7 0 0 0 0 0 -syscon misc MISCOMPARE 14 0 0 0 0 0 -syscon misc MOVE_MEDIUM 165 0 0 0 0 0 -syscon misc MTCOMPRESSION 0x20 0 0 0 0 0 -syscon misc MTFSFM 11 0 0 0 0 0 -syscon misc MTLOCK 28 0 0 0 0 0 -syscon misc MTMKPART 34 0 0 0 0 0 -syscon misc MTRAS1 14 0 0 0 0 0 -syscon misc MTRAS3 0x10 0 0 0 0 0 -syscon misc MTSETBLK 20 0 0 0 0 0 -syscon misc MTSETDENSITY 21 0 0 0 0 0 -syscon misc MTSETDRVBUFFER 24 0 0 0 0 0 -syscon misc MTSETPART 33 0 0 0 0 0 -syscon misc MTUNLOAD 31 0 0 0 0 0 -syscon misc MTUNLOCK 29 0 0 0 0 0 -syscon misc NCARGS 0x020000 0x040000 0x040000 0x040000 0x040000 0 # bsd consensus -syscon misc NGREG 23 0 0 0 0 0 -syscon misc NOGROUP -1 0xffff 0xffff 0xffff 0xffff 0 # bsd consensus -syscon misc ORDERED_QUEUE_TAG 34 0 0 0 0 0 -syscon misc ORIG_RAX 15 0 0 0 0 0 -syscon misc PIPE_BUF 0x1000 0x0200 0x0200 0x0200 0x0200 0 # bsd consensus -syscon misc PRE_FETCH 52 0 0 0 0 0 -syscon misc QUEUE_FULL 20 0 0 0 0 0 -syscon misc REASSIGN_BLOCKS 7 0 0 0 0 0 -syscon misc RECEIVE_DIAGNOSTIC 28 0 0 0 0 0 -syscon misc RECOVERED_ERROR 1 0 0 0 0 0 -syscon misc RECOVER_BUFFERED_DATA 20 0 0 0 0 0 -syscon misc RELEASE_RECOVERY 0x10 0 0 0 0 0 -syscon misc REQUEST_SENSE 3 0 0 0 0 0 -syscon misc RESERVATION_CONFLICT 12 0 0 0 0 0 -syscon misc RESERVE 22 0 0 0 0 0 -syscon misc RESERVE_10 86 0 0 0 0 0 -syscon misc RESTORE_POINTERS 3 0 0 0 0 0 -syscon misc REZERO_UNIT 1 0 0 0 0 0 -syscon misc RE_DUP_MAX 0x7fff 255 255 255 255 0 # bsd consensus -syscon misc RTCF_DOREDIRECT 0x01000000 0 0 0 0 0 -syscon misc SAVE_POINTERS 2 0 0 0 0 0 -syscon misc SEM_VALUE_MAX 0x7fffffff 0x7fff 0x7fffffff 0xffffffff 0xffffffff 0 -syscon misc SEM_INFO 19 0 11 0 0 0 -syscon misc SEM_STAT 18 0 10 0 0 0 - -syscon misc SHMLBA 0 0x1000 0x1000 0x1000 0x1000 0 # bsd consensus -syscon misc SIMPLE_QUEUE_TAG 0x20 0 0 0 0 0 -syscon misc SPACE 17 0 0 0 0 0 -syscon misc START_STOP 27 0 0 0 0 0 -syscon misc STATUS_MASK 62 0 0 0 0 0 -syscon misc SWAP_FLAG_DISCARD 0x010000 0 0 0 0 0 -syscon misc SYNCHRONIZE_CACHE 53 0 0 0 0 0 -syscon misc UMOUNT_NOFOLLOW 8 0 0 0 0 0 -syscon misc UNIT_ATTENTION 6 0 0 0 0 0 -syscon misc UPDATE_BLOCK 61 0 0 0 0 0 -syscon misc UT_HOSTSIZE 0x0100 0x10 0 0x0100 0x0100 0 -syscon misc UT_LINESIZE 0x20 8 0 8 8 0 -syscon misc UT_NAMESIZE 0x20 8 0 0x20 0x20 0 - -syscon misc WEOF 0xffffffff -1 -1 -1 -1 -1 # bsd consensus (win fake) -syscon misc _LINUX_QUOTA_VERSION 2 0 0 0 0 0 -syscon misc _SEM_SEMUN_UNDEFINED 1 0 0 0 0 0 -syscon misc D_FMT 0x020029 2 2 1 1 0 -syscon misc D_T_FMT 0x020028 1 1 0 0 0 - -syscon misc LOGIN_PROCESS 6 6 6 0 0 0 -syscon misc LOGIN_NAME_MAX 0x0100 0 0 0x20 0x20 0 - -syscon misc T_FMT 0x02002a 3 3 2 2 0 -syscon misc T_FMT_AMPM 0x02002b 4 4 3 3 0 - -syscon misc UL_GETFSIZE 1 1 1 0 0 0 -syscon misc UL_SETFSIZE 2 2 2 0 0 0 - -syscon misc XATTR_CREATE 1 2 0 0 0 0 -syscon misc XATTR_REPLACE 2 4 0 0 0 0 - -syscon misc ACCOUNTING 9 9 0 0 0 0 -syscon misc AHZ 100 0x40 0 0x40 0x40 0 -syscon misc ALT_DIGITS 0x02002f 49 49 0 0 0 -syscon misc AM_STR 0x020026 5 5 4 4 0 -syscon misc B460800 0x1004 0 0x070800 0 0 0 -syscon misc B921600 0x1007 0 0x0e1000 0 0 0 -syscon misc BOOT_TIME 2 2 1 0 0 0 -syscon misc CHARCLASS_NAME_MAX 0x0800 14 14 0 0 0 -syscon misc CLOCKS_PER_SEC 1000000 1000000 0x80 100 100 10000000 -syscon misc CODESET 14 0 0 51 51 0 -syscon misc COLL_WEIGHTS_MAX 255 2 10 2 2 0 -syscon misc CPU_SETSIZE 0x0400 0 0x0100 0 0 0 -syscon misc CRNCYSTR 0x04000f 56 56 50 50 0 -syscon misc CRTSCTS 0x80000000 0x030000 0x030000 0x010000 0x010000 0 -syscon misc CSTATUS 0 20 20 255 255 0 -syscon misc DEAD_PROCESS 8 8 7 0 0 0 -syscon misc FNM_NOSYS -1 -1 -1 2 2 0 -syscon misc INIT_PROCESS 5 5 5 0 0 0 -syscon misc MINSIGSTKSZ 0x0800 0x8000 0x0800 0x3000 0x2000 0 -syscon misc MQ_PRIO_MAX 0x8000 0 0x40 0 0 0 -syscon misc MTERASE 13 0 12 9 9 0 -syscon misc MTLOAD 30 0 19 0 0 0 -syscon misc MTRETEN 9 0 0 8 8 0 -syscon misc NEW_TIME 3 4 3 0 0 0 -syscon misc NFDBITS 0x40 0x20 0x40 0x20 0x20 0 -syscon misc NGROUPS 0x010000 0x10 0x0400 0x10 0x10 0 -syscon misc NGROUPS_MAX 0x010000 0x10 0x03ff 0x10 0x10 0 -syscon misc NOEXPR 0x050001 53 53 49 49 0 -syscon misc NOFILE 0x0100 0x0100 0x40 0x40 0x40 0 -syscon misc NOSTR 0x050003 55 55 48 48 0 -syscon misc OLD_TIME 4 3 2 0 0 0 -syscon misc PM_STR 0x020027 6 6 5 5 0 -syscon misc RADIXCHAR 0x010000 50 50 44 44 0 -syscon misc RUN_LVL 1 1 0 0 0 0 -syscon misc STA_RONLY 0xff00 0 0xff00 0 0 0 -syscon misc SYMLOOP_MAX 0 0 0 0x20 0x20 0 -syscon misc THOUSEP 0x010001 51 51 45 45 0 -syscon misc TIMER_ABSTIME 1 0 1 1 1 0 -syscon misc TIME_UTC 1 0 1 0 0 0 -syscon misc TMP_MAX 0x03a2f8 0x1269ae40 0x1269ae40 0x7fffffff 0x7fffffff 0 -syscon misc TSS_DTOR_ITERATIONS 0 0 4 0 0 0 -syscon misc TTY_NAME_MAX 0x20 0 0 260 260 0 -syscon misc UIO_MAXIOV 0x0400 0 0 0x0400 0x0400 0 -syscon misc USER_PROCESS 7 7 4 0 0 0 -syscon misc YESEXPR 0x050000 52 52 47 47 0 -syscon misc YESSTR 0x050002 54 54 46 46 0 - # System Call Numbers. # -# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD XENIX +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology syscon nr __NR_exit 0x003c 0x2000001 0x0001 0x0001 0x001 0xfff syscon nr __NR_exit_group 0x00e7 0x2000001 0x0001 0x0001 0x001 0xfff syscon nr __NR_read 0x0000 0x2000003 0x0003 0x0003 0x003 0xfff @@ -3495,4 +2668,809 @@ syscon nr __NR_shared_region_map_and_slide_np 0xfff 0x20001b6 0xfff 0xfff syscon nr __NR_guarded_open_dprotected_np 0xfff 0x20001e4 0xfff 0xfff 0xfff 0xfff syscon nr __NR_stack_snapshot_with_config 0xfff 0x20001eb 0xfff 0xfff 0xfff 0xfff +# unilateral undocumented errnos +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon junkerr ECHRNG 44 0 0 0 0 0 +syscon junkerr EL2NSYNC 45 0 0 0 0 0 +syscon junkerr EL3HLT 46 0 0 0 0 0 +syscon junkerr EL3RST 47 0 0 0 0 0 +syscon junkerr ELNRNG 48 0 0 0 0 0 +syscon junkerr EUNATCH 49 0 0 0 0 0 +syscon junkerr ENOCSI 50 0 0 0 0 0 +syscon junkerr EL2HLT 51 0 0 0 0 0 +syscon junkerr EBADE 52 0 0 0 0 0 +syscon junkerr EBADR 53 0 0 0 0 0 +syscon junkerr EXFULL 54 0 0 0 0 0 +syscon junkerr ENOANO 55 0 0 0 0 0 +syscon junkerr EBADRQC 56 0 0 0 0 0 +syscon junkerr EBADSLT 57 0 0 0 0 0 +syscon junkerr ENOPKG 65 0 0 0 0 0 +syscon junkerr EADV 68 0 0 0 0 0 +syscon junkerr ESRMNT 69 0 0 0 0 0 +syscon junkerr ECOMM 70 0 0 0 0 0 +syscon junkerr EDOTDOT 73 0 0 0 0 0 +syscon junkerr ENOTUNIQ 76 0 0 0 0 0 +syscon junkerr EBADFD 77 9 0 0 0 0 +syscon junkerr EREMCHG 78 0 0 0 0 0 +syscon junkerr ELIBACC 79 0 0 0 0 0 +syscon junkerr ELIBBAD 80 0 0 0 0 0 +syscon junkerr ELIBSCN 81 0 0 0 0 0 +syscon junkerr ELIBMAX 82 0 0 0 0 0 +syscon junkerr ELIBEXEC 83 0 0 0 0 0 +syscon junkerr ESTRPIPE 86 0 0 0 0 0 +syscon junkerr EUCLEAN 117 0 0 0 0 0 +syscon junkerr ENOTNAM 118 0 0 0 0 0 +syscon junkerr ENAVAIL 119 0 0 0 0 0 +syscon junkerr EISNAM 120 0 0 0 0 0 +syscon junkerr EREMOTEIO 121 0 0 0 0 0 +syscon junkerr ENOKEY 126 0 0 0 0 0 +syscon junkerr EKEYEXPIRED 127 0 0 0 0 0 +syscon junkerr EKEYREVOKED 128 0 0 0 0 0 +syscon junkerr EKEYREJECTED 129 0 0 0 0 0 +syscon junkerr ERFKILL 132 0 0 0 0 0 +syscon junkerr EHWPOISON 133 0 0 0 0 0 + +# arpanet fork combating human-induced exhaustion of our ipv4 address space +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon ipv6 IPV6_PMTUDISC_DONT 0 0 0 0 0 0 # consensus +syscon ipv6 IPV6_RTHDR_LOOSE 0 0 0 0 0 0 # consensus +syscon ipv6 IPV6_RTHDR_TYPE_0 0 0 0 0 0 0 # consensus +syscon ipv6 IPV6_CHECKSUM 7 26 26 26 26 26 # bsd consensus +syscon ipv6 IPV6_JOIN_GROUP 20 12 12 12 12 12 # bsd consensus +syscon ipv6 IPV6_LEAVE_GROUP 21 13 13 13 13 13 # bsd consensus +syscon ipv6 IPV6_MULTICAST_HOPS 18 10 10 10 10 10 # bsd consensus +syscon ipv6 IPV6_MULTICAST_IF 17 9 9 9 9 9 # bsd consensus +syscon ipv6 IPV6_MULTICAST_LOOP 19 11 11 11 11 11 # bsd consensus +syscon ipv6 IPV6_UNICAST_HOPS 0x10 4 4 4 4 4 # bsd consensus +syscon ipv6 IPV6_V6ONLY 26 27 27 27 27 27 # bsd consensus +syscon ipv6 IPV6_RECVTCLASS 66 35 57 57 57 40 +syscon ipv6 IPV6_TCLASS 67 36 61 61 61 39 +syscon ipv6 IPV6_DONTFRAG 62 0 62 62 62 14 +syscon ipv6 IPV6_HOPLIMIT 52 0 47 47 47 21 +syscon ipv6 IPV6_HOPOPTS 54 0 49 49 49 1 +syscon ipv6 IPV6_PKTINFO 50 0 46 46 46 19 +syscon ipv6 IPV6_RECVRTHDR 56 0 38 38 38 38 +syscon ipv6 IPV6_RTHDR 57 0 51 51 51 0x20 +syscon ipv6 IPV6_DSTOPTS 59 0 50 50 50 0 +syscon ipv6 IPV6_IPSEC_POLICY 34 28 28 0 0 0 +syscon ipv6 IPV6_NEXTHOP 9 0 48 48 48 0 +syscon ipv6 IPV6_PATHMTU 61 0 44 44 44 0 +syscon ipv6 IPV6_RECVDSTOPTS 58 0 40 40 40 0 +syscon ipv6 IPV6_RECVHOPLIMIT 51 0 37 37 37 0 +syscon ipv6 IPV6_RECVHOPOPTS 53 0 39 39 39 0 +syscon ipv6 IPV6_RECVPATHMTU 60 0 43 43 43 0 +syscon ipv6 IPV6_RECVPKTINFO 49 0 36 36 36 0 +syscon ipv6 IPV6_RTHDRDSTOPTS 55 0 35 35 35 0 +syscon ipv6 IPV6_RTHDR_STRICT 1 1 1 0 0 0 +syscon ipv6 IPV6_ADD_MEMBERSHIP 20 0 0 0 0 12 # bsd consensus +syscon ipv6 IPV6_DROP_MEMBERSHIP 21 0 0 0 0 13 # bsd consensus +syscon ipv6 IPV6_HDRINCL 36 0 0 0 0 2 # bsd consensus +syscon ipv6 IPV6_MTU 24 0 0 0 0 72 # bsd consensus +syscon ipv6 IPV6_MTU_DISCOVER 23 0 0 0 0 71 # bsd consensus +syscon ipv6 IPV6_RECVERR 25 0 0 0 0 75 # bsd consensus +syscon ipv6 IPV6_2292DSTOPTS 4 23 0 0 0 0 +syscon ipv6 IPV6_2292HOPLIMIT 8 20 0 0 0 0 +syscon ipv6 IPV6_2292HOPOPTS 3 22 0 0 0 0 +syscon ipv6 IPV6_2292PKTINFO 2 19 0 0 0 0 +syscon ipv6 IPV6_2292PKTOPTIONS 6 25 0 0 0 0 +syscon ipv6 IPV6_2292RTHDR 5 24 0 0 0 0 +syscon ipv6 IPV6_AUTOFLOWLABEL 0 0 59 59 59 0 +syscon ipv6 IPV6_ADDRFORM 1 0 0 0 0 0 +syscon ipv6 IPV6_AUTHHDR 10 0 0 0 0 0 +syscon ipv6 IPV6_JOIN_ANYCAST 27 0 0 0 0 0 +syscon ipv6 IPV6_LEAVE_ANYCAST 28 0 0 0 0 0 +syscon ipv6 IPV6_PMTUDISC_DO 2 0 0 0 0 0 +syscon ipv6 IPV6_PMTUDISC_INTERFACE 4 0 0 0 0 0 +syscon ipv6 IPV6_PMTUDISC_OMIT 5 0 0 0 0 0 +syscon ipv6 IPV6_PMTUDISC_PROBE 3 0 0 0 0 0 +syscon ipv6 IPV6_PMTUDISC_WANT 1 0 0 0 0 0 +syscon ipv6 IPV6_ROUTER_ALERT 22 0 0 0 0 0 +syscon ipv6 IPV6_RXDSTOPTS 59 0 0 0 0 0 +syscon ipv6 IPV6_RXHOPOPTS 54 0 0 0 0 0 +syscon ipv6 IPV6_XFRM_POLICY 35 0 0 0 0 0 +syscon ipv6 IPV6_MINHOPCOUNT 0 0 0 65 65 0 +syscon ipv6 IPV6_ORIGDSTADDR 0 0 72 0 0 0 +syscon ipv6 IPV6_RECVORIGDSTADDR 0 0 72 0 0 0 +syscon ipv6 INET6_ADDRSTRLEN 46 46 46 46 46 65 # unix consensus +syscon icmp6 ICMP6_DST_UNREACH_NOROUTE 0 0 0 0 0 0 # consensus +syscon icmp6 ICMP6_PARAMPROB_HEADER 0 0 0 0 0 0 # consensus +syscon icmp6 ICMP6_TIME_EXCEED_TRANSIT 0 0 0 0 0 0 # consensus +syscon icmp6 ICMP6_DST_UNREACH_ADMIN 1 1 1 1 1 1 # consensus +syscon icmp6 ICMP6_PARAMPROB_NEXTHEADER 1 1 1 1 1 1 # consensus +syscon icmp6 ICMP6_TIME_EXCEED_REASSEMBLY 1 1 1 1 1 1 # consensus +syscon icmp6 ICMP6_DST_UNREACH 1 1 1 1 1 0 # unix consensus +syscon icmp6 ICMP6_FILTER 1 18 18 18 18 0 # bsd consensus +syscon icmp6 ICMP6_DST_UNREACH_BEYONDSCOPE 2 2 2 2 2 2 # consensus +syscon icmp6 ICMP6_PARAMPROB_OPTION 2 2 2 2 2 2 # consensus +syscon icmp6 ICMP6_PACKET_TOO_BIG 2 2 2 2 2 0 # unix consensus +syscon icmp6 ICMP6_DST_UNREACH_ADDR 3 3 3 3 3 3 # consensus +syscon icmp6 ICMP6_TIME_EXCEEDED 3 3 3 3 3 0 # unix consensus +syscon icmp6 ICMP6_DST_UNREACH_NOPORT 4 4 4 4 4 4 # consensus +syscon icmp6 ICMP6_PARAM_PROB 4 4 4 4 4 0 # unix consensus +syscon icmp6 ICMP6_RR_FLAGS_PREVDONE 8 8 8 8 8 0 # unix consensus +syscon icmp6 ICMP6_RR_FLAGS_SPECSITE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus +syscon icmp6 ICMP6_RR_PCOUSE_RAFLAGS_AUTO 0x10 0x40 0x40 0x40 0x40 0 # bsd consensus +syscon icmp6 ICMP6_RR_FLAGS_FORCEAPPLY 0x20 0x20 0x20 0x20 0x20 0 # unix consensus +syscon icmp6 ICMP6_RR_PCOUSE_RAFLAGS_ONLINK 0x20 0x80 0x80 0x80 0x80 0 # bsd consensus +syscon icmp6 ICMP6_RR_FLAGS_REQRESULT 0x40 0x40 0x40 0x40 0x40 0 # unix consensus +syscon icmp6 ICMP6_RR_PCOUSE_FLAGS_DECRPLTIME 0x40 0x40 0x40 0x40 0x40 0 # unix consensus +syscon icmp6 ICMP6_INFOMSG_MASK 0x80 0x80 0x80 0x80 0x80 0x80 # consensus +syscon icmp6 ICMP6_ECHO_REQUEST 0x80 0x80 0x80 0x80 0x80 0 # unix consensus +syscon icmp6 ICMP6_RR_FLAGS_TEST 0x80 0x80 0x80 0x80 0x80 0 # unix consensus +syscon icmp6 ICMP6_RR_PCOUSE_FLAGS_DECRVLTIME 0x80 0x80 0x80 0x80 0x80 0 # unix consensus +syscon icmp6 ICMP6_ECHO_REPLY 129 129 129 129 129 0 # unix consensus +syscon icmp6 ICMP6_ROUTER_RENUMBERING 138 138 138 138 138 0 # unix consensus +syscon icmp6 ICMP6_RR_RESULT_FLAGS_FORBIDDEN 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus +syscon icmp6 ICMP6_RR_RESULT_FLAGS_OOB 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus + +# java's computational model +# solves the sharing problem by defining everything as shared +# +# group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary +syscon misc PTHREAD_MUTEX_STALLED 0 0 0 0 0 0 # consensus +syscon misc PTHREAD_PRIO_NONE 0 0 0 0 0 0 # consensus +syscon misc PTHREAD_PRIO_INHERIT 0 1 1 1 1 0 # bsd consensus +syscon misc PTHREAD_PRIO_PROTECT 0 2 2 2 2 0 # bsd consensus +syscon misc PTHREAD_DESTRUCTOR_ITERATIONS 4 4 4 4 4 0 # unix consensus +syscon misc PTHREAD_PROCESS_SHARED 1 1 1 1 1 0 # unix consensus +syscon misc PTHREAD_CREATE_DETACHED 1 2 1 1 1 0 +syscon misc PTHREAD_KEYS_MAX 0x0400 0x0200 0x0100 0x0100 0x0100 0 +syscon misc PTHREAD_STACK_MIN 0x4000 0x2000 0x0800 0x1000 0x1000 0 +syscon misc PTHREAD_BARRIER_SERIAL_THREAD -1 0 -1 -1 -1 0 +syscon misc PTHREAD_CANCEL_ASYNCHRONOUS 1 0 2 2 2 0 +syscon misc PTHREAD_CANCEL_DISABLE 1 0 1 1 1 0 +syscon misc PTHREAD_INHERIT_SCHED 0 1 4 4 4 0 +syscon misc PTHREAD_SCOPE_SYSTEM 0 1 2 2 2 0 +syscon misc PTHREAD_EXPLICIT_SCHED 1 2 0 0 0 0 +syscon misc PTHREAD_MUTEX_DEFAULT 0 0 1 4 4 0 +syscon misc PTHREAD_MUTEX_ERRORCHECK 0 1 0 1 1 0 +syscon misc PTHREAD_MUTEX_RECURSIVE 0 2 0 2 2 0 +syscon misc PTHREAD_SCOPE_PROCESS 1 2 0 0 0 0 +syscon misc PTHREAD_CANCEL_DEFERRED 0 2 0 0 0 0 +syscon misc PTHREAD_CANCEL_ENABLE 0 1 0 0 0 0 +syscon misc PTHREAD_CREATE_JOINABLE 0 1 0 0 0 0 +syscon misc PTHREAD_MUTEX_NORMAL 0 0 0 3 3 0 +syscon misc PTHREAD_MUTEX_ROBUST 0 0 1 0 0 0 +syscon misc PTHREAD_PROCESS_PRIVATE 0 2 0 0 0 0 + +syscon fan FAN_CLASS_NOTIF 0 0 0 0 0 0 # consensus +syscon fan FAN_ACCESS 1 0 0 0 0 0 +syscon fan FAN_ACCESS_PERM 0x020000 0 0 0 0 0 +syscon fan FAN_ALLOW 1 0 0 0 0 0 +syscon fan FAN_ALL_CLASS_BITS 12 0 0 0 0 0 +syscon fan FAN_ALL_EVENTS 59 0 0 0 0 0 +syscon fan FAN_ALL_INIT_FLAGS 63 0 0 0 0 0 +syscon fan FAN_ALL_MARK_FLAGS 255 0 0 0 0 0 +syscon fan FAN_ALL_OUTGOING_EVENTS 0x03403b 0 0 0 0 0 +syscon fan FAN_ALL_PERM_EVENTS 0x030000 0 0 0 0 0 +syscon fan FAN_CLASS_CONTENT 4 0 0 0 0 0 +syscon fan FAN_CLASS_PRE_CONTENT 8 0 0 0 0 0 +syscon fan FAN_CLOEXEC 1 0 0 0 0 0 +syscon fan FAN_CLOSE 24 0 0 0 0 0 +syscon fan FAN_CLOSE_NOWRITE 0x10 0 0 0 0 0 +syscon fan FAN_CLOSE_WRITE 8 0 0 0 0 0 +syscon fan FAN_DENY 2 0 0 0 0 0 +syscon fan FAN_EVENT_METADATA_LEN 24 0 0 0 0 0 +syscon fan FAN_EVENT_ON_CHILD 0x08000000 0 0 0 0 0 +syscon fan FAN_MARK_ADD 1 0 0 0 0 0 +syscon fan FAN_MARK_DONT_FOLLOW 4 0 0 0 0 0 +syscon fan FAN_MARK_FLUSH 0x80 0 0 0 0 0 +syscon fan FAN_MARK_IGNORED_MASK 0x20 0 0 0 0 0 +syscon fan FAN_MARK_IGNORED_SURV_MODIFY 0x40 0 0 0 0 0 +syscon fan FAN_MARK_MOUNT 0x10 0 0 0 0 0 +syscon fan FAN_MARK_ONLYDIR 8 0 0 0 0 0 +syscon fan FAN_MARK_REMOVE 2 0 0 0 0 0 +syscon fan FAN_MODIFY 2 0 0 0 0 0 +syscon fan FAN_NOFD -1 0 0 0 0 0 +syscon fan FAN_NONBLOCK 2 0 0 0 0 0 +syscon fan FAN_ONDIR 0x40000000 0 0 0 0 0 +syscon fan FAN_OPEN 0x20 0 0 0 0 0 +syscon fan FAN_OPEN_PERM 0x010000 0 0 0 0 0 +syscon fan FAN_Q_OVERFLOW 0x4000 0 0 0 0 0 +syscon fan FAN_UNLIMITED_MARKS 0x20 0 0 0 0 0 +syscon fan FAN_UNLIMITED_QUEUE 0x10 0 0 0 0 0 + +syscon misc FIFOTYPE 54 54 54 54 54 0 # unix consensus +syscon misc GRPQUOTA 1 1 1 1 1 0 # unix consensus +syscon misc IF_NAMESIZE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus +syscon misc INTERMEDIATE_C_GOOD 10 0 0 0 0 0 +syscon misc INTERMEDIATE_GOOD 8 0 0 0 0 0 + +syscon misc IOV_MAX 0x0400 0x0400 0x0400 0x0400 0x0400 16 # unix consensus & MSG_MAXIOVLEN +syscon misc LINE_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus +syscon misc LINKED_CMD_COMPLETE 10 0 0 0 0 0 +syscon misc LINKED_FLG_CMD_COMPLETE 11 0 0 0 0 0 + +syscon misc LNKTYPE 49 49 49 49 49 0 # unix consensus +syscon misc MAXNAMLEN 255 255 255 255 255 0 # unix consensus +syscon misc MAXQUOTAS 2 2 2 2 2 0 # unix consensus +syscon misc MEDIUM_ERROR 3 0 0 0 0 0 +syscon misc MEDIUM_SCAN 56 0 0 0 0 0 + +syscon misc NBBY 8 8 8 8 8 0 # unix consensus +syscon misc NR_DQHASH 43 0 0 0 0 0 +syscon misc NR_DQUOTS 0x0100 0 0 0 0 0 + +syscon misc PERSISTENT_RESERVE_IN 94 0 0 0 0 0 +syscon misc PERSISTENT_RESERVE_OUT 95 0 0 0 0 0 + +syscon misc PRELIM 1 1 1 1 1 0 # unix consensus +syscon misc REGTYPE 48 48 48 48 48 0 # unix consensus +syscon misc RES_PRF_CLASS 4 4 4 4 4 0 # unix consensus +syscon misc RHF_GUARANTEE_START_INIT 0x80 0 0 0 0 0 +syscon misc RHF_NO_LIBRARY_REPLACEMENT 4 0 0 0 0 0 + +syscon misc RRQ 1 1 1 1 1 0 # unix consensus +syscon misc RTF_NOFORWARD 0x1000 0 0 0 0 0 +syscon misc RTF_NOPMTUDISC 0x4000 0 0 0 0 0 + +syscon misc SARMAG 8 8 8 8 8 0 # unix consensus +syscon misc SEGSIZE 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus +syscon misc SEND_DIAGNOSTIC 29 0 0 0 0 0 +syscon misc SEND_VOLUME_TAG 182 0 0 0 0 0 + +syscon misc SET_LIMITS 51 0 0 0 0 0 +syscon misc SET_WINDOW 36 0 0 0 0 0 + +syscon misc SFD_CLOEXEC 0x080000 0 0 0 0 0 +syscon misc SFD_NONBLOCK 0x0800 0 0 0 0 0 + +syscon misc SOMAXCONN 0x80 0x80 0x80 0x80 0x80 0x7fffffff # unix consensus +syscon misc SUBCMDMASK 255 255 255 255 255 0 # unix consensus +syscon misc SUBCMDSHIFT 8 8 8 8 8 0 # unix consensus +syscon misc SYMTYPE 50 50 50 50 50 0 # unix consensus +syscon misc TGEXEC 8 8 8 8 8 0 # unix consensus +syscon misc TGREAD 0x20 0x20 0x20 0x20 0x20 0 # unix consensus +syscon misc TGWRITE 0x10 0x10 0x10 0x10 0x10 0 # unix consensus +syscon misc TMAGLEN 6 6 6 6 6 0 # unix consensus +syscon misc TOEXEC 1 1 1 1 1 0 # unix consensus +syscon misc TOREAD 4 4 4 4 4 0 # unix consensus +syscon misc TOWRITE 2 2 2 2 2 0 # unix consensus +syscon misc TRANSIENT 4 4 4 4 4 0 # unix consensus +syscon misc TRY_AGAIN 2 2 2 2 2 0x2afa # unix consensus +syscon misc TSGID 0x0400 0x0400 0x0400 0x0400 0x0400 0 # unix consensus +syscon misc TSUID 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus +syscon misc TSVTX 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus +syscon misc TUEXEC 0x40 0x40 0x40 0x40 0x40 0 # unix consensus +syscon misc TUREAD 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus +syscon misc TUWRITE 0x80 0x80 0x80 0x80 0x80 0 # unix consensus +syscon misc TVERSLEN 2 2 2 2 2 0 # unix consensus +syscon misc WORD_BIT 0x20 0x20 0x20 0x20 0x20 0 # unix consensus +syscon misc WRQ 2 2 2 2 2 0 # unix consensus +syscon misc SIGEV_THREAD 2 3 2 0 2 0 +syscon misc SIGEV_SIGNAL 0 1 1 0 1 0 +syscon misc SIGEV_NONE 1 0 0 0 0 0 + +syscon misc BC_BASE_MAX 99 99 99 0x7fffffff 0x7fffffff 0 +syscon misc BC_DIM_MAX 0x0800 0x0800 0x0800 0xffff 0xffff 0 +syscon misc BC_SCALE_MAX 99 99 99 0x7fffffff 0x7fffffff 0 +syscon misc BC_STRING_MAX 0x03e8 0x03e8 0x03e8 0x7fffffff 0x7fffffff 0 + +syscon misc ABORTED_COMMAND 11 0 0 0 0 0 +syscon misc ACORE 0 8 8 8 8 0 # bsd consensus +syscon misc AFORK 0 1 1 1 1 0 # bsd consensus +syscon misc AIO_ALLDONE 2 1 3 0 0 0 +syscon misc AIO_NOTCANCELED 1 4 2 0 0 0 +syscon misc AIO_CANCELED 0 2 1 0 0 0 + +syscon misc ALLOW_MEDIUM_REMOVAL 30 0 0 0 0 0 +syscon misc ASU 0 2 2 2 2 0 # bsd consensus +syscon misc ATF_NETMASK 0x20 0 0 0 0 0 +syscon misc AXSIG 0 0x10 0x10 0x10 0x10 0 # bsd consensus +syscon misc B1000000 0x1008 0 0 0 0 0 +syscon misc B110 3 110 110 110 110 0 # bsd consensus +syscon misc B115200 0x1002 0x01c200 0x01c200 0x01c200 0x01c200 0 # bsd consensus +syscon misc B1152000 0x1009 0 0 0 0 0 +syscon misc B1200 9 0x04b0 0x04b0 0x04b0 0x04b0 0 # bsd consensus +syscon misc B134 4 134 134 134 134 0 # bsd consensus +syscon misc B150 5 150 150 150 150 0 # bsd consensus +syscon misc B1500000 0x100a 0 0 0 0 0 +syscon misc B1800 10 0x0708 0x0708 0x0708 0x0708 0 # bsd consensus +syscon misc B19200 14 0x4b00 0x4b00 0x4b00 0x4b00 0 # bsd consensus +syscon misc B200 6 200 200 200 200 0 # bsd consensus +syscon misc B2000000 0x100b 0 0 0 0 0 +syscon misc B230400 0x1003 0x038400 0x038400 0x038400 0x038400 0 # bsd consensus +syscon misc B2400 11 0x0960 0x0960 0x0960 0x0960 0 # bsd consensus +syscon misc B2500000 0x100c 0 0 0 0 0 +syscon misc B300 7 300 300 300 300 0 # bsd consensus +syscon misc B3000000 0x100d 0 0 0 0 0 +syscon misc B3500000 0x100e 0 0 0 0 0 +syscon misc B38400 15 0x9600 0x9600 0x9600 0x9600 0 # bsd consensus +syscon misc B4000000 0x100f 0 0 0 0 0 +syscon misc B4800 12 0x12c0 0x12c0 0x12c0 0x12c0 0 # bsd consensus +syscon misc B50 1 50 50 50 50 0 # bsd consensus +syscon misc B500000 0x1005 0 0 0 0 0 +syscon misc B57600 0x1001 0xe100 0xe100 0xe100 0xe100 0 # bsd consensus +syscon misc B576000 0x1006 0 0 0 0 0 +syscon misc B600 8 600 600 600 600 0 # bsd consensus +syscon misc B75 2 75 75 75 75 0 # bsd consensus +syscon misc B9600 13 0x2580 0x2580 0x2580 0x2580 0 # bsd consensus +syscon misc BITSPERBYTE 8 0 0 0 0 0 +syscon misc BLANK_CHECK 8 0 0 0 0 0 +syscon misc CHANGE_DEFINITION 0x40 0 0 0 0 0 +syscon misc CHARBITS 8 0 0 0 0 0 +syscon misc CHECK_CONDITION 1 0 0 0 0 0 +syscon misc CONDITION_GOOD 2 0 0 0 0 0 +syscon misc CREAD 0x80 0x0800 0x0800 0x0800 0x0800 0 # bsd consensus +syscon misc CSTOPB 0x40 0x0400 0x0400 0x0400 0x0400 0 # bsd consensus +syscon misc DATA_PROTECT 7 0 0 0 0 0 +syscon misc DELAYTIMER_MAX 0x7fffffff 0 0 0 0 0 +syscon misc DMAXEXP 0x0400 0 0 0 0 0 +syscon misc DMINEXP -1021 0 0 0 0 0 +syscon misc DOUBLEBITS 0x40 0 0 0 0 0 +syscon misc ERA_D_FMT 0x02002e 46 46 0 0 0 +syscon misc ERA_D_T_FMT 0x020030 47 47 0 0 0 +syscon misc ERA_T_FMT 0x020031 48 48 0 0 0 + +syscon misc FANOTIFY_METADATA_VERSION 3 0 0 0 0 0 +syscon misc FAPPEND 0x0400 8 8 8 8 0 # bsd consensus +syscon misc FASYNC 0x2000 0x40 0x40 0x40 0x40 0 # bsd consensus +syscon misc FFSYNC 0x101000 0x80 0x80 0x80 0x80 0 # bsd consensus +syscon misc FILENAME_MAX 0x1000 0x0400 0x0400 0x0400 0x0400 0 # bsd consensus +syscon misc FIOGETOWN 0x8903 0x4004667b 0x4004667b 0x4004667b 0x4004667b 0 # bsd consensus +syscon misc FIOSETOWN 0x8901 0x8004667c 0x8004667c 0x8004667c 0x8004667c 0 # bsd consensus +syscon misc FMAXEXP 0x80 0 0 0 0 0 +syscon misc FMINEXP -125 0 0 0 0 0 +syscon misc FNDELAY 0x0800 4 4 4 4 0 # bsd consensus +syscon misc FNONBLOCK 0x0800 4 4 4 4 0 # bsd consensus +syscon misc FOPEN_MAX 0x10 20 20 20 20 0 # bsd consensus +syscon misc FORMAT_UNIT 4 0 0 0 0 0 +syscon misc HARDWARE_ERROR 4 0 0 0 0 0 +syscon misc HEAD_OF_QUEUE_TAG 33 0 0 0 0 0 +syscon misc HUPCL 0x0400 0x4000 0x4000 0x4000 0x4000 0 # bsd consensus +syscon misc IGMP_MEMBERSHIP_QUERY 17 0 0 0 0 0 +syscon misc ILLEGAL_REQUEST 5 0 0 0 0 0 +syscon misc INITIATE_RECOVERY 15 0 0 0 0 0 +syscon misc INITIATOR_ERROR 5 0 0 0 0 0 +syscon misc INQUIRY 18 0 0 0 0 0 +syscon misc MAXHOSTNAMELEN 0x40 0x0100 0x0100 0x0100 0x0100 0 # bsd consensus +syscon misc MAXPATHLEN 255 255 255 255 255 255 # forced consensus +syscon misc MAXSYMLINKS 20 0x20 0x20 0x20 0x20 0 # bsd consensus +syscon misc MESSAGE_REJECT 7 0 0 0 0 0 +syscon misc MISCOMPARE 14 0 0 0 0 0 +syscon misc MOVE_MEDIUM 165 0 0 0 0 0 +syscon misc MTCOMPRESSION 0x20 0 0 0 0 0 +syscon misc MTFSFM 11 0 0 0 0 0 +syscon misc MTLOCK 28 0 0 0 0 0 +syscon misc MTMKPART 34 0 0 0 0 0 +syscon misc MTRAS1 14 0 0 0 0 0 +syscon misc MTRAS3 0x10 0 0 0 0 0 +syscon misc MTSETBLK 20 0 0 0 0 0 +syscon misc MTSETDENSITY 21 0 0 0 0 0 +syscon misc MTSETDRVBUFFER 24 0 0 0 0 0 +syscon misc MTSETPART 33 0 0 0 0 0 +syscon misc MTUNLOAD 31 0 0 0 0 0 +syscon misc MTUNLOCK 29 0 0 0 0 0 +syscon misc NCARGS 0x020000 0x040000 0x040000 0x040000 0x040000 0 # bsd consensus +syscon misc NGREG 23 0 0 0 0 0 +syscon misc NOGROUP -1 0xffff 0xffff 0xffff 0xffff 0 # bsd consensus +syscon misc ORDERED_QUEUE_TAG 34 0 0 0 0 0 +syscon misc ORIG_RAX 15 0 0 0 0 0 +syscon misc PIPE_BUF 0x1000 0x0200 0x0200 0x0200 0x0200 0 # bsd consensus +syscon misc PRE_FETCH 52 0 0 0 0 0 +syscon misc QUEUE_FULL 20 0 0 0 0 0 +syscon misc REASSIGN_BLOCKS 7 0 0 0 0 0 +syscon misc RECEIVE_DIAGNOSTIC 28 0 0 0 0 0 +syscon misc RECOVERED_ERROR 1 0 0 0 0 0 +syscon misc RECOVER_BUFFERED_DATA 20 0 0 0 0 0 +syscon misc RELEASE_RECOVERY 0x10 0 0 0 0 0 +syscon misc REQUEST_SENSE 3 0 0 0 0 0 +syscon misc RESERVATION_CONFLICT 12 0 0 0 0 0 +syscon misc RESERVE 22 0 0 0 0 0 +syscon misc RESERVE_10 86 0 0 0 0 0 +syscon misc RESTORE_POINTERS 3 0 0 0 0 0 +syscon misc REZERO_UNIT 1 0 0 0 0 0 +syscon misc RE_DUP_MAX 0x7fff 255 255 255 255 0 # bsd consensus +syscon misc RTCF_DOREDIRECT 0x01000000 0 0 0 0 0 +syscon misc SAVE_POINTERS 2 0 0 0 0 0 +syscon misc SEM_VALUE_MAX 0x7fffffff 0x7fff 0x7fffffff 0xffffffff 0xffffffff 0 +syscon misc SEM_INFO 19 0 11 0 0 0 +syscon misc SEM_STAT 18 0 10 0 0 0 + +syscon misc SHMLBA 0 0x1000 0x1000 0x1000 0x1000 0 # bsd consensus +syscon misc SIMPLE_QUEUE_TAG 0x20 0 0 0 0 0 +syscon misc SPACE 17 0 0 0 0 0 +syscon misc START_STOP 27 0 0 0 0 0 +syscon misc STATUS_MASK 62 0 0 0 0 0 +syscon misc SWAP_FLAG_DISCARD 0x010000 0 0 0 0 0 +syscon misc SYNCHRONIZE_CACHE 53 0 0 0 0 0 +syscon misc UMOUNT_NOFOLLOW 8 0 0 0 0 0 +syscon misc UNIT_ATTENTION 6 0 0 0 0 0 +syscon misc UPDATE_BLOCK 61 0 0 0 0 0 +syscon misc UT_HOSTSIZE 0x0100 0x10 0 0x0100 0x0100 0 +syscon misc UT_LINESIZE 0x20 8 0 8 8 0 +syscon misc UT_NAMESIZE 0x20 8 0 0x20 0x20 0 + +syscon misc WEOF 0xffffffff -1 -1 -1 -1 -1 # bsd consensus (win fake) +syscon misc _LINUX_QUOTA_VERSION 2 0 0 0 0 0 +syscon misc _SEM_SEMUN_UNDEFINED 1 0 0 0 0 0 +syscon misc D_FMT 0x020029 2 2 1 1 0 +syscon misc D_T_FMT 0x020028 1 1 0 0 0 + +syscon misc LOGIN_PROCESS 6 6 6 0 0 0 +syscon misc LOGIN_NAME_MAX 0x0100 0 0 0x20 0x20 0 + +syscon misc T_FMT 0x02002a 3 3 2 2 0 +syscon misc T_FMT_AMPM 0x02002b 4 4 3 3 0 + +syscon misc UL_GETFSIZE 1 1 1 0 0 0 +syscon misc UL_SETFSIZE 2 2 2 0 0 0 + +syscon misc XATTR_CREATE 1 2 0 0 0 0 +syscon misc XATTR_REPLACE 2 4 0 0 0 0 + +syscon misc ACCOUNTING 9 9 0 0 0 0 +syscon misc AHZ 100 0x40 0 0x40 0x40 0 +syscon misc ALT_DIGITS 0x02002f 49 49 0 0 0 +syscon misc AM_STR 0x020026 5 5 4 4 0 +syscon misc B460800 0x1004 0 0x070800 0 0 0 +syscon misc B921600 0x1007 0 0x0e1000 0 0 0 +syscon misc BOOT_TIME 2 2 1 0 0 0 +syscon misc CHARCLASS_NAME_MAX 0x0800 14 14 0 0 0 +syscon misc CLOCKS_PER_SEC 1000000 1000000 0x80 100 100 10000000 +syscon misc CODESET 14 0 0 51 51 0 +syscon misc COLL_WEIGHTS_MAX 255 2 10 2 2 0 +syscon misc CPU_SETSIZE 0x0400 0 0x0100 0 0 0 +syscon misc CRNCYSTR 0x04000f 56 56 50 50 0 +syscon misc CRTSCTS 0x80000000 0x030000 0x030000 0x010000 0x010000 0 +syscon misc CSTATUS 0 20 20 255 255 0 +syscon misc DEAD_PROCESS 8 8 7 0 0 0 +syscon misc FNM_NOSYS -1 -1 -1 2 2 0 +syscon misc INIT_PROCESS 5 5 5 0 0 0 +syscon misc MINSIGSTKSZ 0x0800 0x8000 0x0800 0x3000 0x2000 0 +syscon misc MQ_PRIO_MAX 0x8000 0 0x40 0 0 0 +syscon misc MTERASE 13 0 12 9 9 0 +syscon misc MTLOAD 30 0 19 0 0 0 +syscon misc MTRETEN 9 0 0 8 8 0 +syscon misc NEW_TIME 3 4 3 0 0 0 +syscon misc NFDBITS 0x40 0x20 0x40 0x20 0x20 0 +syscon misc NGROUPS 0x010000 0x10 0x0400 0x10 0x10 0 +syscon misc NGROUPS_MAX 0x010000 0x10 0x03ff 0x10 0x10 0 +syscon misc NOEXPR 0x050001 53 53 49 49 0 +syscon misc NOFILE 0x0100 0x0100 0x40 0x40 0x40 0 +syscon misc NOSTR 0x050003 55 55 48 48 0 +syscon misc OLD_TIME 4 3 2 0 0 0 +syscon misc PM_STR 0x020027 6 6 5 5 0 +syscon misc RADIXCHAR 0x010000 50 50 44 44 0 +syscon misc RUN_LVL 1 1 0 0 0 0 +syscon misc STA_RONLY 0xff00 0 0xff00 0 0 0 +syscon misc SYMLOOP_MAX 0 0 0 0x20 0x20 0 +syscon misc THOUSEP 0x010001 51 51 45 45 0 +syscon misc TIMER_ABSTIME 1 0 1 1 1 0 +syscon misc TIME_UTC 1 0 1 0 0 0 +syscon misc TMP_MAX 0x03a2f8 0x1269ae40 0x1269ae40 0x7fffffff 0x7fffffff 0 +syscon misc TSS_DTOR_ITERATIONS 0 0 4 0 0 0 +syscon misc TTY_NAME_MAX 0x20 0 0 260 260 0 +syscon misc UIO_MAXIOV 0x0400 0 0 0x0400 0x0400 0 +syscon misc USER_PROCESS 7 7 4 0 0 0 +syscon misc YESEXPR 0x050000 52 52 47 47 0 +syscon misc YESSTR 0x050002 54 54 46 46 0 + +syscon in IN_LOOPBACKNET 127 127 127 127 127 0 # unix consensus +syscon in IN_ACCESS 1 0 0 0 0 0 +syscon in IN_ALL_EVENTS 0x0fff 0 0 0 0 0 +syscon in IN_ATTRIB 4 0 0 0 0 0 +syscon in IN_CLOEXEC 0x080000 0 0 0 0 0 +syscon in IN_CLOSE 24 0 0 0 0 0 +syscon in IN_CLOSE_NOWRITE 0x10 0 0 0 0 0 +syscon in IN_CLOSE_WRITE 8 0 0 0 0 0 +syscon in IN_CREATE 0x0100 0 0 0 0 0 +syscon in IN_DELETE 0x0200 0 0 0 0 0 +syscon in IN_DELETE_SELF 0x0400 0 0 0 0 0 +syscon in IN_DONT_FOLLOW 0x02000000 0 0 0 0 0 +syscon in IN_EXCL_UNLINK 0x04000000 0 0 0 0 0 +syscon in IN_IGNORED 0x8000 0 0 0 0 0 +syscon in IN_ISDIR 0x40000000 0 0 0 0 0 +syscon in IN_MASK_ADD 0x20000000 0 0 0 0 0 +syscon in IN_MODIFY 2 0 0 0 0 0 +syscon in IN_MOVE 192 0 0 0 0 0 +syscon in IN_MOVED_FROM 0x40 0 0 0 0 0 +syscon in IN_MOVED_TO 0x80 0 0 0 0 0 +syscon in IN_MOVE_SELF 0x0800 0 0 0 0 0 +syscon in IN_NONBLOCK 0x0800 0 0 0 0 0 +syscon in IN_ONESHOT 0x80000000 0 0 0 0 0 +syscon in IN_ONLYDIR 0x01000000 0 0 0 0 0 +syscon in IN_OPEN 0x20 0 0 0 0 0 +syscon in IN_Q_OVERFLOW 0x4000 0 0 0 0 0 +syscon in IN_UNMOUNT 0x2000 0 0 0 0 0 + +syscon posix _POSIX_ARG_MAX 0x1000 0x1000 0x1000 0x1000 0x1000 0 # unix consensus +syscon posix _POSIX_CHILD_MAX 25 25 25 25 25 0 # unix consensus +syscon posix _POSIX_HOST_NAME_MAX 255 255 255 255 255 0 # unix consensus +syscon posix _POSIX_LINK_MAX 8 8 8 8 8 0 # unix consensus +syscon posix _POSIX_LOGIN_NAME_MAX 9 9 9 9 9 0 # unix consensus +syscon posix _POSIX_MAX_CANON 255 255 255 255 255 0 # unix consensus +syscon posix _POSIX_MAX_INPUT 255 255 255 255 255 0 # unix consensus +syscon posix _POSIX_NAME_MAX 14 14 14 14 14 14 # forced consensus +syscon posix _POSIX_NGROUPS_MAX 8 8 8 8 8 0 # unix consensus +syscon posix _POSIX_OPEN_MAX 20 20 20 20 20 20 # forced consensus +syscon posix _POSIX_PATH_MAX 255 255 255 255 255 255 # forced consensus +syscon posix _POSIX_PIPE_BUF 0x0200 0x0200 0x0200 0x0200 0x0200 0 # unix consensus +syscon posix _POSIX_RE_DUP_MAX 255 255 255 255 255 0 # unix consensus +syscon posix _POSIX_SEM_NSEMS_MAX 0x0100 0x0100 0x0100 0x0100 0x0100 0 # unix consensus +syscon posix _POSIX_SEM_VALUE_MAX 0x7fff 0x7fff 0x7fff 0x7fff 0x7fff 0 # unix consensus +syscon posix _POSIX_SSIZE_MAX 0x7fff 0x7fff 0x7fff 0x7fff 0x7fff 0 # unix consensus +syscon posix _POSIX_STREAM_MAX 8 8 8 8 8 0 # unix consensus +syscon posix _POSIX_SYMLINK_MAX 255 255 255 255 255 0 # unix consensus +syscon posix _POSIX_SYMLOOP_MAX 8 8 8 8 8 0 # unix consensus +syscon posix _POSIX_THREAD_DESTRUCTOR_ITERATIONS 4 4 4 4 4 0 # unix consensus +syscon posix _POSIX_THREAD_KEYS_MAX 0x80 0x80 0x80 0x80 0x80 0 # unix consensus +syscon posix _POSIX_TTY_NAME_MAX 9 9 9 9 9 0 # unix consensus +syscon posix _POSIX_TZNAME_MAX 6 6 6 6 6 0 # unix consensus +syscon posix _POSIX_CLOCK_SELECTION 0x031069 -1 -1 -1 -1 0 # bsd consensus +syscon posix _POSIX_FSYNC 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_MAPPED_FILES 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_MEMORY_PROTECTION 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_READER_WRITER_LOCKS 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_THREADS 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_THREAD_ATTR_STACKADDR 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_THREAD_ATTR_STACKSIZE 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon posix _POSIX_ADVISORY_INFO 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_ASYNCHRONOUS_IO 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_BARRIERS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_JOB_CONTROL 1 0x030db0 1 1 1 0 +syscon posix _POSIX_MEMLOCK 0x031069 -1 -1 0x030db0 0x030db0 0 +syscon posix _POSIX_MEMLOCK_RANGE 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_MESSAGE_PASSING 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_NO_TRUNC 1 0x030db0 1 1 1 0 +syscon posix _POSIX_RAW_SOCKETS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_REALTIME_SIGNALS 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_REGEXP 1 0x030db0 1 1 1 0 +syscon posix _POSIX_SEMAPHORES 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_SHARED_MEMORY_OBJECTS 0x031069 -1 0x030db0 0x031069 0x031069 0 +syscon posix _POSIX_SHELL 1 0x030db0 1 1 1 0 +syscon posix _POSIX_SPAWN 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_SPIN_LOCKS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_THREAD_PRIORITY_SCHEDULING 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_THREAD_PROCESS_SHARED 0x031069 0x030db0 0x030db0 -1 -1 0 +syscon posix _POSIX_THREAD_SAFE_FUNCTIONS 0x031069 0x030db0 -1 0x030db0 0x030db0 0 +syscon posix _POSIX_THREAD_THREADS_MAX 0x40 0x40 0x40 4 4 0 +syscon posix _POSIX_TIMEOUTS 0x031069 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_TIMERS 0x031069 -1 0x030db0 -1 -1 0 +syscon posix _POSIX_VERSION 0x031069 0x030db0 0x030db0 0x031069 0x031069 0 +syscon posix _POSIX_VDISABLE 0 255 255 255 255 0 # bsd consensus +syscon posix _POSIX_AIO_LISTIO_MAX 2 2 2 0 0 0 +syscon posix _POSIX_AIO_MAX 1 1 1 0 0 0 +syscon posix _POSIX_CHOWN_RESTRICTED 0 0x030db0 1 1 1 0 +syscon posix _POSIX_CLOCKRES_MIN 0x01312d00 0 0x01312d00 0x01312d00 0x01312d00 0 +syscon posix _POSIX_CPUTIME 0 -1 0x030db0 0x031069 0x031069 0 +syscon posix _POSIX_DELAYTIMER_MAX 0x20 0x20 0x20 0 0 0 +syscon posix _POSIX_MONOTONIC_CLOCK 0 -1 0x030db0 0x030db0 0x030db0 0 +syscon posix _POSIX_MQ_OPEN_MAX 8 8 8 0 0 0 +syscon posix _POSIX_MQ_PRIO_MAX 0x20 0x20 0x20 0 0 0 +syscon posix _POSIX_RTSIG_MAX 8 8 8 0 0 0 +syscon posix _POSIX_SAVED_IDS 1 0x030db0 0 1 1 0 +syscon posix _POSIX_SIGQUEUE_MAX 0x20 0x20 0x20 0 0 0 +syscon posix _POSIX_THREAD_CPUTIME 0 -1 0x030db0 0x031069 0x031069 0 +syscon posix _POSIX_TIMER_MAX 0x20 0x20 0x20 0 0 0 +syscon posix _POSIX_IPV6 0x031069 0x030db0 0 0 0 0 +syscon posix _POSIX_SS_REPL_MAX 0 4 4 0 0 0 +syscon posix _POSIX_TRACE_EVENT_NAME_MAX 0 30 30 0 0 0 +syscon posix _POSIX_TRACE_NAME_MAX 0 8 8 0 0 0 +syscon posix _POSIX_TRACE_SYS_MAX 0 8 8 0 0 0 +syscon posix _POSIX_TRACE_USER_EVENT_MAX 0 0x20 0x20 0 0 0 +syscon posix _POSIX_V6_LP64_OFF64 1 1 0 0 0 0 +syscon posix _POSIX_V7_LP64_OFF64 1 1 0 0 0 0 + +syscon misc TYPE_DISK 0 0 0 0 0 0 # consensus +syscon misc TYPE_A 1 1 1 1 1 0 # unix consensus +syscon misc TYPE_E 2 2 2 2 2 0 # unix consensus +syscon misc TYPE_I 3 3 3 3 3 0 # unix consensus +syscon misc TYPE_L 4 4 4 4 4 0 # unix consensus +syscon misc TYPE_ENCLOSURE 13 0 0 0 0 0 +syscon misc TYPE_MEDIUM_CHANGER 8 0 0 0 0 0 +syscon misc TYPE_MOD 7 0 0 0 0 0 +syscon misc TYPE_NO_LUN 127 0 0 0 0 0 +syscon misc TYPE_PROCESSOR 3 0 0 0 0 0 +syscon misc TYPE_ROM 5 0 0 0 0 0 +syscon misc TYPE_SCANNER 6 0 0 0 0 0 +syscon misc TYPE_TAPE 1 0 0 0 0 0 +syscon misc TYPE_WORM 4 0 0 0 0 0 + +syscon misc _POSIX2_BC_BASE_MAX 99 99 99 99 99 0 # unix consensus +syscon misc _POSIX2_BC_DIM_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus +syscon misc _POSIX2_BC_SCALE_MAX 99 99 99 99 99 0 # unix consensus +syscon misc _POSIX2_BC_STRING_MAX 0x03e8 0x03e8 0x03e8 0x03e8 0x03e8 0 # unix consensus +syscon misc _POSIX2_CHARCLASS_NAME_MAX 14 14 14 14 14 0 # unix consensus +syscon misc _POSIX2_COLL_WEIGHTS_MAX 2 2 2 2 2 0 # unix consensus +syscon misc _POSIX2_EXPR_NEST_MAX 0x20 0x20 0x20 0x20 0x20 0 # unix consensus +syscon misc _POSIX2_LINE_MAX 0x0800 0x0800 0x0800 0x0800 0x0800 0 # unix consensus +syscon misc _POSIX2_RE_DUP_MAX 255 255 255 255 255 0 # unix consensus +syscon misc _POSIX2_C_BIND 0x031069 0x030db0 0x030db0 0x030db0 0x030db0 0 # bsd consensus +syscon misc _POSIX2_VERSION 0x031069 0x030db0 0x030a2c 0x031069 0x031069 0 + +syscon nd ND_RA_FLAG_MANAGED 0x80 0x80 0x80 0x80 0x80 0x80 # consensus +syscon nd ND_RA_FLAG_OTHER 0x40 0x40 0x40 0x40 0x40 0x40 # consensus +syscon nd ND_NA_FLAG_OVERRIDE 0x20 0x20 0x20 0x20 0x20 0x20000000 # unix consensus +syscon nd ND_NA_FLAG_ROUTER 0x80 0x80 0x80 0x80 0x80 0x80000000 # unix consensus +syscon nd ND_NA_FLAG_SOLICITED 0x40 0x40 0x40 0x40 0x40 0x40000000 # unix consensus +syscon nd ND_NEIGHBOR_ADVERT 136 136 136 136 136 0 # unix consensus +syscon nd ND_NEIGHBOR_SOLICIT 135 135 135 135 135 0 # unix consensus +syscon nd ND_REDIRECT 137 137 137 137 137 0 # unix consensus +syscon nd ND_ROUTER_ADVERT 134 134 134 134 134 0 # unix consensus +syscon nd ND_ROUTER_SOLICIT 133 133 133 133 133 0 # unix consensus +syscon nd ND_RA_FLAG_HOME_AGENT 0x20 0 0 0 0 0x20 # bsd consensus + +syscon misc N_TTY 0 0 0 0 0 0 # consensus +syscon misc N_6PACK 7 0 0 0 0 0 +syscon misc N_AX25 5 0 0 0 0 0 +syscon misc N_HCI 15 0 0 0 0 0 +syscon misc N_HDLC 13 0 0 0 0 0 +syscon misc N_IRDA 11 0 0 0 0 0 +syscon misc N_MASC 8 0 0 0 0 0 +syscon misc N_MOUSE 2 0 0 0 0 0 +syscon misc N_PPP 3 0 0 0 0 0 +syscon misc N_PROFIBUS_FDL 10 0 0 0 0 0 +syscon misc N_R3964 9 0 0 0 0 0 +syscon misc N_SLIP 1 0 0 0 0 0 +syscon misc N_SMSBLOCK 12 0 0 0 0 0 +syscon misc N_STRIP 4 0 0 0 0 0 +syscon misc N_SYNC_PPP 14 0 0 0 0 0 +syscon misc N_X25 6 0 0 0 0 0 + +syscon misc BLKTYPE 52 52 52 52 52 0 # unix consensus +syscon misc BLKBSZGET 0x80081270 0 0 0 0 0 +syscon misc BLKBSZSET 0x40081271 0 0 0 0 0 +syscon misc BLKFLSBUF 0x1261 0 0 0 0 0 +syscon misc BLKFRAGET 0x1265 0 0 0 0 0 +syscon misc BLKFRASET 0x1264 0 0 0 0 0 +syscon misc BLKGETSIZE 0x1260 0 0 0 0 0 +syscon misc BLKGETSIZE64 0x80081272 0 0 0 0 0 +syscon misc BLKRAGET 0x1263 0 0 0 0 0 +syscon misc BLKRASET 0x1262 0 0 0 0 0 +syscon misc BLKROGET 0x125e 0 0 0 0 0 +syscon misc BLKROSET 0x125d 0 0 0 0 0 +syscon misc BLKRRPART 0x125f 0 0 0 0 0 +syscon misc BLKSECTGET 0x1267 0 0 0 0 0 +syscon misc BLKSECTSET 0x1266 0 0 0 0 0 +syscon misc BLKSSZGET 0x1268 0 0 0 0 0 + +syscon misc ETH_P_CUST 0x6006 0 0 0 0 0 +syscon misc ETH_P_DDCMP 6 0 0 0 0 0 +syscon misc ETH_P_DEC 0x6000 0 0 0 0 0 +syscon misc ETH_P_DIAG 0x6005 0 0 0 0 0 +syscon misc ETH_P_DNA_DL 0x6001 0 0 0 0 0 +syscon misc ETH_P_DNA_RC 0x6002 0 0 0 0 0 +syscon misc ETH_P_DNA_RT 0x6003 0 0 0 0 0 +syscon misc ETH_P_IEEE802154 246 0 0 0 0 0 +syscon misc ETH_P_LAT 0x6004 0 0 0 0 0 +syscon misc ETH_P_LOCALTALK 9 0 0 0 0 0 +syscon misc ETH_P_PPP_MP 8 0 0 0 0 0 +syscon misc ETH_P_RARP 0x8035 0 0 0 0 0 +syscon misc ETH_P_SCA 0x6007 0 0 0 0 0 +syscon misc ETH_P_WAN_PPP 7 0 0 0 0 0 + +syscon scsi SCSI_IOCTL_BENCHMARK_COMMAND 3 0 0 0 0 0 +syscon scsi SCSI_IOCTL_DOORLOCK 0x5380 0 0 0 0 0 +syscon scsi SCSI_IOCTL_DOORUNLOCK 0x5381 0 0 0 0 0 +syscon scsi SCSI_IOCTL_GET_BUS_NUMBER 0x5386 0 0 0 0 0 +syscon scsi SCSI_IOCTL_GET_IDLUN 0x5382 0 0 0 0 0 +syscon scsi SCSI_IOCTL_PROBE_HOST 0x5385 0 0 0 0 0 +syscon scsi SCSI_IOCTL_SEND_COMMAND 1 0 0 0 0 0 +syscon scsi SCSI_IOCTL_START_UNIT 5 0 0 0 0 0 +syscon scsi SCSI_IOCTL_STOP_UNIT 6 0 0 0 0 0 +syscon scsi SCSI_IOCTL_SYNC 4 0 0 0 0 0 +syscon scsi SCSI_IOCTL_TAGGED_DISABLE 0x5384 0 0 0 0 0 +syscon scsi SCSI_IOCTL_TAGGED_ENABLE 0x5383 0 0 0 0 0 +syscon scsi SCSI_IOCTL_TEST_UNIT_READY 2 0 0 0 0 0 +syscon scsi BUS_DEVICE_RESET 12 0 0 0 0 0 # SIGBUS; +syscon scsi READ_10 40 0 0 0 0 0 +syscon scsi READ_12 168 0 0 0 0 0 +syscon scsi READ_6 8 0 0 0 0 0 +syscon scsi READ_BLOCK_LIMITS 5 0 0 0 0 0 +syscon scsi READ_BUFFER 60 0 0 0 0 0 +syscon scsi READ_CAPACITY 37 0 0 0 0 0 +syscon scsi READ_DEFECT_DATA 55 0 0 0 0 0 +syscon scsi READ_ELEMENT_STATUS 184 0 0 0 0 0 +syscon scsi READ_LONG 62 0 0 0 0 0 +syscon scsi READ_POSITION 52 0 0 0 0 0 +syscon scsi READ_REVERSE 15 0 0 0 0 0 +syscon scsi READ_TOC 67 0 0 0 0 0 +syscon scsi WRITE_10 42 0 0 0 0 0 +syscon scsi WRITE_12 170 0 0 0 0 0 +syscon scsi WRITE_6 10 0 0 0 0 0 +syscon scsi WRITE_BUFFER 59 0 0 0 0 0 +syscon scsi WRITE_FILEMARKS 0x10 0 0 0 0 0 +syscon scsi WRITE_LONG 63 0 0 0 0 0 +syscon scsi WRITE_LONG_2 234 0 0 0 0 0 +syscon scsi WRITE_SAME 65 0 0 0 0 0 +syscon scsi WRITE_VERIFY 46 0 0 0 0 0 +syscon scsi WRITE_VERIFY_12 174 0 0 0 0 0 + +syscon log LOG_EMERG 0 0 0 0 0 0 # consensus +syscon log LOG_KERN 0 0 0 0 0 0 # consensus +syscon log LOG_ALERT 1 1 1 1 1 1 # unix consensus +syscon log LOG_PID 1 1 1 1 1 1 # unix consensus +syscon log LOG_CONS 2 2 2 2 2 2 # unix consensus +syscon log LOG_CRIT 2 2 2 2 2 2 # unix consensus +syscon log LOG_ERR 3 3 3 3 3 3 # unix consensus +syscon log LOG_ODELAY 4 4 4 4 4 4 # unix consensus +syscon log LOG_WARNING 4 4 4 4 4 4 # unix consensus +syscon log LOG_NOTICE 5 5 5 5 5 5 # unix consensus +syscon log LOG_INFO 6 6 6 6 6 6 # unix consensus +syscon log LOG_DEBUG 7 7 7 7 7 7 # unix consensus +syscon log LOG_PRIMASK 7 7 7 7 7 7 # unix consensus +syscon log LOG_NDELAY 8 8 8 8 8 8 # unix consensus +syscon log LOG_USER 8 8 8 8 8 8 # unix consensus +syscon log LOG_MAIL 0x10 0x10 0x10 0x10 0x10 0x10 # unix consensus +syscon log LOG_NOWAIT 0x10 0x10 0x10 0x10 0x10 0x10 # unix consensus +syscon log LOG_DAEMON 24 24 24 24 24 24 # unix consensus +syscon log LOG_NFACILITIES 24 25 24 24 24 24 +syscon log LOG_AUTH 0x20 0x20 0x20 0x20 0x20 0x20 # unix consensus +syscon log LOG_PERROR 0x20 0x20 0x20 0x20 0x20 0x20 # unix consensus +syscon log LOG_SYSLOG 40 40 40 40 40 40 # unix consensus +syscon log LOG_LPR 48 48 48 48 48 48 # unix consensus +syscon log LOG_NEWS 56 56 56 56 56 56 # unix consensus +syscon log LOG_UUCP 0x40 0x40 0x40 0x40 0x40 40 # unix consensus +syscon log LOG_CRON 72 72 72 72 72 72 # unix consensus +syscon log LOG_SELECT 76 0 0 0 0 0 +syscon log LOG_SENSE 77 0 0 0 0 0 +syscon log LOG_LOCAL0 0x80 0x80 0x80 0x80 0x80 0x80 # unix consensus +syscon log LOG_LOCAL1 136 136 136 136 136 136 # unix consensus +syscon log LOG_LOCAL2 144 144 144 144 144 144 # unix consensus +syscon log LOG_LOCAL3 152 152 152 152 152 152 # unix consensus +syscon log LOG_LOCAL4 160 160 160 160 160 160 # unix consensus +syscon log LOG_LOCAL5 168 168 168 168 168 168 # unix consensus +syscon log LOG_LOCAL6 176 176 176 176 176 176 # unix consensus +syscon log LOG_LOCAL7 184 184 184 184 184 184 # unix consensus +syscon log LOG_FACMASK 0x03f8 0x03f8 0x03f8 0x03f8 0x03f8 0x03f8 # unix consensus + +syscon sg SG_DXFER_TO_FROM_DEV -4 0 0 0 0 0 +syscon sg SG_DXFER_FROM_DEV -3 0 0 0 0 0 +syscon sg SG_DXFER_TO_DEV -2 0 0 0 0 0 +syscon sg SG_DXFER_NONE -1 0 0 0 0 0 +syscon sg SG_DEF_COMMAND_Q 0 0 0 0 0 0 # consensus +syscon sg SG_DEF_FORCE_LOW_DMA 0 0 0 0 0 0 # consensus +syscon sg SG_DEF_FORCE_PACK_ID 0 0 0 0 0 0 # consensus +syscon sg SG_DEF_KEEP_ORPHAN 0 0 0 0 0 0 # consensus +syscon sg SG_DEF_UNDERRUN_FLAG 0 0 0 0 0 0 # consensus +syscon sg SG_INFO_INDIRECT_IO 0 0 0 0 0 0 # consensus +syscon sg SG_INFO_OK 0 0 0 0 0 0 # consensus +syscon sg SG_SCSI_RESET_NOTHING 0 0 0 0 0 0 # consensus +syscon sg SG_DEFAULT_RETRIES 1 0 0 0 0 0 +syscon sg SG_FLAG_DIRECT_IO 1 0 0 0 0 0 +syscon sg SG_INFO_CHECK 1 0 0 0 0 0 +syscon sg SG_INFO_OK_MASK 1 0 0 0 0 0 +syscon sg SG_SCSI_RESET_DEVICE 1 0 0 0 0 0 +syscon sg SG_FLAG_LUN_INHIBIT 2 0 0 0 0 0 +syscon sg SG_INFO_DIRECT_IO 2 0 0 0 0 0 +syscon sg SG_SCSI_RESET_BUS 2 0 0 0 0 0 +syscon sg SG_SCSI_RESET_HOST 3 0 0 0 0 0 +syscon sg SG_INFO_MIXED_IO 4 0 0 0 0 0 +syscon sg SG_INFO_DIRECT_IO_MASK 6 0 0 0 0 0 +syscon misc VOLUME_OVERFLOW 13 0 0 0 0 0 +syscon sg SG_MAX_QUEUE 0x10 0 0 0 0 0 +syscon sg SG_MAX_SENSE 0x10 0 0 0 0 0 +syscon sg SG_DEFAULT_TIMEOUT 0x1770 0 0 0 0 0 +syscon sg SG_SET_TIMEOUT 0x2201 0 0 0 0 0 +syscon sg SG_GET_TIMEOUT 0x2202 0 0 0 0 0 +syscon sg SG_EMULATED_HOST 0x2203 0 0 0 0 0 +syscon sg SG_SET_TRANSFORM 0x2204 0 0 0 0 0 +syscon sg SG_GET_TRANSFORM 0x2205 0 0 0 0 0 +syscon sg SG_GET_COMMAND_Q 0x2270 0 0 0 0 0 +syscon sg SG_SET_COMMAND_Q 0x2271 0 0 0 0 0 +syscon sg SG_GET_RESERVED_SIZE 0x2272 0 0 0 0 0 +syscon sg SG_SET_RESERVED_SIZE 0x2275 0 0 0 0 0 +syscon sg SG_GET_SCSI_ID 0x2276 0 0 0 0 0 +syscon sg SG_SET_FORCE_LOW_DMA 0x2279 0 0 0 0 0 +syscon sg SG_GET_LOW_DMA 0x227a 0 0 0 0 0 +syscon sg SG_SET_FORCE_PACK_ID 0x227b 0 0 0 0 0 +syscon sg SG_GET_PACK_ID 0x227c 0 0 0 0 0 +syscon sg SG_GET_NUM_WAITING 0x227d 0 0 0 0 0 +syscon sg SG_SET_DEBUG 0x227e 0 0 0 0 0 +syscon sg SG_GET_SG_TABLESIZE 0x227f 0 0 0 0 0 +syscon sg SG_GET_VERSION_NUM 0x2282 0 0 0 0 0 +syscon sg SG_NEXT_CMD_LEN 0x2283 0 0 0 0 0 +syscon sg SG_SCSI_RESET 0x2284 0 0 0 0 0 +syscon sg SG_IO 0x2285 0 0 0 0 0 +syscon sg SG_GET_REQUEST_TABLE 0x2286 0 0 0 0 0 +syscon sg SG_SET_KEEP_ORPHAN 0x2287 0 0 0 0 0 +syscon sg SG_GET_KEEP_ORPHAN 0x2288 0 0 0 0 0 +syscon sg SG_BIG_BUFF 0x8000 0 0 0 0 0 +syscon sg SG_DEF_RESERVED_SIZE 0x8000 0 0 0 0 0 +syscon sg SG_SCATTER_SZ 0x8000 0 0 0 0 0 +syscon sg SG_FLAG_NO_DXFER 0x010000 0 0 0 0 0 + # https://youtu.be/GUQUD3IMbb4?t=85 diff --git a/libc/sysv/consts/AI_ADDRCONFIG.S b/libc/sysv/consts/AI_ADDRCONFIG.S deleted file mode 100644 index 500884140..000000000 --- a/libc/sysv/consts/AI_ADDRCONFIG.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_ADDRCONFIG,0x20,0x0400,0x0400,0x40,0x40,0x0400 diff --git a/libc/sysv/consts/AI_ALL.S b/libc/sysv/consts/AI_ALL.S deleted file mode 100644 index 2193f2ef2..000000000 --- a/libc/sysv/consts/AI_ALL.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_ALL,0x10,0x0100,0x0100,0,0,0x0100 diff --git a/libc/sysv/consts/AI_CANONNAME.S b/libc/sysv/consts/AI_CANONNAME.S deleted file mode 100644 index 6639a8f20..000000000 --- a/libc/sysv/consts/AI_CANONNAME.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_CANONNAME,2,2,2,2,2,2 diff --git a/libc/sysv/consts/AI_NUMERICHOST.S b/libc/sysv/consts/AI_NUMERICHOST.S deleted file mode 100644 index 08404d179..000000000 --- a/libc/sysv/consts/AI_NUMERICHOST.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_NUMERICHOST,4,4,4,4,4,4 diff --git a/libc/sysv/consts/AI_NUMERICSERV.S b/libc/sysv/consts/AI_NUMERICSERV.S deleted file mode 100644 index f93e42cf8..000000000 --- a/libc/sysv/consts/AI_NUMERICSERV.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_NUMERICSERV,0x0400,0x1000,8,0x10,0x10,8 diff --git a/libc/sysv/consts/AI_PASSIVE.S b/libc/sysv/consts/AI_PASSIVE.S deleted file mode 100644 index fd32e1718..000000000 --- a/libc/sysv/consts/AI_PASSIVE.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_PASSIVE,1,1,1,1,1,1 diff --git a/libc/sysv/consts/AI_V4MAPPED.S b/libc/sysv/consts/AI_V4MAPPED.S deleted file mode 100644 index 08122a1b1..000000000 --- a/libc/sysv/consts/AI_V4MAPPED.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon gai,AI_V4MAPPED,8,0x0800,0x0800,0,0,0x0800 diff --git a/libc/sysv/consts/BIG_ENDIAN.S b/libc/sysv/consts/BIG_ENDIAN.S deleted file mode 100644 index 8e37ec0ad..000000000 --- a/libc/sysv/consts/BIG_ENDIAN.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BIG_ENDIAN,0x10e1,0x10e1,0x10e1,0x10e1,0x10e1,0 diff --git a/libc/sysv/consts/BLK_BYTECOUNT.S b/libc/sysv/consts/BLK_BYTECOUNT.S deleted file mode 100644 index 848166f5d..000000000 --- a/libc/sysv/consts/BLK_BYTECOUNT.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BLK_BYTECOUNT,2,2,2,2,2,0 diff --git a/libc/sysv/consts/BLK_EOF.S b/libc/sysv/consts/BLK_EOF.S deleted file mode 100644 index 9d9439db7..000000000 --- a/libc/sysv/consts/BLK_EOF.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BLK_EOF,0x40,0x40,0x40,0x40,0x40,0 diff --git a/libc/sysv/consts/BLK_EOR.S b/libc/sysv/consts/BLK_EOR.S deleted file mode 100644 index 1acd0dfe5..000000000 --- a/libc/sysv/consts/BLK_EOR.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BLK_EOR,0x80,0x80,0x80,0x80,0x80,0 diff --git a/libc/sysv/consts/BLK_ERRORS.S b/libc/sysv/consts/BLK_ERRORS.S deleted file mode 100644 index 34b41a252..000000000 --- a/libc/sysv/consts/BLK_ERRORS.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BLK_ERRORS,0x20,0x20,0x20,0x20,0x20,0 diff --git a/libc/sysv/consts/BLK_RESTART.S b/libc/sysv/consts/BLK_RESTART.S deleted file mode 100644 index 75cdc463a..000000000 --- a/libc/sysv/consts/BLK_RESTART.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BLK_RESTART,0x10,0x10,0x10,0x10,0x10,0 diff --git a/libc/sysv/consts/BUS_DEVICE_RESET.S b/libc/sysv/consts/BUS_DEVICE_RESET.S index 1d56d622f..0e33b3170 100644 --- a/libc/sysv/consts/BUS_DEVICE_RESET.S +++ b/libc/sysv/consts/BUS_DEVICE_RESET.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,BUS_DEVICE_RESET,12,0,0,0,0,0 +.syscon scsi,BUS_DEVICE_RESET,12,0,0,0,0,0 diff --git a/libc/sysv/consts/CR0.S b/libc/sysv/consts/CR0.S index 79e096b2e..02c2e947d 100644 --- a/libc/sysv/consts/CR0.S +++ b/libc/sysv/consts/CR0.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,CR0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0x0,0x0,0b0000000000000000 +.syscon termios,CR0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0,0b000000000000000000,0b0000000000000000 diff --git a/libc/sysv/consts/CR1.S b/libc/sysv/consts/CR1.S index 49059cb78..480913c80 100644 --- a/libc/sysv/consts/CR1.S +++ b/libc/sysv/consts/CR1.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,CR1,0b0000001000000000,0b000001000000000000,0b000001000000000000,0x0,0x0,0b0000001000000000 +.syscon termios,CR1,0b0000001000000000,0b000001000000000000,0b000001000000000000,0,0b000001000000000000,0b0000001000000000 diff --git a/libc/sysv/consts/CR2.S b/libc/sysv/consts/CR2.S index dbdc665bb..bb27c6949 100644 --- a/libc/sysv/consts/CR2.S +++ b/libc/sysv/consts/CR2.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,CR2,0b0000010000000000,0b000010000000000000,0b000010000000000000,0x0,0x0,0b0000010000000000 +.syscon termios,CR2,0b0000010000000000,0b000010000000000000,0b000010000000000000,0,0b000000010000000000,0b0000010000000000 diff --git a/libc/sysv/consts/CR3.S b/libc/sysv/consts/CR3.S index 3cebe9dcf..5e65c9feb 100644 --- a/libc/sysv/consts/CR3.S +++ b/libc/sysv/consts/CR3.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,CR3,0b0000011000000000,0b000011000000000000,0b000011000000000000,0x0,0x0,0b0000011000000000 +.syscon termios,CR3,0b0000011000000000,0b000011000000000000,0b000011000000000000,0,0b000000011000000000,0b0000011000000000 diff --git a/libc/sysv/consts/EBADFD.S b/libc/sysv/consts/EBADFD.S index 3d23845e6..451998577 100644 --- a/libc/sysv/consts/EBADFD.S +++ b/libc/sysv/consts/EBADFD.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,EBADFD,77,9,9,9,9,6 +.syscon junkerr,EBADFD,77,9,0,0,0,0 diff --git a/libc/sysv/consts/EMEDIUMTYPE.S b/libc/sysv/consts/EMEDIUMTYPE.S index 488841a65..e2397891f 100644 --- a/libc/sysv/consts/EMEDIUMTYPE.S +++ b/libc/sysv/consts/EMEDIUMTYPE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,EMEDIUMTYPE,124,0,0,86,86,0 +.syscon errno,EMEDIUMTYPE,124,0,0,86,0,0 diff --git a/libc/sysv/consts/EMPTY.S b/libc/sysv/consts/EMPTY.S index 26cd5eb86..5a9295f00 100644 --- a/libc/sysv/consts/EMPTY.S +++ b/libc/sysv/consts/EMPTY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,EMPTY,0,0,0,0,0,0 +.syscon termios,EMPTY,0,0,0,0,0,0 diff --git a/libc/sysv/consts/EMULTIHOP.S b/libc/sysv/consts/EMULTIHOP.S index cfff27b7d..434e78754 100644 --- a/libc/sysv/consts/EMULTIHOP.S +++ b/libc/sysv/consts/EMULTIHOP.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,EMULTIHOP,72,95,90,0,94,0 +.syscon errno,EMULTIHOP,72,95,90,0,94,0 diff --git a/libc/sysv/consts/ENODATA.S b/libc/sysv/consts/ENODATA.S index c406f9ae6..e9b938bbb 100644 --- a/libc/sysv/consts/ENODATA.S +++ b/libc/sysv/consts/ENODATA.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,ENODATA,61,96,0,0,89,0 +.syscon errno,ENODATA,61,96,0,0,89,0 diff --git a/libc/sysv/consts/ENOLINK.S b/libc/sysv/consts/ENOLINK.S index 9d890098e..875ea6b1b 100644 --- a/libc/sysv/consts/ENOLINK.S +++ b/libc/sysv/consts/ENOLINK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,ENOLINK,67,97,91,0,95,0 +.syscon errno,ENOLINK,67,97,91,0,95,0 diff --git a/libc/sysv/consts/ENOMEDIUM.S b/libc/sysv/consts/ENOMEDIUM.S index 7243999b8..e7125df4b 100644 --- a/libc/sysv/consts/ENOMEDIUM.S +++ b/libc/sysv/consts/ENOMEDIUM.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,ENOMEDIUM,123,0,0,85,85,0 +.syscon errno,ENOMEDIUM,123,0,0,85,0,0 diff --git a/libc/sysv/consts/ENOSR.S b/libc/sysv/consts/ENOSR.S index ea4d91cf0..6dde58df3 100644 --- a/libc/sysv/consts/ENOSR.S +++ b/libc/sysv/consts/ENOSR.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,ENOSR,63,98,0,0,90,0 +.syscon errno,ENOSR,63,98,0,90,90,0 diff --git a/libc/sysv/consts/ENOSTR.S b/libc/sysv/consts/ENOSTR.S index 9dde57279..749624fa4 100644 --- a/libc/sysv/consts/ENOSTR.S +++ b/libc/sysv/consts/ENOSTR.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon junkerr,ENOSTR,60,99,0,0,91,0 +.syscon errno,ENOSTR,60,99,0,0,91,0 diff --git a/libc/sysv/consts/ERA.S b/libc/sysv/consts/ERA.S index 59abd10c8..b356dc911 100644 --- a/libc/sysv/consts/ERA.S +++ b/libc/sysv/consts/ERA.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ERA,0x02002c,45,45,0,0,0 +.syscon termios,ERA,0x02002c,45,45,0,0,0 diff --git a/libc/sysv/consts/ERESTART.S b/libc/sysv/consts/ERESTART.S index 646d27dfd..8cd52c1b9 100644 --- a/libc/sysv/consts/ERESTART.S +++ b/libc/sysv/consts/ERESTART.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon errno,ERESTART,85,0,0,0,-3,0 +.syscon errno,ERESTART,85,-1,-1,-1,-3,0 diff --git a/libc/sysv/consts/EXTA.S b/libc/sysv/consts/EXTA.S index 802c182c5..9c56df558 100644 --- a/libc/sysv/consts/EXTA.S +++ b/libc/sysv/consts/EXTA.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,EXTA,14,0x4b00,0x4b00,0x4b00,0x4b00,0 +.syscon termios,EXTA,14,0x4b00,0x4b00,0x4b00,0x4b00,0 diff --git a/libc/sysv/consts/EXTB.S b/libc/sysv/consts/EXTB.S index 5177cf47a..82b0ece7d 100644 --- a/libc/sysv/consts/EXTB.S +++ b/libc/sysv/consts/EXTB.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,EXTB,15,0x9600,0x9600,0x9600,0x9600,0 +.syscon termios,EXTB,15,0x9600,0x9600,0x9600,0x9600,0 diff --git a/libc/sysv/consts/F_GETLK.S b/libc/sysv/consts/F_GETLK.S index 1dd0b999a..474192801 100644 --- a/libc/sysv/consts/F_GETLK.S +++ b/libc/sysv/consts/F_GETLK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_GETLK,5,7,11,7,7,0 +.syscon fcntl,F_GETLK,5,7,11,7,7,5 diff --git a/libc/sysv/consts/F_GETLK64.S b/libc/sysv/consts/F_GETLK64.S index 6d2fcfeed..b4a57a902 100644 --- a/libc/sysv/consts/F_GETLK64.S +++ b/libc/sysv/consts/F_GETLK64.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_GETLK64,5,0,0,0,0,0 +.syscon compat,F_GETLK64,5,7,11,7,7,5 diff --git a/libc/sysv/consts/F_SETLK.S b/libc/sysv/consts/F_SETLK.S index a229d7181..6f4ec8b29 100644 --- a/libc/sysv/consts/F_SETLK.S +++ b/libc/sysv/consts/F_SETLK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_SETLK,6,8,12,8,8,0 +.syscon fcntl,F_SETLK,6,8,12,8,8,6 diff --git a/libc/sysv/consts/F_SETLK64.S b/libc/sysv/consts/F_SETLK64.S index 7aa999969..0747c5aed 100644 --- a/libc/sysv/consts/F_SETLK64.S +++ b/libc/sysv/consts/F_SETLK64.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_SETLK64,6,0,0,0,0,0 +.syscon compat,F_SETLK64,6,8,12,8,8,6 diff --git a/libc/sysv/consts/F_SETLKW.S b/libc/sysv/consts/F_SETLKW.S index 48896c36e..042933722 100644 --- a/libc/sysv/consts/F_SETLKW.S +++ b/libc/sysv/consts/F_SETLKW.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_SETLKW,7,9,13,9,9,0 +.syscon fcntl,F_SETLKW,7,9,13,9,9,7 diff --git a/libc/sysv/consts/F_SETLKW64.S b/libc/sysv/consts/F_SETLKW64.S index 31dca8a64..569ba2b70 100644 --- a/libc/sysv/consts/F_SETLKW64.S +++ b/libc/sysv/consts/F_SETLKW64.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_SETLKW64,7,0,0,0,0,0 +.syscon compat,F_SETLKW64,7,9,13,9,9,7 diff --git a/libc/sysv/consts/F_UNLCK.S b/libc/sysv/consts/F_UNLCK.S index 5811e4222..6e40c08f4 100644 --- a/libc/sysv/consts/F_UNLCK.S +++ b/libc/sysv/consts/F_UNLCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_UNLCK,2,2,2,2,2,0 +.syscon fcntl,F_UNLCK,2,2,2,2,2,2 diff --git a/libc/sysv/consts/F_WRLCK.S b/libc/sysv/consts/F_WRLCK.S index 6f0aa3d8a..bf4098372 100644 --- a/libc/sysv/consts/F_WRLCK.S +++ b/libc/sysv/consts/F_WRLCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon fcntl,F_WRLCK,1,3,3,3,3,0 +.syscon fcntl,F_WRLCK,1,3,3,3,3,1 diff --git a/libc/sysv/consts/INADDR_ALLHOSTS_GROUP.S b/libc/sysv/consts/INADDR_ALLHOSTS_GROUP.S deleted file mode 100644 index 44972e02d..000000000 --- a/libc/sysv/consts/INADDR_ALLHOSTS_GROUP.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_ALLHOSTS_GROUP,0xe0000001,0xe0000001,0xe0000001,0xe0000001,0xe0000001,0 diff --git a/libc/sysv/consts/INADDR_ALLRTRS_GROUP.S b/libc/sysv/consts/INADDR_ALLRTRS_GROUP.S deleted file mode 100644 index 1685edf35..000000000 --- a/libc/sysv/consts/INADDR_ALLRTRS_GROUP.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_ALLRTRS_GROUP,0xe0000002,0xe0000002,0xe0000002,0,0,0 diff --git a/libc/sysv/consts/INADDR_ANY.S b/libc/sysv/consts/INADDR_ANY.S deleted file mode 100644 index 0c0731778..000000000 --- a/libc/sysv/consts/INADDR_ANY.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_ANY,0,0,0,0,0,0 diff --git a/libc/sysv/consts/INADDR_BROADCAST.S b/libc/sysv/consts/INADDR_BROADCAST.S deleted file mode 100644 index 44147bdca..000000000 --- a/libc/sysv/consts/INADDR_BROADCAST.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_BROADCAST,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff diff --git a/libc/sysv/consts/INADDR_LOOPBACK.S b/libc/sysv/consts/INADDR_LOOPBACK.S deleted file mode 100644 index 035b6dcfb..000000000 --- a/libc/sysv/consts/INADDR_LOOPBACK.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_LOOPBACK,0x7f000001,0x7f000001,0x7f000001,0x7f000001,0x7f000001,0x7f000001 diff --git a/libc/sysv/consts/INADDR_MAX_LOCAL_GROUP.S b/libc/sysv/consts/INADDR_MAX_LOCAL_GROUP.S deleted file mode 100644 index dc6a9b1c6..000000000 --- a/libc/sysv/consts/INADDR_MAX_LOCAL_GROUP.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_MAX_LOCAL_GROUP,0xe00000ff,0xe00000ff,0xe00000ff,0xe00000ff,0xe00000ff,0 diff --git a/libc/sysv/consts/INADDR_NONE.S b/libc/sysv/consts/INADDR_NONE.S deleted file mode 100644 index 7add73654..000000000 --- a/libc/sysv/consts/INADDR_NONE.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_NONE,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff diff --git a/libc/sysv/consts/INADDR_UNSPEC_GROUP.S b/libc/sysv/consts/INADDR_UNSPEC_GROUP.S deleted file mode 100644 index b9533448b..000000000 --- a/libc/sysv/consts/INADDR_UNSPEC_GROUP.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,INADDR_UNSPEC_GROUP,0xe0000000,0xe0000000,0xe0000000,0xe0000000,0xe0000000,0 diff --git a/libc/sysv/consts/IUCLC.S b/libc/sysv/consts/IUCLC.S index d6e17f9c3..a65687f66 100644 --- a/libc/sysv/consts/IUCLC.S +++ b/libc/sysv/consts/IUCLC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,IUCLC,0b0000001000000000,0,0,0b0001000000000000,0b0001000000000000,0b0000001000000000 +.syscon termios,IUCLC,0b0000001000000000,0,0,0b0001000000000000,0,0b0000001000000000 diff --git a/libc/sysv/consts/LITTLE_ENDIAN.S b/libc/sysv/consts/LITTLE_ENDIAN.S deleted file mode 100644 index 0999effaa..000000000 --- a/libc/sysv/consts/LITTLE_ENDIAN.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,LITTLE_ENDIAN,0x04d2,0x04d2,0x04d2,0x04d2,0x04d2,0 diff --git a/libc/sysv/consts/MAP_CONCEAL.S b/libc/sysv/consts/MAP_CONCEAL.S index 8b4b49875..8077b1792 100644 --- a/libc/sysv/consts/MAP_CONCEAL.S +++ b/libc/sysv/consts/MAP_CONCEAL.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon mmap,MAP_CONCEAL,0,0,0x20000,0x8000,0x8000,0 +.syscon mmap,MAP_CONCEAL,0,0,0,0x8000,0,0 diff --git a/libc/sysv/consts/MAP_TYPE.S b/libc/sysv/consts/MAP_TYPE.S index 8b2daa23b..d530757ed 100644 --- a/libc/sysv/consts/MAP_TYPE.S +++ b/libc/sysv/consts/MAP_TYPE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon mmap,MAP_TYPE,15,0,0,0,0,0 +.syscon mmap,MAP_TYPE,15,15,15,15,15,15 diff --git a/libc/sysv/consts/MODE_B.S b/libc/sysv/consts/MODE_B.S deleted file mode 100644 index 7385006b9..000000000 --- a/libc/sysv/consts/MODE_B.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_B,2,2,2,2,2,0 diff --git a/libc/sysv/consts/MODE_C.S b/libc/sysv/consts/MODE_C.S deleted file mode 100644 index 8b01e205d..000000000 --- a/libc/sysv/consts/MODE_C.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_C,3,3,3,3,3,0 diff --git a/libc/sysv/consts/MODE_S.S b/libc/sysv/consts/MODE_S.S deleted file mode 100644 index 80a10d825..000000000 --- a/libc/sysv/consts/MODE_S.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_S,1,1,1,1,1,0 diff --git a/libc/sysv/consts/MODE_SELECT.S b/libc/sysv/consts/MODE_SELECT.S deleted file mode 100644 index 3521ec2b3..000000000 --- a/libc/sysv/consts/MODE_SELECT.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_SELECT,21,0,0,0,0,0 diff --git a/libc/sysv/consts/MODE_SELECT_10.S b/libc/sysv/consts/MODE_SELECT_10.S deleted file mode 100644 index 72200e6c6..000000000 --- a/libc/sysv/consts/MODE_SELECT_10.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_SELECT_10,85,0,0,0,0,0 diff --git a/libc/sysv/consts/MODE_SENSE.S b/libc/sysv/consts/MODE_SENSE.S deleted file mode 100644 index 40108da52..000000000 --- a/libc/sysv/consts/MODE_SENSE.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_SENSE,26,0,0,0,0,0 diff --git a/libc/sysv/consts/MODE_SENSE_10.S b/libc/sysv/consts/MODE_SENSE_10.S deleted file mode 100644 index ca1fd682b..000000000 --- a/libc/sysv/consts/MODE_SENSE_10.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon misc,MODE_SENSE_10,90,0,0,0,0,0 diff --git a/libc/sysv/consts/NL0.S b/libc/sysv/consts/NL0.S index 073fa6c8d..5d667bfd8 100644 --- a/libc/sysv/consts/NL0.S +++ b/libc/sysv/consts/NL0.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,NL0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0,0,0b0000000000000000 +.syscon termios,NL0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0,0b000000000000000000,0b0000000000000000 diff --git a/libc/sysv/consts/NL1.S b/libc/sysv/consts/NL1.S index e63bb35cd..c2d164edb 100644 --- a/libc/sysv/consts/NL1.S +++ b/libc/sysv/consts/NL1.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,NL1,0b0000000100000000,0b000000000100000000,0b000000000100000000,0,0,0b0000000100000000 +.syscon termios,NL1,0b0000000100000000,0b000000000100000000,0b000000000100000000,0,0b000000000100000000,0b0000000100000000 diff --git a/libc/sysv/consts/NL2.S b/libc/sysv/consts/NL2.S index 8407a6aca..38a536303 100644 --- a/libc/sysv/consts/NL2.S +++ b/libc/sysv/consts/NL2.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,NL2,0,0b000000001000000000,0b000000001000000000,0,0,0 +.syscon termios,NL2,0,0b000000001000000000,0b000000001000000000,0,0b000000001000000000,0 diff --git a/libc/sysv/consts/NL3.S b/libc/sysv/consts/NL3.S index 1f9e92aec..6bd74840d 100644 --- a/libc/sysv/consts/NL3.S +++ b/libc/sysv/consts/NL3.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,NL3,0,0b000000001100000000,0b000000001100000000,0,0,0 +.syscon termios,NL3,0,0b000000001100000000,0b000000001100000000,0,0b000000001100000000,0 diff --git a/libc/sysv/consts/OLCUC.S b/libc/sysv/consts/OLCUC.S index 8122a71db..9a644b4a5 100644 --- a/libc/sysv/consts/OLCUC.S +++ b/libc/sysv/consts/OLCUC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,OLCUC,0b0000000000000010,0b000000000000000000,0,0b0000000000100000,0b0000000000100000,0b0000000000000010 +.syscon termios,OLCUC,0b0000000000000010,0,0,0b0000000000100000,0,0b0000000000000010 diff --git a/libc/sysv/consts/O_APPEND.S b/libc/sysv/consts/O_APPEND.S index 11f5843fd..ee7a00db2 100644 --- a/libc/sysv/consts/O_APPEND.S +++ b/libc/sysv/consts/O_APPEND.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_APPEND,0x0400,8,8,8,8,0x00000004 +.syscon open,O_APPEND,0x00000400,8,8,8,8,0x00000004 diff --git a/libc/sysv/consts/O_ASYNC.S b/libc/sysv/consts/O_ASYNC.S index 6e7067b75..63cbb6d9b 100644 --- a/libc/sysv/consts/O_ASYNC.S +++ b/libc/sysv/consts/O_ASYNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_ASYNC,0x2000,0x40,0x40,0x40,0x40,0 +.syscon open,O_ASYNC,0x00002000,0x00000040,0x00000040,0x00000040,0x00000040,0 diff --git a/libc/sysv/consts/O_CLOEXEC.S b/libc/sysv/consts/O_CLOEXEC.S index 828eab068..77de11e39 100644 --- a/libc/sysv/consts/O_CLOEXEC.S +++ b/libc/sysv/consts/O_CLOEXEC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_CLOEXEC,0x080000,0x01000000,0x100000,0x010000,0x400000,0x00080000 +.syscon open,O_CLOEXEC,0x00080000,0x01000000,0x00100000,0x00010000,0x00400000,0x00080000 diff --git a/libc/sysv/consts/O_CREAT.S b/libc/sysv/consts/O_CREAT.S index ca37ec698..9924d1ecc 100644 --- a/libc/sysv/consts/O_CREAT.S +++ b/libc/sysv/consts/O_CREAT.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_CREAT,0x40,0x0200,0x0200,0x0200,0x0200,0x00000040 +.syscon open,O_CREAT,0x00000040,0x00000200,0x00000200,0x00000200,0x00000200,0x00000040 diff --git a/libc/sysv/consts/O_DIRECT.S b/libc/sysv/consts/O_DIRECT.S index 65395064f..f9be1eb9c 100644 --- a/libc/sysv/consts/O_DIRECT.S +++ b/libc/sysv/consts/O_DIRECT.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_DIRECT,0x4000,0,0x010000,0,0x80000,0x00200000 +.syscon open,O_DIRECT,0x00004000,0,0x00010000,0,0x00080000,0x00200000 diff --git a/libc/sysv/consts/O_DIRECTORY.S b/libc/sysv/consts/O_DIRECTORY.S index 33865aec6..00fcac4dd 100644 --- a/libc/sysv/consts/O_DIRECTORY.S +++ b/libc/sysv/consts/O_DIRECTORY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_DIRECTORY,0x010000,0x100000,0x020000,0x020000,0x200000,0x02000000 +.syscon open,O_DIRECTORY,0x00010000,0x00100000,0x00020000,0x00020000,0x00200000,0x02000000 diff --git a/libc/sysv/consts/O_DSYNC.S b/libc/sysv/consts/O_DSYNC.S index 1d55d6983..1670c5476 100644 --- a/libc/sysv/consts/O_DSYNC.S +++ b/libc/sysv/consts/O_DSYNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_DSYNC,0x1000,0x400000,0,0x80,0x10000,0 +.syscon open,O_DSYNC,0x00001000,0x00400000,0,0x00000080,0x00010000,0 diff --git a/libc/sysv/consts/O_EXCL.S b/libc/sysv/consts/O_EXCL.S index 3798997c8..07615526b 100644 --- a/libc/sysv/consts/O_EXCL.S +++ b/libc/sysv/consts/O_EXCL.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_EXCL,0x80,0x0800,0x0800,0x0800,0x0800,0x00000080 +.syscon open,O_EXCL,0x00000080,0x00000800,0x00000800,0x00000800,0x00000800,0x00000080 diff --git a/libc/sysv/consts/O_EXEC.S b/libc/sysv/consts/O_EXEC.S index 28dadbf86..f8668386a 100644 --- a/libc/sysv/consts/O_EXEC.S +++ b/libc/sysv/consts/O_EXEC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_EXEC,0,0,0x040000,0,0x4000000,0 +.syscon open,O_EXEC,0,0,0x00040000,0,0x04000000,0 diff --git a/libc/sysv/consts/O_NDELAY.S b/libc/sysv/consts/O_NDELAY.S index 4aa123055..bb2191050 100644 --- a/libc/sysv/consts/O_NDELAY.S +++ b/libc/sysv/consts/O_NDELAY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_NDELAY,0x0800,4,4,4,4,0 +.syscon open,O_NDELAY,0x00000800,0x00000004,0x00000004,0x00000004,0x00000004,0x00000800 diff --git a/libc/sysv/consts/O_NOATIME.S b/libc/sysv/consts/O_NOATIME.S index 107d98f67..3ae86dc6a 100644 --- a/libc/sysv/consts/O_NOATIME.S +++ b/libc/sysv/consts/O_NOATIME.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_NOATIME,0x040000,0,0,0,0,0 +.syscon open,O_NOATIME,0x00040000,0,0,0,0,0 diff --git a/libc/sysv/consts/O_NOCTTY.S b/libc/sysv/consts/O_NOCTTY.S index 48e72b6f3..f6a1df496 100644 --- a/libc/sysv/consts/O_NOCTTY.S +++ b/libc/sysv/consts/O_NOCTTY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_NOCTTY,0x0100,0x020000,0x8000,0x8000,0x8000,0 +.syscon open,O_NOCTTY,0x00000100,0x00020000,0x00008000,0x00008000,0x00008000,0 diff --git a/libc/sysv/consts/O_NOFOLLOW.S b/libc/sysv/consts/O_NOFOLLOW.S index 242824fe1..b0b255d23 100644 --- a/libc/sysv/consts/O_NOFOLLOW.S +++ b/libc/sysv/consts/O_NOFOLLOW.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_NOFOLLOW,0x020000,0x0100,0x0100,0x0100,0x0100,0 +.syscon open,O_NOFOLLOW,0x00020000,0x00000100,0x00000100,0x00000100,0x00000100,0 diff --git a/libc/sysv/consts/O_NONBLOCK.S b/libc/sysv/consts/O_NONBLOCK.S index e62c9728b..1ca6b0780 100644 --- a/libc/sysv/consts/O_NONBLOCK.S +++ b/libc/sysv/consts/O_NONBLOCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_NONBLOCK,0x0800,4,4,4,4,0x00000800 +.syscon open,O_NONBLOCK,0x00000800,0x00000004,0x00000004,0x00000004,0x00000004,0x00000800 diff --git a/libc/sysv/consts/O_PATH.S b/libc/sysv/consts/O_PATH.S index 11af8d8db..d4603a0c7 100644 --- a/libc/sysv/consts/O_PATH.S +++ b/libc/sysv/consts/O_PATH.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_PATH,0x200000,0,0,0,0,0 +.syscon open,O_PATH,0x00200000,0,0,0,0,0 diff --git a/libc/sysv/consts/O_RSYNC.S b/libc/sysv/consts/O_RSYNC.S index 891355ee5..e412b6b76 100644 --- a/libc/sysv/consts/O_RSYNC.S +++ b/libc/sysv/consts/O_RSYNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_RSYNC,0x101000,0,0,0x80,0x20000,0 +.syscon open,O_RSYNC,0x00101000,0,0,0x00000080,0x00020000,0 diff --git a/libc/sysv/consts/O_SYNC.S b/libc/sysv/consts/O_SYNC.S index 4845b58a7..fefb1ee29 100644 --- a/libc/sysv/consts/O_SYNC.S +++ b/libc/sysv/consts/O_SYNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_SYNC,0x101000,0x80,0x80,0x80,0x80,0 +.syscon open,O_SYNC,0x00101000,0x00000080,0x00000080,0x00000080,0x00000080,0 diff --git a/libc/sysv/consts/O_TMPFILE.S b/libc/sysv/consts/O_TMPFILE.S index 628774f13..3c060605b 100644 --- a/libc/sysv/consts/O_TMPFILE.S +++ b/libc/sysv/consts/O_TMPFILE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_TMPFILE,0x410000,0,0,0,0,0x04000100 +.syscon open,O_TMPFILE,0x00410000,0,0,0,0,0x04000100 diff --git a/libc/sysv/consts/O_TRUNC.S b/libc/sysv/consts/O_TRUNC.S index b701802f6..c54bfc428 100644 --- a/libc/sysv/consts/O_TRUNC.S +++ b/libc/sysv/consts/O_TRUNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_TRUNC,0x0200,0x0400,0x0400,0x0400,0x0400,0x00000200 +.syscon open,O_TRUNC,0x00000200,0x00000400,0x00000400,0x00000400,0x00000400,0x00000200 diff --git a/libc/sysv/consts/O_TTY_INIT.S b/libc/sysv/consts/O_TTY_INIT.S index 470a7568a..f5d9281e4 100644 --- a/libc/sysv/consts/O_TTY_INIT.S +++ b/libc/sysv/consts/O_TTY_INIT.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon open,O_TTY_INIT,0,0,0x080000,0,0,0 +.syscon open,O_TTY_INIT,0,0,0x00080000,0,0,0 diff --git a/libc/sysv/consts/READ_10.S b/libc/sysv/consts/READ_10.S index 0fb0766cc..61042b8e0 100644 --- a/libc/sysv/consts/READ_10.S +++ b/libc/sysv/consts/READ_10.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_10,40,0,0,0,0,0 +.syscon scsi,READ_10,40,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_12.S b/libc/sysv/consts/READ_12.S index fa76a110c..048ee93e2 100644 --- a/libc/sysv/consts/READ_12.S +++ b/libc/sysv/consts/READ_12.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_12,168,0,0,0,0,0 +.syscon scsi,READ_12,168,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_6.S b/libc/sysv/consts/READ_6.S index 6651219bc..00dae3890 100644 --- a/libc/sysv/consts/READ_6.S +++ b/libc/sysv/consts/READ_6.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_6,8,0,0,0,0,0 +.syscon scsi,READ_6,8,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_BLOCK_LIMITS.S b/libc/sysv/consts/READ_BLOCK_LIMITS.S index 7a3404c48..91b01f5ff 100644 --- a/libc/sysv/consts/READ_BLOCK_LIMITS.S +++ b/libc/sysv/consts/READ_BLOCK_LIMITS.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_BLOCK_LIMITS,5,0,0,0,0,0 +.syscon scsi,READ_BLOCK_LIMITS,5,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_BUFFER.S b/libc/sysv/consts/READ_BUFFER.S index 98b75d659..66c0bdf8e 100644 --- a/libc/sysv/consts/READ_BUFFER.S +++ b/libc/sysv/consts/READ_BUFFER.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_BUFFER,60,0,0,0,0,0 +.syscon scsi,READ_BUFFER,60,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_CAPACITY.S b/libc/sysv/consts/READ_CAPACITY.S index 93c58e317..6dd06c03e 100644 --- a/libc/sysv/consts/READ_CAPACITY.S +++ b/libc/sysv/consts/READ_CAPACITY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_CAPACITY,37,0,0,0,0,0 +.syscon scsi,READ_CAPACITY,37,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_DEFECT_DATA.S b/libc/sysv/consts/READ_DEFECT_DATA.S index e24c8202e..84fb61d8d 100644 --- a/libc/sysv/consts/READ_DEFECT_DATA.S +++ b/libc/sysv/consts/READ_DEFECT_DATA.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_DEFECT_DATA,55,0,0,0,0,0 +.syscon scsi,READ_DEFECT_DATA,55,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_ELEMENT_STATUS.S b/libc/sysv/consts/READ_ELEMENT_STATUS.S index 68c3368e1..14a56851f 100644 --- a/libc/sysv/consts/READ_ELEMENT_STATUS.S +++ b/libc/sysv/consts/READ_ELEMENT_STATUS.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_ELEMENT_STATUS,184,0,0,0,0,0 +.syscon scsi,READ_ELEMENT_STATUS,184,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_LONG.S b/libc/sysv/consts/READ_LONG.S index f5f595aac..7be143b66 100644 --- a/libc/sysv/consts/READ_LONG.S +++ b/libc/sysv/consts/READ_LONG.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_LONG,62,0,0,0,0,0 +.syscon scsi,READ_LONG,62,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_POSITION.S b/libc/sysv/consts/READ_POSITION.S index eb4477af9..393cb5ca4 100644 --- a/libc/sysv/consts/READ_POSITION.S +++ b/libc/sysv/consts/READ_POSITION.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_POSITION,52,0,0,0,0,0 +.syscon scsi,READ_POSITION,52,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_REVERSE.S b/libc/sysv/consts/READ_REVERSE.S index 6d3c641ef..d206e3f65 100644 --- a/libc/sysv/consts/READ_REVERSE.S +++ b/libc/sysv/consts/READ_REVERSE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_REVERSE,15,0,0,0,0,0 +.syscon scsi,READ_REVERSE,15,0,0,0,0,0 diff --git a/libc/sysv/consts/READ_TOC.S b/libc/sysv/consts/READ_TOC.S index ae5e483a4..46d1301be 100644 --- a/libc/sysv/consts/READ_TOC.S +++ b/libc/sysv/consts/READ_TOC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,READ_TOC,67,0,0,0,0,0 +.syscon scsi,READ_TOC,67,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_BENCHMARK_COMMAND.S b/libc/sysv/consts/SCSI_IOCTL_BENCHMARK_COMMAND.S index 3676f6549..cc60c5210 100644 --- a/libc/sysv/consts/SCSI_IOCTL_BENCHMARK_COMMAND.S +++ b/libc/sysv/consts/SCSI_IOCTL_BENCHMARK_COMMAND.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_BENCHMARK_COMMAND,3,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_BENCHMARK_COMMAND,3,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_DOORLOCK.S b/libc/sysv/consts/SCSI_IOCTL_DOORLOCK.S index d799cb125..6b409e871 100644 --- a/libc/sysv/consts/SCSI_IOCTL_DOORLOCK.S +++ b/libc/sysv/consts/SCSI_IOCTL_DOORLOCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_DOORLOCK,0x5380,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_DOORLOCK,0x5380,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_DOORUNLOCK.S b/libc/sysv/consts/SCSI_IOCTL_DOORUNLOCK.S index 3454dcad0..9aa9d4b65 100644 --- a/libc/sysv/consts/SCSI_IOCTL_DOORUNLOCK.S +++ b/libc/sysv/consts/SCSI_IOCTL_DOORUNLOCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_DOORUNLOCK,0x5381,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_DOORUNLOCK,0x5381,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_GET_BUS_NUMBER.S b/libc/sysv/consts/SCSI_IOCTL_GET_BUS_NUMBER.S index 470b22e22..1b96bdc54 100644 --- a/libc/sysv/consts/SCSI_IOCTL_GET_BUS_NUMBER.S +++ b/libc/sysv/consts/SCSI_IOCTL_GET_BUS_NUMBER.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_GET_BUS_NUMBER,0x5386,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_GET_BUS_NUMBER,0x5386,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_GET_IDLUN.S b/libc/sysv/consts/SCSI_IOCTL_GET_IDLUN.S index ddf20bd62..d5d88ee0b 100644 --- a/libc/sysv/consts/SCSI_IOCTL_GET_IDLUN.S +++ b/libc/sysv/consts/SCSI_IOCTL_GET_IDLUN.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_GET_IDLUN,0x5382,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_GET_IDLUN,0x5382,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_PROBE_HOST.S b/libc/sysv/consts/SCSI_IOCTL_PROBE_HOST.S index cddb4487d..70413e978 100644 --- a/libc/sysv/consts/SCSI_IOCTL_PROBE_HOST.S +++ b/libc/sysv/consts/SCSI_IOCTL_PROBE_HOST.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_PROBE_HOST,0x5385,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_PROBE_HOST,0x5385,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_SEND_COMMAND.S b/libc/sysv/consts/SCSI_IOCTL_SEND_COMMAND.S index 5c87b94e6..db838a26e 100644 --- a/libc/sysv/consts/SCSI_IOCTL_SEND_COMMAND.S +++ b/libc/sysv/consts/SCSI_IOCTL_SEND_COMMAND.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_SEND_COMMAND,1,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_SEND_COMMAND,1,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_START_UNIT.S b/libc/sysv/consts/SCSI_IOCTL_START_UNIT.S index 48f5002dd..b46792db3 100644 --- a/libc/sysv/consts/SCSI_IOCTL_START_UNIT.S +++ b/libc/sysv/consts/SCSI_IOCTL_START_UNIT.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_START_UNIT,5,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_START_UNIT,5,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_STOP_UNIT.S b/libc/sysv/consts/SCSI_IOCTL_STOP_UNIT.S index cd516713a..5ddc444d4 100644 --- a/libc/sysv/consts/SCSI_IOCTL_STOP_UNIT.S +++ b/libc/sysv/consts/SCSI_IOCTL_STOP_UNIT.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_STOP_UNIT,6,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_STOP_UNIT,6,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_SYNC.S b/libc/sysv/consts/SCSI_IOCTL_SYNC.S index bfc87375e..d73e8e343 100644 --- a/libc/sysv/consts/SCSI_IOCTL_SYNC.S +++ b/libc/sysv/consts/SCSI_IOCTL_SYNC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_SYNC,4,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_SYNC,4,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_TAGGED_DISABLE.S b/libc/sysv/consts/SCSI_IOCTL_TAGGED_DISABLE.S index 5058c16da..943f46500 100644 --- a/libc/sysv/consts/SCSI_IOCTL_TAGGED_DISABLE.S +++ b/libc/sysv/consts/SCSI_IOCTL_TAGGED_DISABLE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_TAGGED_DISABLE,0x5384,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_TAGGED_DISABLE,0x5384,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_TAGGED_ENABLE.S b/libc/sysv/consts/SCSI_IOCTL_TAGGED_ENABLE.S index a91bf1959..5c3ebf578 100644 --- a/libc/sysv/consts/SCSI_IOCTL_TAGGED_ENABLE.S +++ b/libc/sysv/consts/SCSI_IOCTL_TAGGED_ENABLE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_TAGGED_ENABLE,0x5383,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_TAGGED_ENABLE,0x5383,0,0,0,0,0 diff --git a/libc/sysv/consts/SCSI_IOCTL_TEST_UNIT_READY.S b/libc/sysv/consts/SCSI_IOCTL_TEST_UNIT_READY.S index 739647d14..eb37c0d1c 100644 --- a/libc/sysv/consts/SCSI_IOCTL_TEST_UNIT_READY.S +++ b/libc/sysv/consts/SCSI_IOCTL_TEST_UNIT_READY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,SCSI_IOCTL_TEST_UNIT_READY,2,0,0,0,0,0 +.syscon scsi,SCSI_IOCTL_TEST_UNIT_READY,2,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_APPEND.S b/libc/sysv/consts/ST_APPEND.S index 5cf26b7bc..99483b6f6 100644 --- a/libc/sysv/consts/ST_APPEND.S +++ b/libc/sysv/consts/ST_APPEND.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_APPEND,0x0100,0,0,0,0,0 +.syscon statvfs,ST_APPEND,0x0100,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_IMMUTABLE.S b/libc/sysv/consts/ST_IMMUTABLE.S index afa4f8978..7eaaf6d57 100644 --- a/libc/sysv/consts/ST_IMMUTABLE.S +++ b/libc/sysv/consts/ST_IMMUTABLE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_IMMUTABLE,0x0200,0,0,0,0,0 +.syscon statvfs,ST_IMMUTABLE,0x0200,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_MANDLOCK.S b/libc/sysv/consts/ST_MANDLOCK.S index a45c2a337..e23e806b8 100644 --- a/libc/sysv/consts/ST_MANDLOCK.S +++ b/libc/sysv/consts/ST_MANDLOCK.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_MANDLOCK,0x40,0,0,0,0,0 +.syscon statvfs,ST_MANDLOCK,0x40,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_NOATIME.S b/libc/sysv/consts/ST_NOATIME.S index 1c4a0aa96..1e4d7b62d 100644 --- a/libc/sysv/consts/ST_NOATIME.S +++ b/libc/sysv/consts/ST_NOATIME.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_NOATIME,0x0400,0,0,0,0,0 +.syscon statvfs,ST_NOATIME,0x0400,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_NODEV.S b/libc/sysv/consts/ST_NODEV.S index 5afa0a969..7b8b1e936 100644 --- a/libc/sysv/consts/ST_NODEV.S +++ b/libc/sysv/consts/ST_NODEV.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_NODEV,4,0,0,0,0,0 +.syscon statvfs,ST_NODEV,4,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_NODIRATIME.S b/libc/sysv/consts/ST_NODIRATIME.S index 25e1e82fe..6e0a3e7a7 100644 --- a/libc/sysv/consts/ST_NODIRATIME.S +++ b/libc/sysv/consts/ST_NODIRATIME.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_NODIRATIME,0x0800,0,0,0,0,0 +.syscon statvfs,ST_NODIRATIME,0x0800,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_NOEXEC.S b/libc/sysv/consts/ST_NOEXEC.S index a55d2b0bd..ac55a179d 100644 --- a/libc/sysv/consts/ST_NOEXEC.S +++ b/libc/sysv/consts/ST_NOEXEC.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_NOEXEC,8,0,0,0,0,0 +.syscon statvfs,ST_NOEXEC,8,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_NOSUID.S b/libc/sysv/consts/ST_NOSUID.S index e67715457..288a22f4f 100644 --- a/libc/sysv/consts/ST_NOSUID.S +++ b/libc/sysv/consts/ST_NOSUID.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_NOSUID,2,2,2,2,2,0 +.syscon statvfs,ST_NOSUID,2,2,2,2,2,0 diff --git a/libc/sysv/consts/ST_RDONLY.S b/libc/sysv/consts/ST_RDONLY.S index 31a3e1224..9d677d2ee 100644 --- a/libc/sysv/consts/ST_RDONLY.S +++ b/libc/sysv/consts/ST_RDONLY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_RDONLY,1,1,1,1,1,0 +.syscon statvfs,ST_RDONLY,1,1,1,1,1,0 diff --git a/libc/sysv/consts/ST_RELATIME.S b/libc/sysv/consts/ST_RELATIME.S index 91730ce25..6b075f500 100644 --- a/libc/sysv/consts/ST_RELATIME.S +++ b/libc/sysv/consts/ST_RELATIME.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_RELATIME,0x1000,0,0,0,0,0 +.syscon statvfs,ST_RELATIME,0x1000,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_SYNCHRONOUS.S b/libc/sysv/consts/ST_SYNCHRONOUS.S index 36fff28e3..3ee29b546 100644 --- a/libc/sysv/consts/ST_SYNCHRONOUS.S +++ b/libc/sysv/consts/ST_SYNCHRONOUS.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_SYNCHRONOUS,0x10,0,0,0,0,0 +.syscon statvfs,ST_SYNCHRONOUS,0x10,0,0,0,0,0 diff --git a/libc/sysv/consts/ST_WRITE.S b/libc/sysv/consts/ST_WRITE.S index f8a6f39f1..1945a52f7 100644 --- a/libc/sysv/consts/ST_WRITE.S +++ b/libc/sysv/consts/ST_WRITE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,ST_WRITE,0x80,0,0,0,0,0 +.syscon statvfs,ST_WRITE,0x80,0,0,0,0,0 diff --git a/libc/sysv/consts/TAB0.S b/libc/sysv/consts/TAB0.S index a7ef5756b..05e012fe7 100644 --- a/libc/sysv/consts/TAB0.S +++ b/libc/sysv/consts/TAB0.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,TAB0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0,0,0b0000000000000000 +.syscon termios,TAB0,0b0000000000000000,0b000000000000000000,0b000000000000000000,0,0b000000000000000000,0b0000000000000000 diff --git a/libc/sysv/consts/TAB1.S b/libc/sysv/consts/TAB1.S index 313a37ee5..5ca4f4c9f 100644 --- a/libc/sysv/consts/TAB1.S +++ b/libc/sysv/consts/TAB1.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,TAB1,0b0000100000000000,0b000000010000000000,0b000000010000000000,0,0,0b0000100000000000 +.syscon termios,TAB1,0b0000100000000000,0b000000010000000000,0b000000010000000000,0,0b000000010000000000,0b0000100000000000 diff --git a/libc/sysv/consts/TAB2.S b/libc/sysv/consts/TAB2.S index cdb741395..75b6974e5 100644 --- a/libc/sysv/consts/TAB2.S +++ b/libc/sysv/consts/TAB2.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,TAB2,0b0001000000000000,0b000000100000000000,0b000000100000000000,0,0,0b0001000000000000 +.syscon termios,TAB2,0b0001000000000000,0b000000100000000000,0b000000100000000000,0,0b000000100000000000,0b0001000000000000 diff --git a/libc/sysv/consts/TAB3.S b/libc/sysv/consts/TAB3.S index 620414746..986b1f80a 100644 --- a/libc/sysv/consts/TAB3.S +++ b/libc/sysv/consts/TAB3.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,TAB3,0b0001100000000000,0b000000000000000100,0b000000000000000100,0,0,0b0001100000000000 +.syscon termios,TAB3,0b0001100000000000,0b000000000000000100,0b000000000000000100,0,0b000000000000000100,0b0001100000000000 diff --git a/libc/sysv/consts/TCFLSH.S b/libc/sysv/consts/TCFLSH.S index af83feb9a..aa355d64d 100644 --- a/libc/sysv/consts/TCFLSH.S +++ b/libc/sysv/consts/TCFLSH.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCFLSH,0x540b,0,0,0,0,0 +.syscon termios,TCFLSH,0x540b,0,0,0,0,0 diff --git a/libc/sysv/consts/TCIFLUSH.S b/libc/sysv/consts/TCIFLUSH.S index 5acc423f2..c70852957 100644 --- a/libc/sysv/consts/TCIFLUSH.S +++ b/libc/sysv/consts/TCIFLUSH.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCIFLUSH,0,1,1,1,1,0 +.syscon termios,TCIFLUSH,0,1,1,1,1,0 diff --git a/libc/sysv/consts/TCIOFF.S b/libc/sysv/consts/TCIOFF.S index 58f8bd649..0b0543479 100644 --- a/libc/sysv/consts/TCIOFF.S +++ b/libc/sysv/consts/TCIOFF.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCIOFF,2,3,3,3,3,0 +.syscon termios,TCIOFF,2,3,3,3,3,0 diff --git a/libc/sysv/consts/TCIOFLUSH.S b/libc/sysv/consts/TCIOFLUSH.S index a62678a37..c407ecd1f 100644 --- a/libc/sysv/consts/TCIOFLUSH.S +++ b/libc/sysv/consts/TCIOFLUSH.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCIOFLUSH,2,3,3,3,3,0 +.syscon termios,TCIOFLUSH,2,3,3,3,3,0 diff --git a/libc/sysv/consts/TCION.S b/libc/sysv/consts/TCION.S index b8e5f2d72..9020fa38b 100644 --- a/libc/sysv/consts/TCION.S +++ b/libc/sysv/consts/TCION.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCION,3,4,4,4,4,0 +.syscon termios,TCION,3,4,4,4,4,0 diff --git a/libc/sysv/consts/TCOFLUSH.S b/libc/sysv/consts/TCOFLUSH.S index 717e0ad82..a7e8523a6 100644 --- a/libc/sysv/consts/TCOFLUSH.S +++ b/libc/sysv/consts/TCOFLUSH.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCOFLUSH,1,2,2,2,2,0 +.syscon termios,TCOFLUSH,1,2,2,2,2,0 diff --git a/libc/sysv/consts/TCOOFF.S b/libc/sysv/consts/TCOOFF.S index 6a35619a0..d2f98b06b 100644 --- a/libc/sysv/consts/TCOOFF.S +++ b/libc/sysv/consts/TCOOFF.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCOOFF,0,1,1,1,1,0 +.syscon termios,TCOOFF,0,1,1,1,1,0 diff --git a/libc/sysv/consts/TCOON.S b/libc/sysv/consts/TCOON.S index 01b15fc94..38c3f077f 100644 --- a/libc/sysv/consts/TCOON.S +++ b/libc/sysv/consts/TCOON.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,TCOON,1,2,2,2,2,0 +.syscon termios,TCOON,1,2,2,2,2,0 diff --git a/libc/sysv/consts/WRITE_10.S b/libc/sysv/consts/WRITE_10.S index a8a413959..951e911df 100644 --- a/libc/sysv/consts/WRITE_10.S +++ b/libc/sysv/consts/WRITE_10.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_10,42,0,0,0,0,0 +.syscon scsi,WRITE_10,42,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_12.S b/libc/sysv/consts/WRITE_12.S index 4a601fe9c..ce4375bc2 100644 --- a/libc/sysv/consts/WRITE_12.S +++ b/libc/sysv/consts/WRITE_12.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_12,170,0,0,0,0,0 +.syscon scsi,WRITE_12,170,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_6.S b/libc/sysv/consts/WRITE_6.S index ec8a380f0..63727b623 100644 --- a/libc/sysv/consts/WRITE_6.S +++ b/libc/sysv/consts/WRITE_6.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_6,10,0,0,0,0,0 +.syscon scsi,WRITE_6,10,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_BUFFER.S b/libc/sysv/consts/WRITE_BUFFER.S index 082690233..80fc29492 100644 --- a/libc/sysv/consts/WRITE_BUFFER.S +++ b/libc/sysv/consts/WRITE_BUFFER.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_BUFFER,59,0,0,0,0,0 +.syscon scsi,WRITE_BUFFER,59,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_FILEMARKS.S b/libc/sysv/consts/WRITE_FILEMARKS.S index dd55e6e69..5058a5023 100644 --- a/libc/sysv/consts/WRITE_FILEMARKS.S +++ b/libc/sysv/consts/WRITE_FILEMARKS.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_FILEMARKS,0x10,0,0,0,0,0 +.syscon scsi,WRITE_FILEMARKS,0x10,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_LONG.S b/libc/sysv/consts/WRITE_LONG.S index 429178468..5dc83904e 100644 --- a/libc/sysv/consts/WRITE_LONG.S +++ b/libc/sysv/consts/WRITE_LONG.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_LONG,63,0,0,0,0,0 +.syscon scsi,WRITE_LONG,63,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_LONG_2.S b/libc/sysv/consts/WRITE_LONG_2.S index 016c7f948..6c6cea5e8 100644 --- a/libc/sysv/consts/WRITE_LONG_2.S +++ b/libc/sysv/consts/WRITE_LONG_2.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_LONG_2,234,0,0,0,0,0 +.syscon scsi,WRITE_LONG_2,234,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_SAME.S b/libc/sysv/consts/WRITE_SAME.S index 7ed5916b9..175f4989a 100644 --- a/libc/sysv/consts/WRITE_SAME.S +++ b/libc/sysv/consts/WRITE_SAME.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_SAME,65,0,0,0,0,0 +.syscon scsi,WRITE_SAME,65,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_VERIFY.S b/libc/sysv/consts/WRITE_VERIFY.S index d06004a31..80fcc36d0 100644 --- a/libc/sysv/consts/WRITE_VERIFY.S +++ b/libc/sysv/consts/WRITE_VERIFY.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_VERIFY,46,0,0,0,0,0 +.syscon scsi,WRITE_VERIFY,46,0,0,0,0,0 diff --git a/libc/sysv/consts/WRITE_VERIFY_12.S b/libc/sysv/consts/WRITE_VERIFY_12.S index 8b350e2a2..47d70f139 100644 --- a/libc/sysv/consts/WRITE_VERIFY_12.S +++ b/libc/sysv/consts/WRITE_VERIFY_12.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon misc,WRITE_VERIFY_12,174,0,0,0,0,0 +.syscon scsi,WRITE_VERIFY_12,174,0,0,0,0,0 diff --git a/libc/sysv/consts/XCASE.S b/libc/sysv/consts/XCASE.S index 307c411e7..5e711bf41 100644 --- a/libc/sysv/consts/XCASE.S +++ b/libc/sysv/consts/XCASE.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,XCASE,0b0000000000000100,0,0,16777216,16777216,0b0000000000000100 +.syscon termios,XCASE,0b0000000000000100,0,0,16777216,0,0b0000000000000100 diff --git a/libc/sysv/consts/XTABS.S b/libc/sysv/consts/XTABS.S index f0b7db658..d6e186fef 100644 --- a/libc/sysv/consts/XTABS.S +++ b/libc/sysv/consts/XTABS.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon termios,XTABS,0b0001100000000000,0b000000000000000000,0b000000110000000000,0,0,0b0001100000000000 +.syscon termios,XTABS,0b0001100000000000,0b000000110000000000,0b000000110000000000,0,0b000000110000000000,0b0001100000000000 diff --git a/libc/sysv/consts/ai.h b/libc/sysv/consts/ai.h deleted file mode 100644 index 21b637401..000000000 --- a/libc/sysv/consts/ai.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_SYSV_CONSTS_AI_H_ -#define COSMOPOLITAN_LIBC_SYSV_CONSTS_AI_H_ -#include "libc/runtime/symbolic.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -extern const long AI_ADDRCONFIG; -extern const long AI_ALL; -extern const long AI_CANONNAME; -extern const long AI_NUMERICHOST; -extern const long AI_NUMERICSERV; -extern const long AI_PASSIVE; -extern const long AI_V4MAPPED; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ - -#define AI_ADDRCONFIG SYMBOLIC(AI_ADDRCONFIG) -#define AI_ALL SYMBOLIC(AI_ALL) -#define AI_CANONNAME LITERALLY(2) -#define AI_NUMERICHOST LITERALLY(4) -#define AI_NUMERICSERV SYMBOLIC(AI_NUMERICSERV) -#define AI_PASSIVE LITERALLY(1) -#define AI_V4MAPPED SYMBOLIC(AI_V4MAPPED) - -#endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_AI_H_ */ diff --git a/libc/sysv/consts/blk.h b/libc/sysv/consts/blk.h deleted file mode 100644 index d1159ac8a..000000000 --- a/libc/sysv/consts/blk.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_SYSV_CONSTS_BLK_H_ -#define COSMOPOLITAN_LIBC_SYSV_CONSTS_BLK_H_ -#include "libc/runtime/symbolic.h" - -#define BLK_BYTECOUNT SYMBOLIC(BLK_BYTECOUNT) -#define BLK_EOF SYMBOLIC(BLK_EOF) -#define BLK_EOR SYMBOLIC(BLK_EOR) -#define BLK_ERRORS SYMBOLIC(BLK_ERRORS) -#define BLK_RESTART SYMBOLIC(BLK_RESTART) - -#define BLKBSZGET SYMBOLIC(BLKBSZGET) -#define BLKBSZSET SYMBOLIC(BLKBSZSET) -#define BLKFLSBUF SYMBOLIC(BLKFLSBUF) -#define BLKFRAGET SYMBOLIC(BLKFRAGET) -#define BLKFRASET SYMBOLIC(BLKFRASET) -#define BLKGETSIZE SYMBOLIC(BLKGETSIZE) -#define BLKGETSIZE64 SYMBOLIC(BLKGETSIZE64) -#define BLKRAGET SYMBOLIC(BLKRAGET) -#define BLKRASET SYMBOLIC(BLKRASET) -#define BLKROGET SYMBOLIC(BLKROGET) -#define BLKROSET SYMBOLIC(BLKROSET) -#define BLKRRPART SYMBOLIC(BLKRRPART) -#define BLKSECTGET SYMBOLIC(BLKSECTGET) -#define BLKSECTSET SYMBOLIC(BLKSECTSET) -#define BLKSSZGET SYMBOLIC(BLKSSZGET) -#define BLKTYPE SYMBOLIC(BLKTYPE) - -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -extern const long BLK_BYTECOUNT; -extern const long BLK_EOF; -extern const long BLK_EOR; -extern const long BLK_ERRORS; -extern const long BLK_RESTART; - -extern const long BLKBSZGET; -extern const long BLKBSZSET; -extern const long BLKFLSBUF; -extern const long BLKFRAGET; -extern const long BLKFRASET; -extern const long BLKGETSIZE64; -extern const long BLKGETSIZE; -extern const long BLKRAGET; -extern const long BLKRASET; -extern const long BLKROGET; -extern const long BLKROSET; -extern const long BLKRRPART; -extern const long BLKSECTGET; -extern const long BLKSECTSET; -extern const long BLKSSZGET; -extern const long BLKTYPE; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_BLK_H_ */ diff --git a/libc/sysv/consts/inaddr.h b/libc/sysv/consts/inaddr.h index cf8260058..08773d6fc 100644 --- a/libc/sysv/consts/inaddr.h +++ b/libc/sysv/consts/inaddr.h @@ -1,40 +1,22 @@ #ifndef COSMOPOLITAN_LIBC_SYSV_CONSTS_INADDR_H_ #define COSMOPOLITAN_LIBC_SYSV_CONSTS_INADDR_H_ -#include "libc/runtime/symbolic.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ /** * @fileoverview Well-known Internet addresses. * These need to be hard-coded as little-endian, so htonl() is needed. */ -extern const long INADDR_ALLHOSTS_GROUP; -extern const long INADDR_ALLRTRS_GROUP; -extern const long INADDR_ANY; -extern const long INADDR_BROADCAST; -extern const long INADDR_LOOPBACK; -extern const long INADDR_MAX_LOCAL_GROUP; -extern const long INADDR_NONE; -extern const long INADDR_TESTNET1; -extern const long INADDR_TESTNET2; -extern const long INADDR_TESTNET3; -extern const long INADDR_UNSPEC_GROUP; +#define INADDR_ANY 0x00000000u /* 0.0.0.0 */ +#define INADDR_BROADCAST 0xFFFFFFFFu /* 255.255.255.255 */ +#define INADDR_NONE 0xFFFFFFFFu /* 255.255.255.255 */ +#define INADDR_LOOPBACK 0x7F000001u /* 127.0.0.1 */ +#define INADDR_TESTNET1 0xC0000200u /* 192.0.2.0/24 (RFC5737§3) */ +#define INADDR_TESTNET2 0xC6336400u /* 198.51.100.0/24 */ +#define INADDR_TESTNET3 0xCB007100u /* 203.0.113.0/24 */ -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ - -#define INADDR_ANY LITERALLY(0x00000000u) /* 0.0.0.0 */ -#define INADDR_BROADCAST LITERALLY(0xffffffffu) /* 255.255.255.255 */ -#define INADDR_NONE LITERALLY(0xffffffffu) /* 255.255.255.255 */ -#define INADDR_LOOPBACK LITERALLY(0x7f000001u) /* 127.0.0.1 */ -#define INADDR_TESTNET1 LITERALLY(0xc0000200u) /* 192.0.2.0/24 (RFC5737§3) */ -#define INADDR_TESTNET2 LITERALLY(0xc6336400u) /* 198.51.100.0/24 */ -#define INADDR_TESTNET3 LITERALLY(0xcb007100u) /* 203.0.113.0/24 */ - -#define INADDR_ALLHOSTS_GROUP SYMBOLIC(INADDR_ALLHOSTS_GROUP) -#define INADDR_ALLRTRS_GROUP SYMBOLIC(INADDR_ALLRTRS_GROUP) -#define INADDR_MAX_LOCAL_GROUP SYMBOLIC(INADDR_MAX_LOCAL_GROUP) -#define INADDR_UNSPEC_GROUP SYMBOLIC(INADDR_UNSPEC_GROUP) +#define INADDR_ALLHOSTS_GROUP 0xE0000001u +#define INADDR_ALLRTRS_GROUP 0xE0000002u +#define INADDR_MAX_LOCAL_GROUP 0xE00000FFu +#define INADDR_UNSPEC_GROUP 0xE0000000u #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_INADDR_H_ */ diff --git a/libc/sysv/consts/map.h b/libc/sysv/consts/map.h index 9748e3b3a..05d656800 100644 --- a/libc/sysv/consts/map.h +++ b/libc/sysv/consts/map.h @@ -21,7 +21,7 @@ extern const long MAP_NORESERVE; extern const long MAP_POPULATE; extern const long MAP_PRIVATE; extern const long MAP_SHARED; -extern const long MAP_TYPE; +extern const long MAP_CONCEAL; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ @@ -29,6 +29,7 @@ COSMOPOLITAN_C_END_ #define MAP_FILE 0 #define MAP_SHARED 1 #define MAP_PRIVATE 2 +#define MAP_TYPE 15 #define MAP_FIXED 16 #define MAP_32BIT SYMBOLIC(MAP_32BIT) @@ -44,7 +45,7 @@ COSMOPOLITAN_C_END_ #define MAP_NONBLOCK SYMBOLIC(MAP_NONBLOCK) #define MAP_NORESERVE SYMBOLIC(MAP_NORESERVE) #define MAP_POPULATE SYMBOLIC(MAP_POPULATE) -#define MAP_TYPE SYMBOLIC(MAP_TYPE) +#define MAP_CONCEAL SYMBOLIC(MAP_CONCEAL) #define MAP_ANON MAP_ANONYMOUS #define MAP_NOCORE MAP_CONCEAL diff --git a/libc/sysv/syscalls.sh b/libc/sysv/syscalls.sh index 09976ccae..7acd419cb 100755 --- a/libc/sysv/syscalls.sh +++ b/libc/sysv/syscalls.sh @@ -112,7 +112,7 @@ scall msgget 0x0e10e10e12103044 globl # won't polyfill for windows scall msgsnd 0x0e20e20e22104045 globl # won't polyfill for windows scall msgrcv 0x0e30e30e32105046 globl # won't polyfill for windows scall msgctl 0x1bc1291ff2102047 globl # won't polyfill for windows -scall sys_fcntl 0x05c05c05c205c048 globl hidden +scall __sys_fcntl 0x05c05c05c205c048 globl hidden scall sys_flock 0x0830830832083049 globl hidden scall sys_fsync 0x05f05f05f205f04a globl hidden scall sys_fdatasync 0x0f105f22620bb04b globl hidden # fsync() on openbsd diff --git a/libc/sysv/systemfive.S b/libc/sysv/systemfive.S index 019339aae..8d8964404 100644 --- a/libc/sysv/systemfive.S +++ b/libc/sysv/systemfive.S @@ -108,13 +108,13 @@ __systemfive: systemfive_linux: and $0xfff,%eax cmp $0xfff,%eax - je systemfive_enosys + je systemfive_enosys # never taken branches cost nothing mov %rcx,%r10 # syscall instruction clobbers %rcx push %rbp # linux never reads args from stack mov %rsp,%rbp # having frame will help backtraces syscall # this is known as a context switch pop %rbp # next we check to see if it failed - cmp $-4095,%rax # system five nexgen32e abi § A.2.1 + cmp $-4095,%rax # system five nexgen32e abi § a.2.1 jae systemfive_error # encodes errno as neg return value ret .endfn systemfive_linux,globl,hidden @@ -374,7 +374,7 @@ _init_systemfive_stack: # determinism ftw! mov %rsp,%rbp // 𝑠𝑙𝑖𝑑𝑒 _init_systemfive_syscall: -/* mov __NR_msyscall,%eax # syscall origin protect + mov __NR_msyscall,%eax # syscall origin protect cmp $0xfff,%ax # openbsd is pretty cool jae _init_systemfive_done push %rdi @@ -385,7 +385,7 @@ _init_systemfive_syscall: mov $__privileged_size,%esi syscall pop %rsi - pop %rdi*/ + pop %rdi // 𝑠𝑙𝑖𝑑𝑒 #endif /* TINY */ _init_systemfive_done: diff --git a/libc/nexgen32e/tzcnt.S b/libc/testlib/blocktronics.S similarity index 67% rename from libc/nexgen32e/tzcnt.S rename to libc/testlib/blocktronics.S index e709d02a6..54247a16a 100644 --- a/libc/nexgen32e/tzcnt.S +++ b/libc/testlib/blocktronics.S @@ -18,26 +18,16 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" .source __FILE__ +.rodata -// Finds lowest set bit in 𝑥. -// -// uint32 𝑥 bsf(𝑥) tzcnt(𝑥) ffs(𝑥) bsr(𝑥) lzcnt(𝑥) -// 0x00000000 wut 32 0 wut 32 -// 0x00000001 0 0 1 0 31 -// 0x80000001 0 0 1 31 0 -// 0x80000000 31 31 32 31 0 -// 0x00000010 4 4 5 4 27 -// 0x08000010 4 4 5 27 4 -// 0x08000000 27 27 28 27 4 -// 0xffffffff 0 0 1 31 0 -// -// @param edi is 32-bit unsigned 𝑥 value -// @return eax number in range [0,32) or 32 if 𝑥 is 0 -// @see also treasure trove of nearly identical functions -tzcnt: .leafprologue - .profilable - mov $32,%esi - bsf %edi,%eax - cmovz %esi,%eax - .leafepilogue - .endfn tzcnt,globl +// Nontrivial NUL-terminated string test vector. + .align 1 +kBlocktronics: +0: .incbin "libc/testlib/blocktronics.txt" +1: .byte 0 + .endobj kBlocktronics,globl + + .align 8 +kBlocktronicsSize: + .quad 1b-0b + .endobj kBlocktronicsSize,globl diff --git a/libc/testlib/blocktronics.h b/libc/testlib/blocktronics.h new file mode 100644 index 000000000..43fdaf036 --- /dev/null +++ b/libc/testlib/blocktronics.h @@ -0,0 +1,12 @@ +#ifndef COSMOPOLITAN_LIBC_TESTLIB_BLOCKTRONICS_H_ +#define COSMOPOLITAN_LIBC_TESTLIB_BLOCKTRONICS_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +extern size_t kBlocktronicsSize; +extern char kBlocktronics[]; +extern uint8_t kBlocktronicsZip[]; + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_TESTLIB_BLOCKTRONICS_H_ */ diff --git a/libc/testlib/blocktronics.txt b/libc/testlib/blocktronics.txt new file mode 100644 index 000000000..8f8d5613e --- /dev/null +++ b/libc/testlib/blocktronics.txt @@ -0,0 +1,32 @@ + ░▄██▒▄█ ▐██ ░░░ ▀▀████▒▀█▄ + ▐███▓██░ ██▌ ▀████▄■█▄ + ▐█▓███▀█░██▀ ░ ░▀█████▓▄ + ▐█▓██▀▄█▒██▀ ▄▄░ ▄▄▄ ░░░ ░▀████▒▄ + ▐████▀▄█■█▀ ▀▀ ░█████░ + ▐█▓█▀████▀ ░ ▐▓███▒ + █░███▀▀ ░░░ ▄█ ░░░ █████ + ▐█▓█░▀▀ ░░▄█▄▄▄▄▄ ▀▄ ▌▄▄▄░▄▄▄▄▄ ▐████░ + ▐███▌ ▄▀█████████▄ ▌▐▄████████▄ ▐▓███░ + ▐███░░░▀▄█▀▄▄████▄▀░ ▐████████▒ ▀ ░███░ + ░████░ ▓▀ ▄███████▀▌ ▀▄■████▀▀█▀ ██▀█ + ▓███░ ░▄▀▀░░░ ▀ ░░▌ ▄▀▀▄░░▀░▄▀▄ ▐██▀▄ + ░███░ ▄▓▓▄▄░▀▀█▀█ ▌░░ ▀█▀█▀▀ ▐██▀ + █▀▄▐██ ▀░░ ▄▀ ▐ █ ▀ ▄▄▄░ ░▀▄█▄▀█ + ▌▄ █▓ ▒ ░ █▄█▄▀▄▄▄███▄▀▄ ░░ ░ ▀ █▌ + █▌▄░▌ ░░░▄▀█▀███████▄▀▄▀▄▀▀▄▄▄ █▀█░▐ + ██▄ ░░░▄█▄▀██▄█■██████▄█▄█▄■▀█░ ▐░▐ + ▀██░ ░▄██████████████████▄█▄█ ░█ ░ ▄▀ + ▀▓█▄▓░░ ▒█▀█████████████████████▒ ██▀ + ▀███ ▓▒ ██████████████▀▀▀▀█▄▀ ░▄█▒ + ▀███ ▀█▄▀▄█████▀▀ ▓▓▓▄░ ▐ ░▄██ + ▀██ ▄███████▄████████▀░░ ░▄██ + ▄██▀▀▄ █▄▀▄██▒▒███████████▀▀▀▄░ ░███░ + ▄██▀▄▄░░▀▐▄████▄ █████▀▄░░█▀▄▀░░ ▄██░ + █████▄▄▄███▀░█▌██▄▀▀█████▄▄░░░▄▄███▀██▄ ▄▀▀▀▄▄ + ▀██████▀■▄█▄▄ ░▀███████████████▓▓░░▄██▀▄████▄▄▀▄ + + █▀█ █ █▀█ █▀█ █▄▀ ▐▀█▀▌█▀█ █▀█ █▄ █ ▀█▀ █▀█ █▀▀ + █▀▄ █ █ █ █ █ ▀▄ █ █▀▄ █ █ █ ▀█ █ █ ▀▀█ + █▄█ █▄▌█▄█ █▄█ █ █ █ █ █ █▄█ █ █ ▄█▄ █▄█ █▄█ + + THERE WILL BE BLOCKS march 01 2017 diff --git a/libc/testlib/testlib.mk b/libc/testlib/testlib.mk index 85ccc3c6a..676f47ff4 100644 --- a/libc/testlib/testlib.mk +++ b/libc/testlib/testlib.mk @@ -13,11 +13,13 @@ LIBC_TESTLIB_A = o/$(MODE)/libc/testlib/testlib.a LIBC_TESTLIB_A_CHECKS = $(LIBC_TESTLIB_A).pkg LIBC_TESTLIB_A_ASSETS = \ - libc/testlib/hyperion.txt + libc/testlib/hyperion.txt \ + libc/testlib/blocktronics.txt LIBC_TESTLIB_A_HDRS = \ libc/testlib/bench.h \ libc/testlib/ezbench.h \ + libc/testlib/blocktronics.h \ libc/testlib/hyperion.h \ libc/testlib/testlib.h @@ -25,6 +27,7 @@ LIBC_TESTLIB_A_SRCS_S = \ libc/testlib/bench.S \ libc/testlib/combo.S \ libc/testlib/fixture.S \ + libc/testlib/blocktronics.S \ libc/testlib/hyperion.S \ libc/testlib/testcase.S \ libc/testlib/thrashcodecache.S \ @@ -165,6 +168,7 @@ LIBC_TESTMAIN_DIRECTDEPS = \ LIBC_CALLS \ LIBC_INTRIN \ LIBC_LOG \ + LIBC_MEM \ LIBC_NEXGEN32E \ LIBC_RUNTIME \ LIBC_STDIO \ diff --git a/libc/testlib/testmain.c b/libc/testlib/testmain.c index 3220673db..8f18af359 100644 --- a/libc/testlib/testmain.c +++ b/libc/testlib/testmain.c @@ -21,6 +21,7 @@ #include "libc/calls/internal.h" #include "libc/log/log.h" #include "libc/nexgen32e/x86feature.h" +#include "libc/runtime/symbols.internal.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/ex.h" #include "libc/sysv/consts/exit.h" @@ -70,9 +71,13 @@ static testonly void GetOpts(int argc, char *argv[]) { * Generic test program main function. */ testonly int main(int argc, char *argv[]) { + const char *comdbg; __log_level = kLogInfo; GetOpts(argc, argv); showcrashreports(); + if ((comdbg = FindDebugBinary())) { + setenv("COMDBG", comdbg, true); + } g_testlib_shoulddebugbreak = IsDebuggerPresent(false); sys_getpid(); /* make strace easier to read */ testlib_clearxmmregisters(); diff --git a/libc/testlib/testrunner.c b/libc/testlib/testrunner.c index 50bf8d8aa..d7a5ca9ca 100644 --- a/libc/testlib/testrunner.c +++ b/libc/testlib/testrunner.c @@ -25,6 +25,7 @@ #include "libc/macros.internal.h" #include "libc/nt/process.h" #include "libc/runtime/runtime.h" +#include "libc/runtime/symbols.internal.h" #include "libc/stdio/stdio.h" #include "libc/testlib/testlib.h" #include "libc/x/x.h" diff --git a/libc/time/strftime.c b/libc/time/strftime.c index 5a610e6d0..fce7fb70b 100644 --- a/libc/time/strftime.c +++ b/libc/time/strftime.c @@ -19,6 +19,7 @@ #include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" +#include "libc/fmt/itoa.h" #include "libc/macros.internal.h" #include "libc/nexgen32e/nexgen32e.h" #include "libc/time/struct/tm.h" @@ -31,24 +32,24 @@ Copyright 1989 The Regents of the University of California\""); asm(".include \"libc/disclaimer.inc\""); static char *strftime_add(char *p, const char *pe, const char *str) { - while (p < pe && (*p = *str++) != '\0') ++p; + while (p < pe && (*p = *str++)) ++p; return p; } static char *strftime_conv(char *p, const char *pe, int n, const char *format) { - char buf[INT_STRLEN_MAXIMUM(int) + 1]; + char buf[22]; (snprintf)(buf, sizeof(buf), format, n); return strftime_add(p, pe, buf); } static char *strftime_secs(char *p, const char *pe, const struct tm *t) { - static char buf[INT_STRLEN_MAXIMUM(int) + 1]; + char ibuf[21]; struct tm tmp; int64_t s; tmp = *t; /* Make a copy, mktime(3) modifies the tm struct. */ s = mktime(&tmp); - (snprintf)(buf, sizeof(buf), "%ld", s); - return strftime_add(p, pe, buf); + int64toarray_radix10(s, ibuf); + return strftime_add(p, pe, ibuf); } static char *strftime_timefmt(char *p, const char *pe, const char *format, diff --git a/libc/tinymath/fsum.c b/libc/tinymath/fsum.c new file mode 100644 index 000000000..cdfdd2285 --- /dev/null +++ b/libc/tinymath/fsum.c @@ -0,0 +1,31 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/math.h" +#include "libc/str/str.h" + +/** + * Adds doubles in array. + */ +double fsum(const double *p, size_t n) { + size_t i; + double s; + if (n > 8) return fsum(p, n / 2) + fsum(p + n / 2, n - n / 2); + for (s = i = 0; i < n; ++i) s += p[i]; + return s; +} diff --git a/libc/tinymath/fsumf.c b/libc/tinymath/fsumf.c new file mode 100644 index 000000000..c0c42ba7e --- /dev/null +++ b/libc/tinymath/fsumf.c @@ -0,0 +1,31 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/math.h" +#include "libc/str/str.h" + +/** + * Adds floats in array. + */ +float fsumf(const float *p, size_t n) { + float s; + size_t i; + if (n > 8) return fsumf(p, n / 2) + fsumf(p + n / 2, n - n / 2); + for (s = i = 0; i < n; ++i) s += p[i]; + return s; +} diff --git a/libc/zip.h b/libc/zip.h index cf5c0c7e0..359a5298e 100644 --- a/libc/zip.h +++ b/libc/zip.h @@ -49,8 +49,10 @@ #define kZipCdirHdrLinkableSize \ ROUNDUP(kZipCfileHdrMinSize + PATH_MAX, kZipCdirAlign) -#define kZipCdir64HdrMagic 0x06064b50 /* PK♣♠ "PK\6\6" */ -#define kZipCdir64HdrMinSize 56 +#define kZipCdir64HdrMagic 0x06064b50 /* PK♣♠ "PK\6\6" */ +#define kZipCdir64HdrMinSize 56 +#define kZipCdir64LocatorMagic 0x07064b50 /* PK♠• "PK\6\7" */ +#define kZipCdir64LocatorSize 20 #define kZipCfileHdrMagic 0x02014b50 /* PK☺☻ "PK\1\2" */ #define kZipCfileHdrMinSize 46 @@ -112,7 +114,11 @@ #define ZIP_CDIR64_OFFSET(P) READ64LE((P) + 48) #define ZIP_CDIR64_COMMENTSIZE(P) \ (ZIP_CDIR64_HDRSIZE(P) >= 56 ? ZIP_CDIR64_HDRSIZE(P) - 56 : 0) -#define ZIP_CDIR64_COMMENT(P) ((P) + 56) /* recommend stopping at nul */ +#define ZIP_CDIR64_COMMENT(P) ((P) + 56) /* recommend stopping at nul */ +#define ZIP_LOCATE64_MAGIC(P) READ32LE(P) +#define ZIP_LOCATE64_STARTINGDISK(P) READ32LE((P) + 4) +#define ZIP_LOCATE64_OFFSET(P) READ64LE((P) + 8) +#define ZIP_LOCATE64_TOTALDISKS(P) READ32LE((P) + 12) /* central directory file header */ #define ZIP_CFILE_MAGIC(P) READ32LE(P) @@ -178,13 +184,14 @@ #define ZIP_EXTRA_CONTENT(P) ((P) + 4) #define ZIP_EXTRA_SIZE(P) (ZIP_EXTRA_CONTENTSIZE(P) + kZipExtraHdrSize) -uint8_t *GetZipCdir(const uint8_t *, size_t); +void *GetZipCdir(const uint8_t *, size_t); bool IsZipCdir32(const uint8_t *, size_t, size_t); bool IsZipCdir64(const uint8_t *, size_t, size_t); int GetZipCfileMode(const uint8_t *); uint64_t GetZipCdirOffset(const uint8_t *); uint64_t GetZipCdirRecords(const uint8_t *); void *GetZipCdirComment(const uint8_t *); +uint64_t GetZipCdirSize(const uint8_t *); uint64_t GetZipCdirCommentSize(const uint8_t *); uint64_t GetZipCfileUncompressedSize(const uint8_t *); uint64_t GetZipCfileCompressedSize(const uint8_t *); diff --git a/net/http/escapejsstringliteral.c b/net/http/escapejsstringliteral.c index ca8cf274f..a5b49d593 100644 --- a/net/http/escapejsstringliteral.c +++ b/net/http/escapejsstringliteral.c @@ -16,11 +16,23 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/bits/likely.h" #include "libc/str/thompike.h" #include "libc/str/utf16.h" #include "libc/x/x.h" #include "net/http/escape.h" +static const char kEscapeLiteral[128] = { + 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 2, 9, 4, 3, 9, 9, // 0x00 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 0x10 + 0, 0, 7, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 6, // 0x20 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, // 0x30 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, // 0x50 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, // 0x70 +}; + /** * Escapes UTF-8 data for JavaScript or JSON string literal. * @@ -66,141 +78,51 @@ char *EscapeJsStringLiteral(const char *p, size_t n, size_t *z) { } } } - switch (x) { - case ' ': - case '!': - case '#': - case '$': - case '%': - case '(': - case ')': - case '*': - case '+': - case ',': - case '-': - case '.': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case ':': - case ';': - case '?': - case '@': - case 'A': - case 'B': - case 'C': - case 'D': - case 'E': - case 'F': - case 'G': - case 'H': - case 'I': - case 'J': - case 'K': - case 'L': - case 'M': - case 'N': - case 'O': - case 'P': - case 'Q': - case 'R': - case 'S': - case 'T': - case 'U': - case 'V': - case 'W': - case 'X': - case 'Y': - case 'Z': - case '[': - case ']': - case '^': - case '_': - case '`': - case 'a': - case 'b': - case 'c': - case 'd': - case 'e': - case 'f': - case 'g': - case 'h': - case 'i': - case 'j': - case 'k': - case 'l': - case 'm': - case 'n': - case 'o': - case 'p': - case 'q': - case 'r': - case 's': - case 't': - case 'u': - case 'v': - case 'w': - case 'x': - case 'y': - case 'z': - case '{': - case '|': - case '}': - case '~': + switch (0 <= x && x <= 127 ? kEscapeLiteral[x] : 9) { + case 0: *q++ = x; break; - case '\t': + case 1: q[0] = '\\'; q[1] = 't'; q += 2; break; - case '\n': + case 2: q[0] = '\\'; q[1] = 'n'; q += 2; break; - case '\r': + case 3: q[0] = '\\'; q[1] = 'r'; q += 2; break; - case '\f': + case 4: q[0] = '\\'; q[1] = 'f'; q += 2; break; - case '\\': + case 5: q[0] = '\\'; q[1] = '\\'; q += 2; break; - case '/': + case 6: q[0] = '\\'; q[1] = '/'; q += 2; break; - case '"': + case 7: q[0] = '\\'; q[1] = '"'; q += 2; break; - case '\'': + case 8: q[0] = '\\'; q[1] = '\''; q += 2; break; - case '<': - case '>': - case '&': - case '=': - default: + case 9: w = EncodeUtf16(x); do { q[0] = '\\'; @@ -212,6 +134,8 @@ char *EscapeJsStringLiteral(const char *p, size_t n, size_t *z) { q += 6; } while ((w >>= 16)); break; + default: + unreachable; } } if (z) *z = q - r; diff --git a/net/http/geturischeme.gperf b/net/http/geturischeme.gperf deleted file mode 100644 index 8c70ccb6b..000000000 --- a/net/http/geturischeme.gperf +++ /dev/null @@ -1,24 +0,0 @@ -%{ -#include "libc/str/str.h" -#include "net/http/uri.h" -#define GPERF_DOWNCASE -%} -%compare-strncmp -%ignore-case -%language=ANSI-C -%pic -%readonly-tables -%struct-type -struct UriSchemeSlot { unsigned char name; unsigned char code; }; -%% -http,kUriSchemeHttp -https,kUriSchemeHttps -file,kUriSchemeFile -data,kUriSchemeData -zip,kUriSchemeZip -sip,kUriSchemeSip -sips,kUriSchemeSips -tel,kUriSchemeTel -ssh,kUriSchemeSsh -gs,kUriSchemeGs -s3,kUriSchemeS3 diff --git a/net/http/geturischeme.inc b/net/http/geturischeme.inc deleted file mode 100644 index b02cce378..000000000 --- a/net/http/geturischeme.inc +++ /dev/null @@ -1,213 +0,0 @@ -/* ANSI-C code produced by gperf version 3.0.4 */ -/* Command-line: gperf net/http/geturischeme.gperf */ -/* Computed positions: -k'1-2' */ - -#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ - && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ - && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \ - && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \ - && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \ - && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \ - && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \ - && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \ - && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \ - && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \ - && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \ - && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \ - && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \ - && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \ - && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \ - && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \ - && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \ - && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \ - && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \ - && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \ - && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \ - && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \ - && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126)) -/* The character set is not based on ISO-646. */ -#error "gperf generated tables don't work with this execution character set. Please report a bug to ." -#endif - -#line 1 "net/http/geturischeme.gperf" - -#include "libc/str/str.h" -#include "net/http/uri.h" -#define GPERF_DOWNCASE -#line 12 "net/http/geturischeme.gperf" -struct UriSchemeSlot { unsigned char name; unsigned char code; }; - -#define TOTAL_KEYWORDS 11 -#define MIN_WORD_LENGTH 2 -#define MAX_WORD_LENGTH 5 -#define MIN_HASH_VALUE 2 -#define MAX_HASH_VALUE 19 -/* maximum key range = 18, duplicates = 0 */ - -#ifndef GPERF_DOWNCASE -#define GPERF_DOWNCASE 1 -static unsigned char gperf_downcase[256] = - { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, - 122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, - 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255 - }; -#endif - -#ifndef GPERF_CASE_STRNCMP -#define GPERF_CASE_STRNCMP 1 -static int -gperf_case_strncmp (register const char *s1, register const char *s2, register unsigned int n) -{ - for (; n > 0;) - { - unsigned char c1 = gperf_downcase[(unsigned char)*s1++]; - unsigned char c2 = gperf_downcase[(unsigned char)*s2++]; - if (c1 != 0 && c1 == c2) - { - n--; - continue; - } - return (int)c1 - (int)c2; - } - return 0; -} -#endif - -#ifdef __GNUC__ -__inline -#else -#ifdef __cplusplus -inline -#endif -#endif -static unsigned int -hash (register const char *str, register unsigned int len) -{ - static const unsigned char asso_values[] = - { - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 5, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 10, 20, 20, 5, 15, - 5, 0, 0, 5, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 0, 0, 20, 20, 20, 20, 20, - 5, 20, 20, 20, 20, 20, 20, 10, 20, 20, - 5, 15, 5, 0, 0, 5, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 0, 0, 20, 20, 20, - 20, 20, 5, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20 - }; - return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]]; -} - -struct stringpool_t - { - char stringpool_str2[sizeof("gs")]; - char stringpool_str3[sizeof("ssh")]; - char stringpool_str4[sizeof("http")]; - char stringpool_str5[sizeof("https")]; - char stringpool_str7[sizeof("s3")]; - char stringpool_str8[sizeof("sip")]; - char stringpool_str9[sizeof("sips")]; - char stringpool_str13[sizeof("zip")]; - char stringpool_str14[sizeof("file")]; - char stringpool_str18[sizeof("tel")]; - char stringpool_str19[sizeof("data")]; - }; -static const struct stringpool_t stringpool_contents = - { - "gs", - "ssh", - "http", - "https", - "s3", - "sip", - "sips", - "zip", - "file", - "tel", - "data" - }; -#define stringpool ((const char *) &stringpool_contents) -const struct UriSchemeSlot * -in_word_set (register const char *str, register unsigned int len) -{ - static const struct UriSchemeSlot wordlist[] = - { - {-1}, {-1}, -#line 23 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str2,kUriSchemeGs}, -#line 22 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str3,kUriSchemeSsh}, -#line 14 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str4,kUriSchemeHttp}, -#line 15 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str5,kUriSchemeHttps}, - {-1}, -#line 24 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str7,kUriSchemeS3}, -#line 19 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str8,kUriSchemeSip}, -#line 20 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str9,kUriSchemeSips}, - {-1}, {-1}, {-1}, -#line 18 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str13,kUriSchemeZip}, -#line 16 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str14,kUriSchemeFile}, - {-1}, {-1}, {-1}, -#line 21 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str18,kUriSchemeTel}, -#line 17 "net/http/geturischeme.gperf" - {(int)(long)&((struct stringpool_t *)0)->stringpool_str19,kUriSchemeData} - }; - - if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) - { - register int key = hash (str, len); - - if (key <= MAX_HASH_VALUE && key >= 0) - { - register int o = wordlist[key].name; - if (o >= 0) - { - register const char *s = o + stringpool; - - if ((((unsigned char)*str ^ (unsigned char)*s) & ~32) == 0 && !gperf_case_strncmp (str, s, len) && s[len] == '\0') - return &wordlist[key]; - } - } - } - return 0; -} diff --git a/net/http/parseforwarded.c b/net/http/parseforwarded.c index b5b5cbe35..4b25697eb 100644 --- a/net/http/parseforwarded.c +++ b/net/http/parseforwarded.c @@ -26,6 +26,8 @@ * * X-Forwarded-For: 203.0.113.42:31337 * + * The port is optional and will be set to zero if absent. + * * @param s is input data * @param n if -1 implies strlen * @param ip receives ip on success if not NULL @@ -34,26 +36,32 @@ * @see RFC7239's poorly designed Forwarded header */ int ParseForwarded(const char *s, size_t n, uint32_t *ip, uint16_t *port) { + int c, t; size_t i; uint32_t x; - int c, j, t; if (n == -1) n = s ? strlen(s) : 0; - for (t = x = j = i = 0; i < n;) { - c = s[i++] & 255; - if (isdigit(c)) { - t *= 10; - t += c - '0'; - if (t > 255) return -1; - } else if (c == '.') { - x <<= 8; - x |= t; - t = 0; - ++j; - } else if (c == ':') { - x <<= 8; - x |= t; - t = 0; - if (j != 3) return -1; + if (n) { + t = x = i = 0; + do { + c = s[i++] & 255; + if (isdigit(c)) { + t *= 10; + t += c - '0'; + if (t > 255) return -1; + } else if (c == '.') { + x <<= 8; + x |= t; + t = 0; + } else if (c == ':') { + break; + } else { + return -1; + } + } while (i < n); + x <<= 8; + x |= t; + t = 0; + if (c == ':') { while (i < n) { c = s[i++] & 255; if (isdigit(c)) { @@ -64,13 +72,11 @@ int ParseForwarded(const char *s, size_t n, uint32_t *ip, uint16_t *port) { return -1; } } - if (!x || !t) return -1; - if (ip) *ip = x; - if (port) *port = t; - return 0; - } else { - return -1; } + if (ip) *ip = x; + if (port) *port = t; + return 0; + } else { + return -1; } - return -1; } diff --git a/net/http/parseurl.c b/net/http/parseurl.c index 5de1fbc07..cf5a8c0a6 100644 --- a/net/http/parseurl.c +++ b/net/http/parseurl.c @@ -24,36 +24,38 @@ #include "net/http/url.h" struct UrlParser { - int i; - int c; - const char *data; - int size; - bool isform; - bool islatin1; - bool isopaque; - char *p; - char *q; + char *p, *q; + const char *s; + unsigned c, i, n; + char isform, islatin1, isopaque; }; -static void EmitLatin1(struct UrlParser *u, int c) { - u->p[0] = 0300 | c >> 6; - u->p[1] = 0200 | c & 077; - u->p += 2; +static void EmitLatin1(char **p, int c) { + (*p)[0] = 0300 | c >> 6; + (*p)[1] = 0200 | c & 077; + *p += 2; } -static void EmitKey(struct UrlParser *u, struct UrlParams *h) { - h->p = xrealloc(h->p, ++h->n * sizeof(*h->p)); - h->p[h->n - 1].key.p = u->q; - h->p[h->n - 1].key.n = u->p - u->q; - u->q = u->p; +static bool EmitKey(struct UrlParser *u, struct UrlParams *h) { + struct UrlParam *p; + if ((p = realloc(h->p, ++h->n * sizeof(*h->p)))) { + p[h->n - 1].key.p = u->q; + p[h->n - 1].key.n = u->p - u->q; + u->q = u->p; + h->p = p; + return true; + } else { + return false; + } } static void EmitVal(struct UrlParser *u, struct UrlParams *h, bool t) { if (!t) { if (u->p > u->q || u->c != '?') { - EmitKey(u, h); - h->p[h->n - 1].val.p = NULL; - h->p[h->n - 1].val.n = 0; + if (EmitKey(u, h)) { + h->p[h->n - 1].val.p = NULL; + h->p[h->n - 1].val.n = 0; + } } } else { h->p[h->n - 1].val.p = u->q; @@ -64,9 +66,8 @@ static void EmitVal(struct UrlParser *u, struct UrlParams *h, bool t) { static void ParseEscape(struct UrlParser *u) { int a, b, c = '%'; - if (u->i + 2 <= u->size && - ((a = kHexToInt[u->data[u->i + 0] & 0xff]) != -1 && - (b = kHexToInt[u->data[u->i + 1] & 0xff]) != -1)) { + if (u->i + 2 <= u->n && ((a = kHexToInt[u->s[u->i + 0] & 255]) != -1 && + (b = kHexToInt[u->s[u->i + 1] & 255]) != -1)) { c = a << 4 | b; u->i += 2; } @@ -74,10 +75,10 @@ static void ParseEscape(struct UrlParser *u) { } static bool ParseScheme(struct UrlParser *u, struct Url *h) { - while (u->i < u->size) { - u->c = u->data[u->i++] & 0xff; + while (u->i < u->n) { + u->c = u->s[u->i++] & 255; if (u->c == '/') { - if (u->i == 1 && u->i < u->size && u->data[u->i] == '/') { + if (u->i == 1 && u->i < u->n && u->s[u->i] == '/') { ++u->i; return true; } else { @@ -88,8 +89,8 @@ static bool ParseScheme(struct UrlParser *u, struct Url *h) { h->scheme.p = u->q; h->scheme.n = u->p - u->q; u->q = u->p; - if (u->i < u->size && u->data[u->i] == '/') { - if (u->i + 1 < u->size && u->data[u->i + 1] == '/') { + if (u->i < u->n && u->s[u->i] == '/') { + if (u->i + 1 < u->n && u->s[u->i + 1] == '/') { u->i += 2; return true; } else { @@ -108,16 +109,18 @@ static bool ParseScheme(struct UrlParser *u, struct Url *h) { ParseEscape(u); return false; } else if (u->c >= 0200 && u->islatin1) { - EmitLatin1(u, u->c); + EmitLatin1(&u->p, u->c); return false; } else { *u->p++ = u->c; if (u->i == 1) { - if (!isalpha(u->c)) { + if (!(('A' <= u->c && u->c <= 'Z') || ('a' <= u->c && u->c <= 'z'))) { return false; } } else { - if (!isalnum(u->c) && u->c != '+' && u->c != '-' && u->c != '.') { + if (!(('0' <= u->c && u->c <= '9') || ('A' <= u->c && u->c <= 'Z') || + ('a' <= u->c && u->c <= 'z') || u->c == '+' || u->c == '-' || + u->c == '.')) { return false; } } @@ -127,17 +130,17 @@ static bool ParseScheme(struct UrlParser *u, struct Url *h) { } static void ParseAuthority(struct UrlParser *u, struct Url *h) { - int t = 0; + unsigned t = 1; const char *c = NULL; - while (u->i < u->size) { - u->c = u->data[u->i++] & 0xff; + while (u->i < u->n) { + u->c = u->s[u->i++] & 255; if (u->c == '/' || u->c == '#' || u->c == '?') { break; } else if (u->c == '[') { - t = -1; - } else if (u->c == ']') { t = 0; - } else if (u->c == ':' && t >= 0) { + } else if (u->c == ']') { + t = 1; + } else if (u->c == ':' && t > 0) { *u->p++ = ':'; c = u->p; ++t; @@ -148,7 +151,7 @@ static void ParseAuthority(struct UrlParser *u, struct Url *h) { h->pass.p = c; h->pass.n = u->p - c; c = NULL; - t = 0; + t = 1; } else { h->user.p = u->q; h->user.n = u->p - u->q; @@ -157,12 +160,12 @@ static void ParseAuthority(struct UrlParser *u, struct Url *h) { } else if (u->c == '%') { ParseEscape(u); } else if (u->c >= 0200 && u->islatin1) { - EmitLatin1(u, u->c); + EmitLatin1(&u->p, u->c); } else { *u->p++ = u->c; } } - if (t == 1) { + if (t == 2) { h->host.p = u->q; h->host.n = c - 1 - u->q; h->port.p = c; @@ -179,8 +182,8 @@ static void ParseAuthority(struct UrlParser *u, struct Url *h) { } static void ParsePath(struct UrlParser *u, struct UrlView *h) { - while (u->i < u->size) { - u->c = u->data[u->i++] & 0xff; + while (u->i < u->n) { + u->c = u->s[u->i++] & 255; if (u->c == '#') { break; } else if (u->c == '?' && !u->isopaque) { @@ -188,7 +191,7 @@ static void ParsePath(struct UrlParser *u, struct UrlView *h) { } else if (u->c == '%') { ParseEscape(u); } else if (u->c >= 0200 && u->islatin1) { - EmitLatin1(u, u->c); + EmitLatin1(&u->p, u->c); } else { *u->p++ = u->c; } @@ -200,9 +203,9 @@ static void ParsePath(struct UrlParser *u, struct UrlView *h) { static void ParseQuery(struct UrlParser *u, struct UrlParams *h) { bool t = false; - if (!h->p) h->p = xmalloc(0); - while (u->i < u->size) { - u->c = u->data[u->i++] & 0xff; + if (!h->p) h->p = malloc(0); + while (u->i < u->n) { + u->c = u->s[u->i++] & 255; if (u->c == '#') { break; } else if (u->c == '%') { @@ -214,13 +217,12 @@ static void ParseQuery(struct UrlParser *u, struct UrlParams *h) { t = false; } else if (u->c == '=') { if (!t) { - EmitKey(u, h); - t = true; + t = EmitKey(u, h); } else { *u->p++ = '='; } } else if (u->c >= 0200 && u->islatin1) { - EmitLatin1(u, u->c); + EmitLatin1(&u->p, u->c); } else { *u->p++ = u->c; } @@ -229,12 +231,12 @@ static void ParseQuery(struct UrlParser *u, struct UrlParams *h) { } static void ParseFragment(struct UrlParser *u, struct UrlView *h) { - while (u->i < u->size) { - u->c = u->data[u->i++] & 0xff; + while (u->i < u->n) { + u->c = u->s[u->i++] & 255; if (u->c == '%') { ParseEscape(u); } else if (u->c >= 0200 && u->islatin1) { - EmitLatin1(u, u->c); + EmitLatin1(&u->p, u->c); } else { *u->p++ = u->c; } @@ -244,25 +246,26 @@ static void ParseFragment(struct UrlParser *u, struct UrlView *h) { u->q = u->p; } -static char *ParseUrlImpl(const char *data, size_t size, struct Url *h, - bool latin1) { +static char *ParseUrlImpl(const char *s, size_t n, struct Url *h, bool latin1) { char *m; struct UrlParser u; - if (size == -1) size = data ? strlen(data) : 0; + if (n == -1) n = s ? strlen(s) : 0; u.i = 0; u.c = 0; + u.s = s; + u.n = n; u.isform = false; - u.islatin1 = latin1; u.isopaque = false; - u.data = data; - u.size = size; + u.islatin1 = latin1; memset(h, 0, sizeof(*h)); - u.q = u.p = m = xmalloc(latin1 ? u.size * 2 : u.size); - if (ParseScheme(&u, h)) ParseAuthority(&u, h); - if (u.c != '#' && u.c != '?') ParsePath(&u, &h->path); - if (u.c == '?') ParseQuery(&u, &h->params); - if (u.c == '#') ParseFragment(&u, &h->fragment); - return xrealloc(m, u.p - m); + if ((m = malloc(latin1 ? u.n * 2 : u.n))) { + u.q = u.p = m; + if (ParseScheme(&u, h)) ParseAuthority(&u, h); + if (u.c != '#' && u.c != '?') ParsePath(&u, &h->path); + if (u.c == '?') ParseQuery(&u, &h->params); + if (u.c == '#') ParseFragment(&u, &h->fragment); + } + return m; } /** @@ -277,7 +280,7 @@ static char *ParseUrlImpl(const char *data, size_t size, struct Url *h, * There's no failure condition for this routine. This is a permissive * parser. This doesn't normalize path segments like `.` or `..` so use * IsAcceptablePath() to check for those. No restrictions are imposed - * beyond that which is strictly necessary for parsing. All the data + * beyond that which is strictly necessary for parsing. All the s * that is provided will be consumed to the one of the fields. Strict * conformance is enforced on some fields more than others, like scheme, * since it's the most non-deterministically defined field of them all. @@ -287,18 +290,18 @@ static char *ParseUrlImpl(const char *data, size_t size, struct Url *h, * for the things we won't do, like tokenizing path segments into an * array and then nesting another array beneath each of those for * storing semicolon parameters. So this parser won't make SIP easy. - * What it can do is parse HTTP URLs and most URIs like data:opaque, + * What it can do is parse HTTP URLs and most URIs like s:opaque, * better in fact than most things which claim to be URI parsers. * - * @param data is value like `/hi?x=y&z` or `http://a.example/hi#x` - * @param size is byte length and -1 implies strlen + * @param s is value like `/hi?x=y&z` or `http://a.example/hi#x` + * @param n is byte length and -1 implies strlen * @param h is assumed to be uninitialized * @return memory backing UrlView needing free (and h.params.p too) * @see URI Generic Syntax RFC3986 RFC2396 * @see EncodeUrl() */ -char *ParseUrl(const char *data, size_t size, struct Url *h) { - return ParseUrlImpl(data, size, h, false); +char *ParseUrl(const char *s, size_t n, struct Url *h) { + return ParseUrlImpl(s, n, h, false); } /** @@ -315,13 +318,13 @@ char *ParseUrl(const char *data, size_t size, struct Url *h) { * necessary for parsing. This doesn't normalize path segments like `.` * or `..`. Use IsAcceptablePath() to check for those. * - * @param data is value like `/hi?x=y&z` or `http://a.example/hi#x` - * @param size is byte length and -1 implies strlen + * @param s is value like `/hi?x=y&z` or `http://a.example/hi#x` + * @param n is byte length and -1 implies strlen * @param h is assumed to be uninitialized * @return memory backing UrlView needing free (and h.params.p too) */ -char *ParseRequestUri(const char *data, size_t size, struct Url *h) { - return ParseUrlImpl(data, size, h, true); +char *ParseRequestUri(const char *s, size_t n, struct Url *h) { + return ParseUrlImpl(s, n, h, true); } /** @@ -339,24 +342,26 @@ char *ParseRequestUri(const char *data, size_t size, struct Url *h) { * parser that doesn't impose character restrictions beyond what is * necessary for parsing. * - * @param data is value like `foo=bar&x=y&z` - * @param size is byte length and -1 implies strlen + * @param s is value like `foo=bar&x=y&z` + * @param n is byte length and -1 implies strlen * @param h must be zeroed by caller and this appends if reused - * @return UrlView memory with same size needing free (h.p needs free too) + * @return UrlView memory with same n needing free (h.p needs free too) */ -char *ParseParams(const char *data, size_t size, struct UrlParams *h) { +char *ParseParams(const char *s, size_t n, struct UrlParams *h) { char *m; struct UrlParser u; - if (size == -1) size = data ? strlen(data) : 0; + if (n == -1) n = s ? strlen(s) : 0; u.i = 0; + u.s = s; + u.n = n; u.c = '?'; u.isform = true; u.islatin1 = false; u.isopaque = false; - u.data = data; - u.size = size; - u.q = u.p = m = xmalloc(u.size); - ParseQuery(&u, h); + if ((m = malloc(u.n))) { + u.q = u.p = m; + ParseQuery(&u, h); + } return m; } @@ -379,23 +384,25 @@ char *ParseParams(const char *data, size_t size, struct UrlParams *h) { * called conditionally after ParseRequestUri() if the host is absent. * Fields unrelated to authority won't be impacted by this function. * - * @param data is value like `127.0.0.1` or `foo.example:80` - * @param size is byte length and -1 implies strlen + * @param s is value like `127.0.0.1` or `foo.example:80` + * @param n is byte length and -1 implies strlen * @param h is needs to be initialized by caller * @return memory backing UrlView needing free */ -char *ParseHost(const char *data, size_t size, struct Url *h) { +char *ParseHost(const char *s, size_t n, struct Url *h) { char *m; struct UrlParser u; - if (size == -1) size = data ? strlen(data) : 0; + if (n == -1) n = s ? strlen(s) : 0; u.i = 0; u.c = 0; + u.s = s; + u.n = n; u.isform = false; u.islatin1 = true; u.isopaque = false; - u.data = data; - u.size = size; - u.q = u.p = m = xmalloc(u.size * 2); - ParseAuthority(&u, h); - return xrealloc(m, u.p - m); + if ((m = malloc(u.n * 2))) { + u.q = u.p = m; + ParseAuthority(&u, h); + } + return m; } diff --git a/test/libc/calls/sigaction_test.c b/test/libc/calls/sigaction_test.c index e96615d16..1d9d7c8e2 100644 --- a/test/libc/calls/sigaction_test.c +++ b/test/libc/calls/sigaction_test.c @@ -103,10 +103,14 @@ void OnFpe(int sig, struct siginfo *si, struct ucontext *ctx) { ctx->uc_mcontext.rdx = 0; } -TEST(sigaction, sigFpe_handlerCanEditProcessStateAndRecoverExecution) { +noubsan void ubsanTrumpsSystemsEngineering(void) { struct sigaction saint = {.sa_sigaction = OnFpe, .sa_flags = SA_SIGINFO}; EXPECT_NE(-1, sigaction(SIGFPE, &saint, &oldsa)); volatile long x = 0; EXPECT_EQ(42, 666 / x); /* systems engineering trumps math */ EXPECT_NE(-1, sigaction(SIGFPE, &oldsa, NULL)); } + +TEST(sigaction, sigFpe_handlerCanEditProcessStateAndRecoverExecution) { + ubsanTrumpsSystemsEngineering(); +} diff --git a/test/libc/calls/writev_test.c b/test/libc/calls/writev_test.c index 0c712743a..ac7c99980 100644 --- a/test/libc/calls/writev_test.c +++ b/test/libc/calls/writev_test.c @@ -18,9 +18,12 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/calls/struct/iovec.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/macros.internal.h" +#include "libc/runtime/runtime.h" #include "libc/sock/sock.h" +#include "libc/sysv/consts/auxv.h" #include "libc/sysv/consts/o.h" #include "libc/testlib/testlib.h" @@ -59,6 +62,29 @@ TEST(writev, big_fullCompletion) { EXPECT_NE(-1, close(fd)); } +TEST(writev, asanError_efaults) { + if (!IsAsan()) return; + void *malloc_(size_t) asm("malloc"); + void free_(void *) asm("free"); + void *p; + int fd; + p = malloc_(32); + EXPECT_NE(-1, (fd = open("asan", O_RDWR | O_CREAT | O_TRUNC, 0644))); + EXPECT_EQ(32, write(fd, p, 32)); + EXPECT_NE(-1, lseek(fd, 0, SEEK_SET)); + EXPECT_EQ(32, read(fd, p, 32)); + EXPECT_EQ(-1, write(fd, p, 33)); + EXPECT_EQ(EFAULT, errno); + EXPECT_EQ(-1, write(fd, p, -1)); + EXPECT_EQ(EFAULT, errno); + free_(p); + EXPECT_EQ(-1, write(fd, p, 32)); + EXPECT_EQ(EFAULT, errno); + EXPECT_EQ(-1, read(fd, p, 32)); + EXPECT_EQ(EFAULT, errno); + close(fd); +} + TEST(writev, empty_stillPerformsIoOperation) { int fd; struct iovec iov[] = {{"", 0}, {NULL, 0}}; diff --git a/test/libc/dns/comparednsnames_test.c b/test/libc/dns/comparednsnames_test.c new file mode 100644 index 000000000..2a0d8aadc --- /dev/null +++ b/test/libc/dns/comparednsnames_test.c @@ -0,0 +1,105 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dns/dns.h" +#include "libc/mem/mem.h" +#include "libc/str/str.h" +#include "libc/testlib/testlib.h" + +TEST(CompareDnsNames, testEmpty) { + char *A = strcpy(malloc(1), ""); + char *B = strcpy(malloc(1), ""); + EXPECT_EQ(CompareDnsNames(A, B), 0); + EXPECT_EQ(CompareDnsNames(A, A), 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testDotless_caseInsensitiveBehavior) { + char *A = malloc(2); + char *B = malloc(2); + EXPECT_EQ(CompareDnsNames(strcpy(A, "a"), strcpy(B, "a")), 0); + EXPECT_EQ(CompareDnsNames(A, A), 0); + EXPECT_EQ(CompareDnsNames(strcpy(A, "a"), strcpy(B, "A")), 0); + EXPECT_EQ(CompareDnsNames(strcpy(A, "A"), strcpy(B, "a")), 0); + EXPECT_LT(CompareDnsNames(strcpy(A, "a"), strcpy(B, "b")), 0); + EXPECT_LT(CompareDnsNames(strcpy(A, "a"), strcpy(B, "B")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "d"), strcpy(B, "a")), 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testMultiLabel_lexiReverse) { + char *A = malloc(16); + char *B = malloc(16); + EXPECT_EQ(CompareDnsNames(strcpy(A, "a.example"), strcpy(B, "a.example")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "b.example"), strcpy(B, "a.example")), 0); + EXPECT_LT(CompareDnsNames(strcpy(A, "b.example"), strcpy(B, "a.examplz")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "a.zxample"), strcpy(B, "a.examplz")), 0); + EXPECT_EQ(CompareDnsNames(strcpy(A, "c.a.example"), strcpy(B, "c.a.example")), + 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "d.a.example"), strcpy(B, "c.a.example")), + 0); + EXPECT_LT(CompareDnsNames(strcpy(A, "cat.example"), strcpy(B, "lol.example")), + 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testTldDotQualifier_canBeEqualToDottedNames) { + char *A = malloc(16); + char *B = malloc(16); + EXPECT_EQ( + CompareDnsNames(strcpy(B, "aaa.example."), strcpy(A, "aaa.example")), 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testFullyQualified_alwaysComesFirst) { + char *A = malloc(16); + char *B = malloc(16); + EXPECT_LT(CompareDnsNames(strcpy(B, "aaa.example."), strcpy(A, "zzz")), 0); + EXPECT_LT(CompareDnsNames(strcpy(B, "zzz.example."), strcpy(A, "aaa")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "zzz"), strcpy(B, "aaa.example.")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "aaa"), strcpy(B, "zzz.example.")), 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testLikelySld_alwaysComesBeforeLocalName) { + char *A = malloc(16); + char *B = malloc(16); + EXPECT_LT(CompareDnsNames(strcpy(B, "z.e"), strcpy(A, "a")), 0); + EXPECT_LT(CompareDnsNames(strcpy(B, "aaa.example"), strcpy(A, "zzz")), 0); + EXPECT_LT(CompareDnsNames(strcpy(B, "zzz.example"), strcpy(A, "aaa")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "zzz"), strcpy(B, "aaa.example")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "aaa"), strcpy(B, "zzz.example")), 0); + free(B); + free(A); +} + +TEST(CompareDnsNames, testLikelySubdomain_alwaysComesAfterSld) { + char *A = malloc(16); + char *B = malloc(16); + EXPECT_LT(CompareDnsNames(strcpy(B, "a.e"), strcpy(A, "z.a.e")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "z.a.e"), strcpy(B, "a.e")), 0); + EXPECT_LT(CompareDnsNames(strcpy(B, "b.e"), strcpy(A, "a.b.e")), 0); + EXPECT_GT(CompareDnsNames(strcpy(A, "a.b.e"), strcpy(B, "b.e")), 0); + free(B); + free(A); +} diff --git a/test/libc/dns/dnsheader_test.c b/test/libc/dns/dnsheader_test.c index 380a86293..d8617cd8d 100644 --- a/test/libc/dns/dnsheader_test.c +++ b/test/libc/dns/dnsheader_test.c @@ -25,25 +25,23 @@ #include "libc/str/str.h" #include "libc/testlib/testlib.h" -TEST(serializednsheader, test) { +TEST(SerializeDnsHeader, test) { + uint8_t buf[12]; struct DnsHeader header; memset(&header, 0, sizeof(header)); header.id = 255; header.bf1 = true; header.qdcount = 1; - uint8_t *buf = malloc(12); - ASSERT_EQ(12, serializednsheader(buf, 12, header)); + SerializeDnsHeader(buf, &header); EXPECT_BINEQ(u" λ☺  ☺      ", buf); - free(buf); } -TEST(serializednsheader, fuzzSymmetry) { - uint8_t *buf; - struct DnsHeader *in, *out; - buf = gc(malloc(12)); - in = rngset(gc(malloc(sizeof(struct DnsHeader))), 12, rand64, -1); - out = rngset(gc(malloc(sizeof(struct DnsHeader))), 12, rand64, -1); - ASSERT_EQ(12, serializednsheader(buf, 12, *in)); - ASSERT_EQ(12, deserializednsheader(out, buf, 12)); - ASSERT_EQ(0, memcmp(in, out, 12), "%#.*s\n\t%#.*s", 12, in, 12, buf); +TEST(SerializeDnsHeader, fuzzSymmetry) { + uint8_t buf[12]; + struct DnsHeader in, out; + rngset(&in, sizeof(in), rand64, -1); + rngset(&out, sizeof(out), rand64, -1); + SerializeDnsHeader(buf, &in); + DeserializeDnsHeader(&out, buf); + ASSERT_EQ(0, memcmp(&in, &out, 12), "%#.*s\n\t%#.*s", 12, in, 12, buf); } diff --git a/test/libc/dns/dnsnamecmp_test.c b/test/libc/dns/dnsnamecmp_test.c deleted file mode 100644 index 55ba2b485..000000000 --- a/test/libc/dns/dnsnamecmp_test.c +++ /dev/null @@ -1,101 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dns/dns.h" -#include "libc/mem/mem.h" -#include "libc/str/str.h" -#include "libc/testlib/testlib.h" - -TEST(dnsnamecmp, testEmpty) { - char *A = strcpy(malloc(1), ""); - char *B = strcpy(malloc(1), ""); - EXPECT_EQ(dnsnamecmp(A, B), 0); - EXPECT_EQ(dnsnamecmp(A, A), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testDotless_caseInsensitiveBehavior) { - char *A = malloc(2); - char *B = malloc(2); - EXPECT_EQ(dnsnamecmp(strcpy(A, "a"), strcpy(B, "a")), 0); - EXPECT_EQ(dnsnamecmp(A, A), 0); - EXPECT_EQ(dnsnamecmp(strcpy(A, "a"), strcpy(B, "A")), 0); - EXPECT_EQ(dnsnamecmp(strcpy(A, "A"), strcpy(B, "a")), 0); - EXPECT_LT(dnsnamecmp(strcpy(A, "a"), strcpy(B, "b")), 0); - EXPECT_LT(dnsnamecmp(strcpy(A, "a"), strcpy(B, "B")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "d"), strcpy(B, "a")), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testMultiLabel_lexiReverse) { - char *A = malloc(16); - char *B = malloc(16); - EXPECT_EQ(dnsnamecmp(strcpy(A, "a.example"), strcpy(B, "a.example")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "b.example"), strcpy(B, "a.example")), 0); - EXPECT_LT(dnsnamecmp(strcpy(A, "b.example"), strcpy(B, "a.examplz")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "a.zxample"), strcpy(B, "a.examplz")), 0); - EXPECT_EQ(dnsnamecmp(strcpy(A, "c.a.example"), strcpy(B, "c.a.example")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "d.a.example"), strcpy(B, "c.a.example")), 0); - EXPECT_LT(dnsnamecmp(strcpy(A, "cat.example"), strcpy(B, "lol.example")), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testTldDotQualifier_canBeEqualToDottedNames) { - char *A = malloc(16); - char *B = malloc(16); - EXPECT_EQ(dnsnamecmp(strcpy(B, "aaa.example."), strcpy(A, "aaa.example")), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testFullyQualified_alwaysComesFirst) { - char *A = malloc(16); - char *B = malloc(16); - EXPECT_LT(dnsnamecmp(strcpy(B, "aaa.example."), strcpy(A, "zzz")), 0); - EXPECT_LT(dnsnamecmp(strcpy(B, "zzz.example."), strcpy(A, "aaa")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "zzz"), strcpy(B, "aaa.example.")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "aaa"), strcpy(B, "zzz.example.")), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testLikelySld_alwaysComesBeforeLocalName) { - char *A = malloc(16); - char *B = malloc(16); - EXPECT_LT(dnsnamecmp(strcpy(B, "z.e"), strcpy(A, "a")), 0); - EXPECT_LT(dnsnamecmp(strcpy(B, "aaa.example"), strcpy(A, "zzz")), 0); - EXPECT_LT(dnsnamecmp(strcpy(B, "zzz.example"), strcpy(A, "aaa")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "zzz"), strcpy(B, "aaa.example")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "aaa"), strcpy(B, "zzz.example")), 0); - free(B); - free(A); -} - -TEST(dnsnamecmp, testLikelySubdomain_alwaysComesAfterSld) { - char *A = malloc(16); - char *B = malloc(16); - EXPECT_LT(dnsnamecmp(strcpy(B, "a.e"), strcpy(A, "z.a.e")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "z.a.e"), strcpy(B, "a.e")), 0); - EXPECT_LT(dnsnamecmp(strcpy(B, "b.e"), strcpy(A, "a.b.e")), 0); - EXPECT_GT(dnsnamecmp(strcpy(A, "a.b.e"), strcpy(B, "b.e")), 0); - free(B); - free(A); -} diff --git a/test/libc/dns/dnsquestion_test.c b/test/libc/dns/dnsquestion_test.c index 0fb1078d0..b745c9a7a 100644 --- a/test/libc/dns/dnsquestion_test.c +++ b/test/libc/dns/dnsquestion_test.c @@ -22,29 +22,25 @@ #include "libc/mem/mem.h" #include "libc/testlib/testlib.h" -TEST(serializednsquestion, test) { - uint8_t *buf = malloc(1 + 3 + 1 + 3 + 1 + 4); - char *name = strdup("foo.bar"); +TEST(SerializeDnsQuestion, test) { struct DnsQuestion dq; + char name[] = "foo.bar"; + uint8_t buf[1 + 3 + 1 + 3 + 1 + 4]; dq.qname = name; dq.qtype = 0x0201; dq.qclass = 0x0102; EXPECT_EQ(1 + 3 + 1 + 3 + 1 + 4, - serializednsquestion(buf, 1 + 3 + 1 + 3 + 1 + 4, dq)); + SerializeDnsQuestion(buf, 1 + 3 + 1 + 3 + 1 + 4, &dq)); EXPECT_BINEQ(u"♥foo♥bar ☻☺☺☻", buf); - free(name); - free(buf); } -TEST(serializednsquestion, testNoSpace) { - uint8_t *buf = malloc(1 + 3 + 1 + 3 + 1 + 3); - char *name = strdup("foo.bar"); +TEST(SerializeDnsQuestion, testNoSpace) { struct DnsQuestion dq; + char name[] = "foo.bar"; + uint8_t buf[1 + 3 + 1 + 3 + 1 + 3]; dq.qname = name; dq.qtype = 0x0201; dq.qclass = 0x0102; - EXPECT_EQ(-1, serializednsquestion(buf, 1 + 3 + 1 + 3 + 1 + 3, dq)); + EXPECT_EQ(-1, SerializeDnsQuestion(buf, 1 + 3 + 1 + 3 + 1 + 3, &dq)); EXPECT_EQ(ENOSPC, errno); - free(name); - free(buf); } diff --git a/test/libc/dns/parsehoststxt_test.c b/test/libc/dns/parsehoststxt_test.c index a0649c957..8d8adb9b9 100644 --- a/test/libc/dns/parsehoststxt_test.c +++ b/test/libc/dns/parsehoststxt_test.c @@ -29,16 +29,16 @@ static const char *ParseIp(unsigned char ip[4]) { return inet_ntop(AF_INET, ip, g_ipbuf, sizeof(g_ipbuf)); } -TEST(parsehoststxt, testEmpty) { +TEST(ParseHostsTxt, testEmpty) { struct HostsTxt *ht = calloc(1, sizeof(struct HostsTxt)); FILE *f = fmemopen(NULL, BUFSIZ, "r+"); - ASSERT_EQ(0, parsehoststxt(ht, f)); + ASSERT_EQ(0, ParseHostsTxt(ht, f)); ASSERT_EQ(0, ht->entries.i); - freehoststxt(&ht); + FreeHostsTxt(&ht); fclose(f); } -TEST(parsehoststxt, testCorrectlyTokenizesAndSorts) { +TEST(ParseHostsTxt, testCorrectlyTokenizesAndSorts) { const char kInput[] = "# this is a comment\n" "# IP HOST1 HOST2\n" "203.0.113.1 lol.example. lol\n" @@ -47,8 +47,8 @@ TEST(parsehoststxt, testCorrectlyTokenizesAndSorts) { FILE *f = fmemopen(NULL, BUFSIZ, "r+"); ASSERT_EQ(1, fwrite(kInput, strlen(kInput), 1, f)); rewind(f); - ASSERT_EQ(0, parsehoststxt(ht, f)); - sorthoststxt(ht); + ASSERT_EQ(0, ParseHostsTxt(ht, f)); + SortHostsTxt(ht); ASSERT_EQ(4, ht->entries.i); EXPECT_STREQ("cat.example.", &ht->strings.p[ht->entries.p[0].name]); EXPECT_STREQ("cat.example.", &ht->strings.p[ht->entries.p[0].canon]); @@ -62,20 +62,20 @@ TEST(parsehoststxt, testCorrectlyTokenizesAndSorts) { EXPECT_STREQ("lol", &ht->strings.p[ht->entries.p[3].name]); EXPECT_STREQ("lol.example.", &ht->strings.p[ht->entries.p[3].canon]); EXPECT_STREQ("203.0.113.1", ParseIp(ht->entries.p[3].ip)); - freehoststxt(&ht); + FreeHostsTxt(&ht); fclose(f); } -TEST(parsehoststxt, testIpv6_isIgnored) { +TEST(ParseHostsTxt, testIpv6_isIgnored) { const char kInput[] = "::1 boop\n" "203.0.113.2 cat # ignore me\n"; struct HostsTxt *ht = calloc(1, sizeof(struct HostsTxt)); FILE *f = fmemopen(kInput, strlen(kInput), "r+"); - ASSERT_EQ(0, parsehoststxt(ht, f)); + ASSERT_EQ(0, ParseHostsTxt(ht, f)); ASSERT_EQ(1, ht->entries.i); EXPECT_STREQ("cat", &ht->strings.p[ht->entries.p[0].name]); EXPECT_STREQ("cat", &ht->strings.p[ht->entries.p[0].canon]); EXPECT_STREQ("203.0.113.2", ParseIp(ht->entries.p[0].ip)); - freehoststxt(&ht); + FreeHostsTxt(&ht); fclose(f); } diff --git a/test/libc/dns/parseresolvconf_test.c b/test/libc/dns/parseresolvconf_test.c index d4a017bc6..b47333601 100644 --- a/test/libc/dns/parseresolvconf_test.c +++ b/test/libc/dns/parseresolvconf_test.c @@ -31,22 +31,22 @@ static const char *FormatIp(struct sockaddr_in *ip) { return inet_ntop(ip->sin_family, &ip->sin_addr.s_addr, g_ipbuf, 16); } -TEST(parseresolvconf, testEmpty) { +TEST(ParseResolvConf, testEmpty) { struct ResolvConf *rv = calloc(1, sizeof(struct ResolvConf)); FILE *f = fmemopen(NULL, BUFSIZ, "r+"); - ASSERT_EQ(0, parseresolvconf(rv, f)); + ASSERT_EQ(0, ParseResolvConf(rv, f)); ASSERT_EQ(0, rv->nameservers.i); - freeresolvconf(&rv); + FreeResolvConf(&rv); fclose(f); } -TEST(parseresolvconf, testCorrectlyTokenizes) { +TEST(ParseResolvConf, testCorrectlyTokenizes) { const char kInput[] = "# this is a comment\n" "nameserver 203.0.113.2 \n" " nameserver 203.0.113.1\n"; struct ResolvConf *rv = calloc(1, sizeof(struct ResolvConf)); FILE *f = fmemopen(kInput, strlen(kInput), "r+"); - ASSERT_EQ(2, parseresolvconf(rv, f)); + ASSERT_EQ(2, ParseResolvConf(rv, f)); ASSERT_EQ(2, rv->nameservers.i); EXPECT_EQ(AF_INET, rv->nameservers.p[0].sin_family); EXPECT_EQ(DNS_PORT, ntohs(rv->nameservers.p[0].sin_port)); @@ -54,17 +54,17 @@ TEST(parseresolvconf, testCorrectlyTokenizes) { EXPECT_EQ(AF_INET, rv->nameservers.p[1].sin_family); EXPECT_EQ(DNS_PORT, ntohs(rv->nameservers.p[1].sin_port)); EXPECT_STREQ("203.0.113.1", FormatIp(&rv->nameservers.p[1])); - freeresolvconf(&rv); + FreeResolvConf(&rv); fclose(f); } -TEST(parseresolvconf, testMulticastDnsThing_getsIgnored) { +TEST(ParseResolvConf, testMulticastDnsThing_getsIgnored) { const char kInput[] = "search local # boop\n"; struct ResolvConf *rv = calloc(1, sizeof(struct ResolvConf)); FILE *f = fmemopen(NULL, BUFSIZ, "r+"); ASSERT_EQ(strlen(kInput), fwrite(kInput, 1, strlen(kInput), f)); - ASSERT_EQ(0, parseresolvconf(rv, f)); + ASSERT_EQ(0, ParseResolvConf(rv, f)); ASSERT_EQ(0, rv->nameservers.i); - freeresolvconf(&rv); + FreeResolvConf(&rv); fclose(f); } diff --git a/test/libc/dns/pascalifydnsname_test.c b/test/libc/dns/pascalifydnsname_test.c index 2b372ec45..b57d40640 100644 --- a/test/libc/dns/pascalifydnsname_test.c +++ b/test/libc/dns/pascalifydnsname_test.c @@ -22,57 +22,57 @@ #include "libc/str/str.h" #include "libc/testlib/testlib.h" -TEST(pascalifydnsname, testEmpty) { +TEST(PascalifyDnsName, testEmpty) { uint8_t *buf = malloc(1); char *name = strdup(""); - EXPECT_EQ(0, pascalifydnsname(buf, 1, name)); + EXPECT_EQ(0, PascalifyDnsName(buf, 1, name)); EXPECT_BINEQ(u" ", buf); free(name); free(buf); } -TEST(pascalifydnsname, testOneLabel) { +TEST(PascalifyDnsName, testOneLabel) { uint8_t *buf = malloc(1 + 3 + 1); char *name = strdup("foo"); - EXPECT_EQ(1 + 3, pascalifydnsname(buf, 1 + 3 + 1, name)); + EXPECT_EQ(1 + 3, PascalifyDnsName(buf, 1 + 3 + 1, name)); EXPECT_BINEQ(u"♥foo ", buf); free(name); free(buf); } -TEST(pascalifydnsname, testTwoLabels) { +TEST(PascalifyDnsName, testTwoLabels) { uint8_t *buf = malloc(1 + 3 + 1 + 3 + 1); char *name = strdup("foo.bar"); - EXPECT_EQ(1 + 3 + 1 + 3, pascalifydnsname(buf, 1 + 3 + 1 + 3 + 1, name)); + EXPECT_EQ(1 + 3 + 1 + 3, PascalifyDnsName(buf, 1 + 3 + 1 + 3 + 1, name)); EXPECT_BINEQ(u"♥foo♥bar ", buf); free(name); free(buf); } -TEST(pascalifydnsname, testFqdnDot_isntIncluded) { +TEST(PascalifyDnsName, testFqdnDot_isntIncluded) { uint8_t *buf = malloc(1 + 3 + 1 + 3 + 1); char *name = strdup("foo.bar."); - EXPECT_EQ(1 + 3 + 1 + 3, pascalifydnsname(buf, 1 + 3 + 1 + 3 + 1, name)); + EXPECT_EQ(1 + 3 + 1 + 3, PascalifyDnsName(buf, 1 + 3 + 1 + 3 + 1, name)); EXPECT_BINEQ(u"♥foo♥bar ", buf); free(name); free(buf); } -TEST(pascalifydnsname, testTooLong) { +TEST(PascalifyDnsName, testTooLong) { uint8_t *buf = malloc(1); char *name = malloc(1000); memset(name, '.', 999); name[999] = '\0'; - EXPECT_EQ(-1, pascalifydnsname(buf, 1, name)); + EXPECT_EQ(-1, PascalifyDnsName(buf, 1, name)); EXPECT_EQ(ENAMETOOLONG, errno); free(name); free(buf); } -TEST(pascalifydnsname, testNoSpace) { +TEST(PascalifyDnsName, testNoSpace) { uint8_t *buf = malloc(1); char *name = strdup("foo"); - EXPECT_EQ(-1, pascalifydnsname(buf, 1, name)); + EXPECT_EQ(-1, PascalifyDnsName(buf, 1, name)); EXPECT_EQ(ENOSPC, errno); free(name); free(buf); diff --git a/test/libc/dns/resolvehoststxt_test.c b/test/libc/dns/resolvehoststxt_test.c index 3f006d09b..758ee4970 100644 --- a/test/libc/dns/resolvehoststxt_test.c +++ b/test/libc/dns/resolvehoststxt_test.c @@ -27,7 +27,7 @@ static const char *EzIp4Lookup(const struct HostsTxt *ht, const char *name) { struct sockaddr_in addr4; - if (resolvehoststxt(ht, AF_INET, name, (void *)&addr4, + if (ResolveHostsTxt(ht, AF_INET, name, (void *)&addr4, sizeof(struct sockaddr_in), NULL) > 0) { static char g_ipbuf[16]; return inet_ntop(AF_INET, &addr4.sin_addr, g_ipbuf, sizeof(g_ipbuf)); @@ -38,18 +38,18 @@ static const char *EzIp4Lookup(const struct HostsTxt *ht, const char *name) { static const char *EzCanonicalize(const struct HostsTxt *ht, const char *name) { const char *res; - return resolvehoststxt(ht, AF_INET, name, NULL, 0, &res) > 0 ? res : NULL; + return ResolveHostsTxt(ht, AF_INET, name, NULL, 0, &res) > 0 ? res : NULL; } static const char kInput[] = "127.0.0.1 localhost\n" "203.0.113.1 lol.example. lol\n" "203.0.113.2 cat.example. cat\n"; -TEST(resolvehoststxt, testBasicLookups) { +TEST(ResolveHostsTxt, testBasicLookups) { struct HostsTxt *ht = calloc(1, sizeof(struct HostsTxt)); FILE *f = fmemopen(kInput, strlen(kInput), "r+"); - ASSERT_EQ(0, parsehoststxt(ht, f)); - sorthoststxt(ht); + ASSERT_EQ(0, ParseHostsTxt(ht, f)); + SortHostsTxt(ht); ASSERT_EQ(5, ht->entries.i); EXPECT_STREQ("127.0.0.1", EzIp4Lookup(ht, "localhost")); EXPECT_STREQ("203.0.113.1", EzIp4Lookup(ht, "lol")); @@ -58,15 +58,15 @@ TEST(resolvehoststxt, testBasicLookups) { EXPECT_STREQ("203.0.113.2", EzIp4Lookup(ht, "cat")); EXPECT_STREQ("203.0.113.2", EzIp4Lookup(ht, "cat.example.")); EXPECT_EQ(NULL, EzIp4Lookup(ht, "boop")); - freehoststxt(&ht); + FreeHostsTxt(&ht); fclose(f); } -TEST(resolvehoststxt, testCanonicalize) { +TEST(ResolveHostsTxt, testCanonicalize) { struct HostsTxt *ht = calloc(1, sizeof(struct HostsTxt)); FILE *f = fmemopen(kInput, strlen(kInput), "r+"); - ASSERT_EQ(0, parsehoststxt(ht, f)); - sorthoststxt(ht); + ASSERT_EQ(0, ParseHostsTxt(ht, f)); + SortHostsTxt(ht); ASSERT_EQ(5, ht->entries.i); EXPECT_STREQ("localhost", EzCanonicalize(ht, "localhost")); EXPECT_STREQ("lol.example.", EzCanonicalize(ht, "lol")); @@ -75,6 +75,6 @@ TEST(resolvehoststxt, testCanonicalize) { EXPECT_STREQ("cat.example.", EzCanonicalize(ht, "cat")); EXPECT_STREQ("cat.example.", EzCanonicalize(ht, "cat.example.")); EXPECT_EQ(NULL, EzCanonicalize(ht, "boop")); - freehoststxt(&ht); + FreeHostsTxt(&ht); fclose(f); } diff --git a/test/libc/fmt/fmt_test.c b/test/libc/fmt/fmt_test.c index 644a76cfb..f35670f9a 100644 --- a/test/libc/fmt/fmt_test.c +++ b/test/libc/fmt/fmt_test.c @@ -276,3 +276,8 @@ TEST(fmt, p) { EXPECT_STREQ("0xffff800000031337", gc(xasprintf("% 10p", 0xffff800000031337))); } + +/* TEST(fmt, funchar) { */ +/* /\* TODO(jart): fix this *\/ */ +/* ASSERT_STREQ("'\\200'", gc(xasprintf("%`'c", 0200))); */ +/* } */ diff --git a/test/libc/intrin/asan_test.c b/test/libc/intrin/asan_test.c new file mode 100644 index 000000000..4cb34740e --- /dev/null +++ b/test/libc/intrin/asan_test.c @@ -0,0 +1,42 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/intrin/asan.internal.h" +#include "libc/mem/mem.h" +#include "libc/testlib/testlib.h" + +TEST(asan, test) { + char *p; + if (!IsAsan()) return; + p = gc(malloc(3)); + EXPECT_TRUE(__asan_is_valid(p, 3)); + EXPECT_FALSE(__asan_is_valid(p, 4)); + EXPECT_TRUE(__asan_is_valid(p + 1, 2)); + EXPECT_FALSE(__asan_is_valid(p + 1, 3)); + p = gc(malloc(8 + 3)); + EXPECT_TRUE(__asan_is_valid(p, 8 + 3)); + EXPECT_FALSE(__asan_is_valid(p, 8 + 4)); + EXPECT_TRUE(__asan_is_valid(p + 1, 8 + 2)); + EXPECT_FALSE(__asan_is_valid(p + 1, 8 + 3)); + p = gc(malloc(64 + 3)); + EXPECT_TRUE(__asan_is_valid(p, 64 + 3)); + EXPECT_FALSE(__asan_is_valid(p, 64 + 4)); + EXPECT_TRUE(__asan_is_valid(p + 1, 64 + 2)); + EXPECT_FALSE(__asan_is_valid(p + 1, 64 + 3)); +} diff --git a/test/libc/intrin/intrin_test.c b/test/libc/intrin/intrin_test.c index 8b50bd8d0..b9b4a3b59 100644 --- a/test/libc/intrin/intrin_test.c +++ b/test/libc/intrin/intrin_test.c @@ -83,6 +83,7 @@ #include "libc/intrin/psrlq.h" #include "libc/intrin/psrlw.h" #include "libc/intrin/psubb.h" +#include "libc/intrin/psubd.h" #include "libc/intrin/psubq.h" #include "libc/intrin/psubsb.h" #include "libc/intrin/psubsw.h" @@ -581,7 +582,7 @@ TEST(punpckhbw, fuzz) { TEST(psubq, fuzz) { int i, j; - int64_t x[2], y[2], a[2], b[2]; + uint64_t x[2], y[2], a[2], b[2]; for (i = 0; i < 100; ++i) { RngSet(x, sizeof(x)); RngSet(y, sizeof(y)); @@ -660,7 +661,7 @@ TEST(psradv, test) { for (i = 0; i < 100; ++i) { RngSet(x, sizeof(x)); for (j = 0; j < 2; ++j) { - y[j] = Rando() % 64; + y[j] = Rando() % 70; } psradv(a, x, y); (psradv)(b, x, y); @@ -843,7 +844,7 @@ TEST(pmullw, fuzz) { TEST(pmulld, fuzz) { int i, j; - int32_t x[4], y[4], a[4], b[4]; + uint32_t x[4], y[4], a[4], b[4]; for (i = 0; i < 1000; ++i) { RngSet(x, sizeof(x)); RngSet(y, sizeof(y)); @@ -906,8 +907,8 @@ TEST(phaddw, fuzz) { TEST(phaddd, fuzz) { int i, j; - int32_t x[4], y[4]; - int32_t a[4], b[4]; + uint32_t x[4], y[4]; + uint32_t a[4], b[4]; for (i = 0; i < 1000; ++i) { RngSet(x, sizeof(x)); RngSet(y, sizeof(y)); @@ -938,8 +939,8 @@ TEST(phsubw, fuzz) { TEST(phsubd, fuzz) { int i, j; - int32_t x[4], y[4]; - int32_t a[4], b[4]; + uint32_t x[4], y[4]; + uint32_t a[4], b[4]; for (i = 0; i < 1000; ++i) { RngSet(x, sizeof(x)); RngSet(y, sizeof(y)); @@ -952,6 +953,22 @@ TEST(phsubd, fuzz) { } } +TEST(psubd, fuzz) { + int i, j; + uint32_t x[4], y[4]; + uint32_t a[4], b[4]; + for (i = 0; i < 1000; ++i) { + RngSet(x, sizeof(x)); + RngSet(y, sizeof(y)); + psubd(a, x, y); + (psubd)(b, x, y); + ASSERT_EQ(0, memcmp(a, b, 16)); + psubd(a, (void *)a, y); + (psubd)(b, (void *)b, y); + ASSERT_EQ(0, memcmp(a, b, 16)); + } +} + TEST(phaddsw, fuzz) { int i, j; int16_t x[8], y[8]; @@ -1157,7 +1174,7 @@ TEST(pandn, fuzz) { TEST(paddq, fuzz) { int i, j; - int64_t x[2], y[2], a[2], b[2]; + uint64_t x[2], y[2], a[2], b[2]; for (i = 0; i < 100; ++i) { for (j = 0; j < 2; ++j) x[j] = Rando(); for (j = 0; j < 2; ++j) y[j] = Rando(); @@ -1444,6 +1461,12 @@ TEST(pabsd, fuzz) { int i, j; int32_t x[4]; uint32_t a[4], b[4]; + x[0] = INT_MIN; + pabsd((uint32_t *)x, x); + EXPECT_EQ(INT_MIN, x[0]); + x[0] = INT_MIN; + (pabsd)((uint32_t *)x, x); + EXPECT_EQ(INT_MIN, x[0]); for (i = 0; i < 100; ++i) { RngSet(x, sizeof(x)); pabsd(a, x); @@ -1497,6 +1520,16 @@ TEST(psignd, fuzz) { } } +TEST(psignd, testBane) { + int32_t x[4] = {INT_MIN, INT_MIN, INT_MIN, INT_MIN}; + int32_t y[4] = {0, 1, -1, INT_MIN}; + psignd(x, x, y); + EXPECT_EQ(0, x[0]); + EXPECT_EQ(INT_MIN, x[1]); + EXPECT_EQ(INT_MIN, x[2]); + EXPECT_EQ(INT_MIN, x[3]); +} + TEST(paddb, fuzz) { int i, j; int8_t x[16], y[16], a[16], b[16]; @@ -1574,7 +1607,12 @@ TEST(psubsw, fuzz) { TEST(paddd, fuzz) { int i, j; - int32_t x[4], y[4], a[4], b[4]; + uint32_t x[4], y[4], a[4], b[4]; + RngSet(x, sizeof(x)); + RngSet(y, sizeof(y)); + x[0] = 0x7fffffff; + y[0] = 0x7fffffff; + (paddd)(b, x, y); for (i = 0; i < 100; ++i) { RngSet(x, sizeof(x)); RngSet(y, sizeof(y)); diff --git a/test/libc/intrin/pmulhrsw_test.c b/test/libc/intrin/pmulhrsw_test.c index 2ddda913e..aa6d06309 100644 --- a/test/libc/intrin/pmulhrsw_test.c +++ b/test/libc/intrin/pmulhrsw_test.c @@ -78,6 +78,13 @@ TEST(pmulhrsw, testLimits) { EXPECT_EQ(kPmulhrswTorture[i][2], A[0], "pmulhrsw(%hd,%hd)→%hd", kPmulhrswTorture[i][0], kPmulhrswTorture[i][1], A[0]); } + for (i = 0; i < ARRAYLEN(kPmulhrswTorture); ++i) { + A[0] = kPmulhrswTorture[i][0]; + B[0] = kPmulhrswTorture[i][1]; + (pmulhrsw)(A, A, B); + EXPECT_EQ(kPmulhrswTorture[i][2], A[0], "pmulhrsw(%hd,%hd)→%hd", + kPmulhrswTorture[i][0], kPmulhrswTorture[i][1], A[0]); + } } TEST(pmulhrsw, testFakeFloat) { diff --git a/test/libc/intrin/test.mk b/test/libc/intrin/test.mk index cfe41817e..a36d3189c 100644 --- a/test/libc/intrin/test.mk +++ b/test/libc/intrin/test.mk @@ -26,6 +26,7 @@ TEST_LIBC_INTRIN_DIRECTDEPS = \ LIBC_FMT \ LIBC_INTRIN \ LIBC_LOG \ + LIBC_MEM \ LIBC_NEXGEN32E \ LIBC_RAND \ LIBC_RUNTIME \ diff --git a/test/libc/nexgen32e/memrchr_test.c b/test/libc/nexgen32e/memrchr_test.c new file mode 100644 index 000000000..b7f0b6ac1 --- /dev/null +++ b/test/libc/nexgen32e/memrchr_test.c @@ -0,0 +1,29 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/nexgen32e/nexgen32e.h" +#include "libc/testlib/ezbench.h" +#include "libc/testlib/testlib.h" + +TEST(memrchr, test) { + EXPECT_STREQ(".there", memrchr("yo.hi.there", '.', 11)); +} + +BENCH(memrchr, bench) { + EZBENCH2("memrchr", donothing, EXPROPRIATE(memrchr("yo.hi.there", '.', 11))); +} diff --git a/test/libc/release/emulate.sh b/test/libc/release/emulate.sh index 9b3457f0c..ec314d01b 100755 --- a/test/libc/release/emulate.sh +++ b/test/libc/release/emulate.sh @@ -4,6 +4,10 @@ if [ "$MODE" = dbg ]; then exit # TODO fi +if [ "$MODE" = opt ]; then + exit +fi + # smoke test userspace binary emulation CMD="o/$MODE/tool/build/blinkenlights.com.dbg o/$MODE/examples/hello.com" if OUTPUT="$($CMD)"; then diff --git a/test/libc/release/metal.sh b/test/libc/release/metal.sh index e12edee59..42e381364 100755 --- a/test/libc/release/metal.sh +++ b/test/libc/release/metal.sh @@ -4,6 +4,10 @@ if [ "$MODE" = dbg ]; then exit # TODO fi +if [ "$MODE" = opt ]; then + exit +fi + # smoke test booting on bare metal and printing data to serial uart CMD="o/$MODE/tool/build/blinkenlights.com.dbg -r o/$MODE/examples/hello.com" if OUTPUT="$($CMD)"; then diff --git a/test/libc/str/memmem_test.c b/test/libc/str/memmem_test.c index 4b2695df0..87f56ce7c 100644 --- a/test/libc/str/memmem_test.c +++ b/test/libc/str/memmem_test.c @@ -21,6 +21,8 @@ #include "libc/mem/mem.h" #include "libc/str/internal.h" #include "libc/str/str.h" +#include "libc/testlib/ezbench.h" +#include "libc/testlib/hyperion.h" #include "libc/testlib/testlib.h" #define MakeMemory(SL) memcpy(malloc(sizeof(SL) - 1), SL, sizeof(SL) - 1) @@ -60,6 +62,22 @@ TEST(memmem, testEndOfMemory) { free(needle); } +TEST(memmem, testOneNo) { + char *needle = MakeMemory("z"); + char *haystk = MakeMemory("abc123"); + EXPECT_EQ(0, memmem(haystk, 6, needle, 1)); + free(haystk); + free(needle); +} + +TEST(memmem, testOneYes) { + char *needle = MakeMemory("3"); + char *haystk = MakeMemory("abc123"); + EXPECT_EQ(&haystk[5], memmem(haystk, 6, needle, 1)); + free(haystk); + free(needle); +} + TEST(memmem, testCrossesSseRegister) { char *needle = MakeMemory("eeeeeeeeeeeeefffffffffffff"); char *haystk = MakeMemory("eeeeeeeeeeeeeeeeffffffffffffffffrrrrrrrrrrrrrrrr"); @@ -113,3 +131,12 @@ TEST(memmem, testEmptyHaystackAndNeedle_returnsHaystack) { TEST(memmem, testWut) { ASSERT_STREQ("x", memmem("x", 1, "x", 1)); } + +BENCH(memmem, bench) { + EZBENCH2("memmem", donothing, + EXPROPRIATE(memmem(kHyperion, kHyperionSize, "THE END", 7))); + EZBENCH2("memmem", donothing, + EXPROPRIATE(memmem( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + 62, "aaaaaab", 7))); +} diff --git a/test/libc/str/regex_test.c b/test/libc/str/regex_test.c index c57e4e2b9..ab92eb74c 100644 --- a/test/libc/str/regex_test.c +++ b/test/libc/str/regex_test.c @@ -83,24 +83,29 @@ TEST(regex, testIpExtended) { regfree(&rx); } +TEST(regex, testUnicodeCharacterClass) { + regex_t rx; + EXPECT_EQ(REG_OK, regcomp(&rx, "^[[:alpha:]][[:alpha:]]$", 0)); + EXPECT_EQ(REG_OK, regexec(&rx, "𝐵𝑏", 0, 0, 0)); + EXPECT_NE(REG_OK, regexec(&rx, "₀₁", 0, 0, 0)); + regfree(&rx); +} + void A(void) { regex_t rx; regcomp(&rx, "^[-._0-9A-Za-z]*$", REG_EXTENDED); regexec(&rx, "foo.com", 0, NULL, 0); regfree(&rx); } - void B(regex_t *rx) { regexec(rx, "foo.com", 0, NULL, 0); } - void C(void) { regex_t rx; regcomp(&rx, "^[-._0-9A-Za-z]*$", 0); regexec(&rx, "foo.com", 0, NULL, 0); regfree(&rx); } - void D(regex_t *rx, regmatch_t *m) { regexec(rx, "127.0.0.1", rx->re_nsub + 1, m, 0); } @@ -113,7 +118,6 @@ BENCH(regex, bench) { regfree(&rx); EZBENCH2("easy api extended", donothing, A()); EZBENCH2("easy api basic", donothing, C()); - EXPECT_EQ(REG_OK, regcomp(&rx, "^" "\\([0-9][0-9]*\\)\\." @@ -126,7 +130,6 @@ BENCH(regex, bench) { EZBENCH2("precompiled basic match", donothing, D(&rx, m)); free(m); regfree(&rx); - EXPECT_EQ(REG_OK, regcomp(&rx, "^" "([0-9]{1,3})\\." @@ -139,7 +142,6 @@ BENCH(regex, bench) { EZBENCH2("precompiled extended match", donothing, D(&rx, m)); free(m); regfree(&rx); - EXPECT_EQ(REG_OK, regcomp(&rx, "^" "([0-9]{1,3})\\." @@ -152,4 +154,17 @@ BENCH(regex, bench) { EZBENCH2("precompiled nosub match", donothing, D(&rx, m)); free(m); regfree(&rx); + EXPECT_EQ(REG_OK, regcomp(&rx, "^[a-z]*$", REG_EXTENDED | REG_NOSUB)); + m = calloc(rx.re_nsub + 1, sizeof(regmatch_t)); + EZBENCH2("precompiled alpha", donothing, + regexec(&rx, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 0, 0, 0)); + free(m); + regfree(&rx); + EXPECT_EQ(REG_OK, + regcomp(&rx, "^[a-z]*$", REG_EXTENDED | REG_NOSUB | REG_ICASE)); + m = calloc(rx.re_nsub + 1, sizeof(regmatch_t)); + EZBENCH2("precompiled alpha icase", donothing, + regexec(&rx, "aaaaaaaaaaaaaaaAAAAAAAAAAAAAA", 0, 0, 0)); + free(m); + regfree(&rx); } diff --git a/test/libc/str/strlen_test.c b/test/libc/str/strlen_test.c index db32b8bb1..e5e9e2713 100644 --- a/test/libc/str/strlen_test.c +++ b/test/libc/str/strlen_test.c @@ -159,25 +159,47 @@ BENCH(strlen, bench) { extern size_t strlen_(const char *) asm("strlen"); extern size_t strlen_pure_(const char *) asm("strlen_pure"); static char b[2048]; + static char c[512]; + static char d[256]; memset(b, -1, sizeof(b) - 1); - EZBENCH2("strlen 1", donothing, strlen_("")); - EZBENCH2("strlen_pure 1", donothing, strlen_pure_("")); - EZBENCH2("strlen 2", donothing, strlen_("1")); - EZBENCH2("strlen_pure 2", donothing, strlen_pure_("1")); - EZBENCH2("strlen 7", donothing, strlen_("123456")); - EZBENCH2("strlen_pure 7", donothing, strlen_pure_("123456")); - EZBENCH2("strlen 8", donothing, strlen_("1234567")); - EZBENCH2("strlen_pure 8", donothing, strlen_pure_("1234567")); - EZBENCH2("strlen 9", donothing, strlen_("12345678")); - EZBENCH2("strlen_pure 9", donothing, strlen_pure_("12345678")); - EZBENCH2("strlen 11", donothing, strlen_("12345678aa")); - EZBENCH2("strlen_pure 11", donothing, strlen_pure_("12345678aa")); - EZBENCH2("strlen 13", donothing, strlen_("12345678aabb")); - EZBENCH2("strlen_pure 13", donothing, strlen_pure_("12345678aabb")); - EZBENCH2("strlen 16", donothing, strlen_("123456781234567")); - EZBENCH2("strlen_pure 16", donothing, strlen_pure_("123456781234567")); - EZBENCH2("strlen 17", donothing, strlen_("123456781234567e")); - EZBENCH2("strlen_pure 17", donothing, strlen_pure_("123456781234567e")); - EZBENCH2("strlen 1023", donothing, strlen_(b)); - EZBENCH2("strlen_pure 1023", donothing, strlen_pure_(b)); + memset(c, -1, sizeof(c) - 1); + memset(d, -1, sizeof(d) - 1); + EZBENCH2("strlen_sse 1", donothing, strlen_("")); + EZBENCH2("strlen_swar 1", donothing, strlen_pure_("")); + EZBENCH2("strlen_sse 2", donothing, strlen_("1")); + EZBENCH2("strlen_swar 2", donothing, strlen_pure_("1")); + EZBENCH2("strlen_sse 3", donothing, strlen_("11")); + EZBENCH2("strlen_swar 3", donothing, strlen_pure_("11")); + EZBENCH2("strlen_sse 4", donothing, strlen_("113")); + EZBENCH2("strlen_swar 4", donothing, strlen_pure_("113")); + EZBENCH2("strlen_sse 7", donothing, strlen_("123456")); + EZBENCH2("strlen_swar 7", donothing, strlen_pure_("123456")); + EZBENCH2("strlen_sse 8", donothing, strlen_("1234567")); + EZBENCH2("strlen_swar 8", donothing, strlen_pure_("1234567")); + EZBENCH2("strlen_sse 9", donothing, strlen_("12345678")); + EZBENCH2("strlen_swar 9", donothing, strlen_pure_("12345678")); + EZBENCH2("strlen_sse 11", donothing, strlen_("12345678aa")); + EZBENCH2("strlen_swar 11", donothing, strlen_pure_("12345678aa")); + EZBENCH2("strlen_sse 13", donothing, strlen_("12345678aabb")); + EZBENCH2("strlen_swar 13", donothing, strlen_pure_("12345678aabb")); + EZBENCH2("strlen_sse 16", donothing, strlen_("123456781234567")); + EZBENCH2("strlen_swar 16", donothing, strlen_pure_("123456781234567")); + EZBENCH2("strlen_sse 17", donothing, strlen_("123456781234567e")); + EZBENCH2("strlen_swar 17", donothing, strlen_pure_("123456781234567e")); + EZBENCH2("strlen_sse 34", donothing, + strlen_("123456781234567e123456781234567ee")); + EZBENCH2("strlen_swar 34", donothing, + strlen_pure_("123456781234567e123456781234567ee")); + EZBENCH2("strlen_sse 68", donothing, + strlen_("123456781234567e123456781234567ee123456781234567e1234567812" + "34567eee")); + EZBENCH2("strlen_swar 68", donothing, + strlen_pure_("123456781234567e123456781234567ee123456781234567e12345" + "6781234567eee")); + EZBENCH2("strlen_sse 256", donothing, strlen_(d)); + EZBENCH2("strlen_swar 256", donothing, strlen_pure_(d)); + EZBENCH2("strlen_sse 512", donothing, strlen_(c)); + EZBENCH2("strlen_swar 512", donothing, strlen_pure_(c)); + EZBENCH2("strlen_sse 2048", donothing, strlen_(b)); + EZBENCH2("strlen_swar 2048", donothing, strlen_pure_(b)); } diff --git a/test/libc/str/strstr_test.c b/test/libc/str/strstr_test.c index 273e54e93..4066424be 100644 --- a/test/libc/str/strstr_test.c +++ b/test/libc/str/strstr_test.c @@ -46,6 +46,12 @@ TEST(strstr, test_notFound) { free(haystack); } +TEST(strstr, test_notFound1) { + MAKESTRING(haystack, "abc123def"); + ASSERT_EQ(NULL, strstr(haystack, gc(strdup("x")))); + free(haystack); +} + TEST(strstr, test_middleOfString) { MAKESTRING(haystack, "abc123def"); ASSERT_STREQ(&haystack[3], strstr(haystack, gc(strdup("123")))); @@ -80,8 +86,40 @@ TEST(strstr, test) { BENCH(strstr, bench) { EZBENCH2("strstr", donothing, EXPROPRIATE(strstr(kHyperion, "THE END"))); - EZBENCH2("strstr", donothing, + EZBENCH2("strstr torture 1", donothing, EXPROPRIATE(strstr( - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", - "aaaaaab"))); + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "b"))); + EZBENCH2("strstr torture 2", donothing, + EXPROPRIATE(strstr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "ab"))); + EZBENCH2("strstr torture 4", donothing, + EXPROPRIATE(strstr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "aaab"))); + EZBENCH2("strstr torture 8", donothing, + EXPROPRIATE(strstr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "aaaaaaab"))); + EZBENCH2("strstr torture 16", donothing, + EXPROPRIATE(strstr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "aaaaaaaaaaaaaaab"))); + EZBENCH2("strstr torture 32", donothing, + EXPROPRIATE(strstr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"))); } diff --git a/test/libc/str/towupper_test.c b/test/libc/str/towupper_test.c new file mode 100644 index 000000000..9284a1be3 --- /dev/null +++ b/test/libc/str/towupper_test.c @@ -0,0 +1,74 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/str/str.h" +#include "libc/testlib/ezbench.h" +#include "libc/testlib/testlib.h" + +TEST(towupper, test) { + EXPECT_EQ(u'!', towupper(u'!')); + EXPECT_EQ(u'A', towupper(u'a')); + EXPECT_EQ(u'À', towupper(u'à')); + if (IsTiny()) return; + EXPECT_EQ(L'𝛥', towupper(L'𝛿')); + EXPECT_EQ(L'Ｂ', towupper(L'ｂ')); +} + +TEST(towlower, test) { + EXPECT_EQ(u'!', towlower(u'!')); + EXPECT_EQ(u'a', towlower(u'A')); + EXPECT_EQ(u'à', towlower(u'À')); + if (IsTiny()) return; + EXPECT_EQ(L'𝛿', towlower(L'𝛥')); + EXPECT_EQ(L'ｂ', towlower(L'Ｂ')); +} + +BENCH(towupper, bench) { + EZBENCH2("towupper ascii", donothing, EXPROPRIATE(towupper(VEIL("r", L'a')))); + EZBENCH2("towupper latin1", donothing, + EXPROPRIATE(towupper(VEIL("r", u'A')))); + if (IsTiny()) return; + EZBENCH2("towupper astral", donothing, + EXPROPRIATE(towupper(VEIL("r", L'𝛿')))); +} + +BENCH(towlower, bench) { + EZBENCH2("towlower ascii", donothing, EXPROPRIATE(towlower(VEIL("r", L'a')))); + EZBENCH2("towlower latin1", donothing, + EXPROPRIATE(towlower(VEIL("r", u'A')))); + if (IsTiny()) return; + EZBENCH2("towlower astral", donothing, + EXPROPRIATE(towlower(VEIL("r", L'𝛿')))); +} + +BENCH(iswupper, bench) { + EZBENCH2("iswupper ascii", donothing, EXPROPRIATE(iswupper(VEIL("r", L'A')))); + EZBENCH2("iswupper latin1", donothing, + EXPROPRIATE(iswupper(VEIL("r", u'A')))); + EZBENCH2("iswupper astral", donothing, + EXPROPRIATE(iswupper(VEIL("r", L'𝛿')))); +} + +BENCH(iswlower, bench) { + EZBENCH2("iswlower ascii", donothing, EXPROPRIATE(iswlower(VEIL("r", L'a')))); + EZBENCH2("iswlower latin1", donothing, + EXPROPRIATE(iswlower(VEIL("r", u'A')))); + EZBENCH2("iswlower astral", donothing, + EXPROPRIATE(iswlower(VEIL("r", L'𝛿')))); +} diff --git a/libc/nexgen32e/lzcnt.S b/test/libc/tinymath/fsum_test.c similarity index 58% rename from libc/nexgen32e/lzcnt.S rename to test/libc/tinymath/fsum_test.c index fff23114a..04227e80a 100644 --- a/libc/nexgen32e/lzcnt.S +++ b/test/libc/tinymath/fsum_test.c @@ -1,7 +1,7 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -17,29 +17,38 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" +#include "libc/math.h" +#include "libc/runtime/gc.internal.h" +#include "libc/testlib/ezbench.h" +#include "libc/testlib/testlib.h" +#include "libc/x/x.h" -// Finds leading bits in 𝑥. -// -// uint32 𝑥 bsf(𝑥) tzcnt(𝑥) ffs(𝑥) bsr(𝑥) lzcnt(𝑥) -// 0x00000000 wut 32 0 wut 32 -// 0x00000001 0 0 1 0 31 -// 0x80000001 0 0 1 31 0 -// 0x80000000 31 31 32 31 0 -// 0x00000010 4 4 5 4 27 -// 0x08000010 4 4 5 27 4 -// 0x08000000 27 27 28 27 4 -// 0xffffffff 0 0 1 31 0 -// -// @param edi is 32-bit unsigned 𝑥 value -// @return eax number in range [0,32) or 32 if 𝑥 is 0 -// @see also treasure trove of nearly identical functions -lzcnt: .leafprologue - .profilable - mov $31,%eax - mov $-1,%edx - bsr %edi,%ecx - cmovz %edx,%ecx - sub %ecx,%eax - .leafepilogue - .endfn lzcnt,globl - .source __FILE__ +#define N 100000 + +float F[N]; +double D[N]; + +void SetUp(void) { + int i; + for (i = 0; i < N / 2; ++i) { + D[i * 2 + 0] = 1000000000.1; + D[i * 2 + 1] = 1.1; + } + for (i = 0; i < N / 2; ++i) { + F[i * 2 + 0] = 1000.1; + F[i * 2 + 1] = 1.1; + } +} + +TEST(fsum, test) { + EXPECT_STREQ("500000000.6", gc(xasprintf("%.15g", fsum(D, N) / N))); +} + +TEST(fsumf, test) { + EXPECT_STREQ("500.6", gc(xasprintf("%.7g", fsumf(F, N) / N))); +} + +BENCH(fsum, bench) { + EZBENCH2("fsum", donothing, fsum(D, N)); + EZBENCH2("fsumf", donothing, fsumf(F, N)); +} diff --git a/test/net/http/hascontrolcodes_test.c b/test/net/http/hascontrolcodes_test.c index 264bf0bd6..b7bc9f7f8 100644 --- a/test/net/http/hascontrolcodes_test.c +++ b/test/net/http/hascontrolcodes_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/testlib/blocktronics.h" #include "libc/testlib/ezbench.h" #include "libc/testlib/hyperion.h" #include "libc/testlib/testlib.h" @@ -48,4 +49,6 @@ BENCH(HasControlCodes, bench) { EZBENCH2("HasControlCodes small", donothing, HasControlCodes("hello", -1, 0)); EZBENCH2("HasControlCodes big", donothing, HasControlCodes(kHyperion, kHyperionSize, kControlC1)); + EZBENCH2("HasControlCodes unicode", donothing, + HasControlCodes(kBlocktronics, kBlocktronicsSize, kControlC1)); } diff --git a/test/net/http/isacceptablehost_test.c b/test/net/http/isacceptablehost_test.c index d212afd20..115d7db21 100644 --- a/test/net/http/isacceptablehost_test.c +++ b/test/net/http/isacceptablehost_test.c @@ -89,43 +89,12 @@ TEST(ParseIp, test) { EXPECT_EQ(-1, ParseIp("hello..example", -1)); } -TEST(ParseForwarded, test) { - uint32_t ip = 7; - uint16_t port = 7; - EXPECT_EQ(-1, ParseForwarded("", -1, &ip, &port)); - EXPECT_EQ(-1, ParseForwarded("0.0.0.0", -1, &ip, &port)); - EXPECT_EQ(-1, ParseForwarded("8.8.8.8", -1, &ip, &port)); - EXPECT_EQ(-1, ParseForwarded("[::1]:123", -1, &ip, &port)); - EXPECT_EQ(7, ip); - EXPECT_EQ(7, port); - EXPECT_EQ(0, ParseForwarded("0.0.0.1:123", -1, &ip, &port)); - EXPECT_EQ(0x00000001, ip); - EXPECT_EQ(123, port); - EXPECT_EQ(0, ParseForwarded("1.2.3.4:123", -1, &ip, &port)); - EXPECT_EQ(0x01020304, ip); - EXPECT_EQ(123, port); - EXPECT_EQ(0, ParseForwarded("128.2.3.4:123", -1, &ip, &port)); - EXPECT_EQ(0x80020304, ip); - EXPECT_EQ(123, port); - EXPECT_EQ(0, ParseForwarded("255.255.255.255:123", -1, &ip, &port)); - EXPECT_EQ(0xFFFFFFFF, ip); - EXPECT_EQ(123, port); - EXPECT_EQ(0, ParseForwarded("203.0.113.0:123", -1, &ip, &port)); - EXPECT_EQ(0xcb007100, ip); - EXPECT_EQ(123, port); - EXPECT_EQ(0, ParseForwarded("203.0.113.42:31337", -1, &ip, &port)); - EXPECT_EQ(-1, ParseForwarded("...:123", -1, &ip, &port)); - EXPECT_EQ(-1, ParseForwarded("203.0.113.0:123123123", -1, &ip, &port)); -} - BENCH(IsAcceptableHost, bench) { uint32_t ip; uint16_t port; EZBENCH2("IsAcceptableHost 127.0.0.1", donothing, IsAcceptableHost("127.0.0.1", 9)); EZBENCH2("IsAcceptablePort 80", donothing, IsAcceptablePort("80", 2)); - EZBENCH2("ParseForwarded 80", donothing, - ParseForwarded("203.0.113.42:31337", 20, &ip, &port)); EZBENCH2("IsAcceptableHost foo.example", donothing, IsAcceptableHost("foo.example:31337", 17)); } diff --git a/test/net/http/parseforwarded_test.c b/test/net/http/parseforwarded_test.c new file mode 100644 index 000000000..f922f12aa --- /dev/null +++ b/test/net/http/parseforwarded_test.c @@ -0,0 +1,60 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/testlib/ezbench.h" +#include "libc/testlib/testlib.h" +#include "net/http/http.h" + +TEST(ParseForwarded, test) { + uint32_t ip = 7; + uint16_t port = 7; + EXPECT_EQ(-1, ParseForwarded("", -1, &ip, &port)); + EXPECT_EQ(-1, ParseForwarded("[::1]:123", -1, &ip, &port)); + EXPECT_EQ(-1, ParseForwarded("203.0.113.0:123123123", -1, &ip, &port)); + EXPECT_EQ(7, ip); + EXPECT_EQ(7, port); + EXPECT_EQ(0, ParseForwarded("0.0.0.0", -1, &ip, &port)); + EXPECT_EQ(0x00000000, ip); + EXPECT_EQ(0, port); + EXPECT_EQ(0, ParseForwarded("8.8.8.8", -1, &ip, &port)); + EXPECT_EQ(0x08080808, ip); + EXPECT_EQ(0, port); + EXPECT_EQ(0, ParseForwarded("0.0.0.1:123", -1, &ip, &port)); + EXPECT_EQ(0x00000001, ip); + EXPECT_EQ(123, port); + EXPECT_EQ(0, ParseForwarded("1.2.3.4:123", -1, &ip, &port)); + EXPECT_EQ(0x01020304, ip); + EXPECT_EQ(123, port); + EXPECT_EQ(0, ParseForwarded("128.2.3.4:123", -1, &ip, &port)); + EXPECT_EQ(0x80020304, ip); + EXPECT_EQ(123, port); + EXPECT_EQ(0, ParseForwarded("255.255.255.255:123", -1, &ip, &port)); + EXPECT_EQ(0xFFFFFFFF, ip); + EXPECT_EQ(123, port); + EXPECT_EQ(0, ParseForwarded("203.0.113.0:123", -1, &ip, &port)); + EXPECT_EQ(0xcb007100, ip); + EXPECT_EQ(123, port); + EXPECT_EQ(0, ParseForwarded("203.0.113.42:31337", -1, &ip, &port)); +} + +BENCH(ParseForwarded, bench) { + uint32_t ip; + uint16_t port; + EZBENCH2("ParseForwarded 80", donothing, + ParseForwarded("203.0.113.42:31337", 20, &ip, &port)); +} diff --git a/test/tool/net/redbean_test.c b/test/tool/net/redbean_test.c index b722e7889..50c46f227 100644 --- a/test/tool/net/redbean_test.c +++ b/test/tool/net/redbean_test.c @@ -32,6 +32,7 @@ #include "libc/sysv/consts/shut.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/sock.h" +#include "libc/sysv/consts/tcp.h" #include "libc/testlib/testlib.h" #include "libc/x/x.h" #include "third_party/regex/regex.h" @@ -58,13 +59,27 @@ void SetUpOnce(void) { close(fdin); } +void Tune(int fd, int a, int b, int x) { + if (!b) return; + setsockopt(fd, a, b, &x, sizeof(x)); +} + +int Socket(void) { + int fd; + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1) { + Tune(fd, IPPROTO_TCP, TCP_CORK, 0); + Tune(fd, IPPROTO_TCP, TCP_NODELAY, 1); + } + return fd; +} + char *SendHttpRequest(const char *s) { int fd; char *p; size_t n; ssize_t rc; struct sockaddr_in addr = {AF_INET, htons(port), {htonl(INADDR_LOOPBACK)}}; - EXPECT_NE(-1, (fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP))); + EXPECT_NE(-1, (fd = Socket())); EXPECT_NE(-1, connect(fd, &addr, sizeof(addr))); n = strlen(s); EXPECT_EQ(n, write(fd, s, n)); @@ -114,10 +129,52 @@ TEST(redbean, testOptions) { "Accept-Charset: utf-8,ISO-8859-1;q=0\\.7,\\*;q=0\\.5\r\n" "Allow: GET, HEAD, POST, PUT, DELETE, OPTIONS\r\n" "Date: .*\r\n" - "Server: redbean/0\\.4\r\n" + "Server: redbean/.*\r\n" "Content-Length: 0\r\n" "\r\n", gc(SendHttpRequest("OPTIONS * HTTP/1.1\n\n")))); EXPECT_NE(-1, kill(pid, SIGTERM)); EXPECT_NE(-1, wait(0)); } + +TEST(redbean, testPipeline) { + if (IsWindows()) return; + char portbuf[16]; + int pid, pipefds[2]; + sigset_t chldmask, savemask; + sigaddset(&chldmask, SIGCHLD); + sigprocmask(SIG_BLOCK, &chldmask, &savemask); + ASSERT_NE(-1, pipe(pipefds)); + ASSERT_NE(-1, (pid = vfork())); + if (!pid) { + close(pipefds[0]); + dup2(pipefds[1], 1); + sigprocmask(SIG_SETMASK, &savemask, NULL); + execv("bin/redbean.com", + (char *const[]){"bin/redbean.com", "-szp0", "-l127.0.0.1", 0}); + _exit(127); + } + EXPECT_NE(-1, close(pipefds[1])); + EXPECT_NE(-1, read(pipefds[0], portbuf, sizeof(portbuf))); + port = atoi(portbuf); + EXPECT_TRUE(Matches("HTTP/1\\.1 200 OK\r\n" + "Accept: \\*/\\*\r\n" + "Accept-Charset: utf-8,ISO-8859-1;q=0\\.7,\\*;q=0\\.5\r\n" + "Allow: GET, HEAD, POST, PUT, DELETE, OPTIONS\r\n" + "Date: .*\r\n" + "Server: redbean/.*\r\n" + "Content-Length: 0\r\n" + "\r\n" + "HTTP/1\\.1 200 OK\r\n" + "Accept: \\*/\\*\r\n" + "Accept-Charset: utf-8,ISO-8859-1;q=0\\.7,\\*;q=0\\.5\r\n" + "Allow: GET, HEAD, POST, PUT, DELETE, OPTIONS\r\n" + "Date: .*\r\n" + "Server: redbean/.*\r\n" + "Content-Length: 0\r\n" + "\r\n", + gc(SendHttpRequest("OPTIONS * HTTP/1.1\n\n" + "OPTIONS * HTTP/1.1\n\n")))); + EXPECT_NE(-1, kill(pid, SIGTERM)); + EXPECT_NE(-1, wait(0)); +} diff --git a/test/tool/net/sqlite_test.c b/test/tool/net/sqlite_test.c new file mode 100644 index 000000000..dc5340790 --- /dev/null +++ b/test/tool/net/sqlite_test.c @@ -0,0 +1,41 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/testlib/testlib.h" +#include "third_party/sqlite3/sqlite3.h" + +char testlib_enable_tmp_setup_teardown; + +void SetUp(void) { + sqlite3_initialize(); +} + +TEST(sqlite, test) { + sqlite3 *db; + sqlite3_stmt *stmt; + ASSERT_EQ(SQLITE_OK, sqlite3_open("foo.sqlite", &db)); + ASSERT_EQ(SQLITE_OK, + sqlite3_prepare_v2(db, "PRAGMA synchronous=0", -1, &stmt, 0)); + ASSERT_EQ(SQLITE_DONE, sqlite3_step(stmt)); + ASSERT_EQ(SQLITE_OK, sqlite3_finalize(stmt)); + ASSERT_EQ(SQLITE_OK, + sqlite3_prepare_v2(db, "CREATE TABLE t (x INTEGER)", -1, &stmt, 0)); + ASSERT_EQ(SQLITE_DONE, sqlite3_step(stmt)); + ASSERT_EQ(SQLITE_OK, sqlite3_finalize(stmt)); + ASSERT_EQ(SQLITE_OK, sqlite3_close(db)); +} diff --git a/test/tool/net/test.mk b/test/tool/net/test.mk index d1341da9b..6b9baed62 100644 --- a/test/tool/net/test.mk +++ b/test/tool/net/test.mk @@ -43,7 +43,8 @@ TEST_TOOL_NET_DIRECTDEPS = \ LIBC_TESTLIB \ LIBC_X \ LIBC_ZIPOS \ - THIRD_PARTY_REGEX + THIRD_PARTY_REGEX \ + THIRD_PARTY_SQLITE3 TEST_TOOL_NET_DEPS := \ $(call uniq,$(foreach x,$(TEST_TOOL_NET_DIRECTDEPS),$($(x)))) diff --git a/third_party/chibicc/test/alignof_test.c b/third_party/chibicc/test/alignof_test.c index 35eeda542..f01d58f4f 100644 --- a/third_party/chibicc/test/alignof_test.c +++ b/third_party/chibicc/test/alignof_test.c @@ -1,4 +1,5 @@ #include "third_party/chibicc/test/test.h" +# int _Alignas(512) g1; int _Alignas(512) g2; diff --git a/third_party/gdtoa/g_xfmt.c b/third_party/gdtoa/g_xfmt.c index b6b1b1e28..b2439ab95 100644 --- a/third_party/gdtoa/g_xfmt.c +++ b/third_party/gdtoa/g_xfmt.c @@ -73,8 +73,8 @@ g_xfmt(char *buf, void *V, int ndig, size_t bufsize) L = (UShort *)V; sign = L[_0] & 0x8000; - bits[1] = (L[_1] << 16) | L[_2]; - bits[0] = (L[_3] << 16) | L[_4]; + bits[1] = ((ULong)L[_1] << 16) | L[_2]; + bits[0] = ((ULong)L[_3] << 16) | L[_4]; if ( (ex = L[_0] & 0x7fff) !=0) { if (ex == 0x7fff) { /* Infinity or NaN */ diff --git a/third_party/gdtoa/g_xfmt_p.c b/third_party/gdtoa/g_xfmt_p.c index 29ee7ecb6..fb8856290 100644 --- a/third_party/gdtoa/g_xfmt_p.c +++ b/third_party/gdtoa/g_xfmt_p.c @@ -75,8 +75,8 @@ g_xfmt_p(char *buf, void *V, int ndig, size_t bufsize, int nik) L = (UShort *)V; sign = L[_0] & 0x8000; - bits[1] = (L[_1] << 16) | L[_2]; - bits[0] = (L[_3] << 16) | L[_4]; + bits[1] = ((ULong)L[_1] << 16) | L[_2]; + bits[0] = ((ULong)L[_3] << 16) | L[_4]; if ( (ex = L[_0] & 0x7fff) !=0) { if (ex == 0x7fff) { /* Infinity or NaN */ diff --git a/third_party/gdtoa/gdtoa.c b/third_party/gdtoa/gdtoa.c index 70d2573ed..bb46f89f3 100644 --- a/third_party/gdtoa/gdtoa.c +++ b/third_party/gdtoa/gdtoa.c @@ -241,7 +241,23 @@ gdtoa(CONST FPI *fpi, int be, ULong *bits, int *kindp, int mode, int ndigits, in dval(&d) *= 1 << j1; word0(&d) += j << Exp_shift - 2 & Exp_mask; #else - word0(&d) += (be + bbits - 1) << Exp_shift; + // TODO: word0(&d) += (be + bbits - 1) << Exp_shift; + // error: third_party/gdtoa/gdtoa.c:244: left shift of negative value -6 'int' 20 'int' + // 4161d8: __die at libc/log/die.c:33 + // 463165: __ubsan_abort at libc/intrin/ubsan.c:270 + // 4632d6: __ubsan_handle_shift_out_of_bounds at libc/intrin/ubsan.c:299 + // 421d42: gdtoa at third_party/gdtoa/gdtoa.c:244 + // 420449: g_dfmt_p at third_party/gdtoa/g_dfmt_p.c:105 + // 413947: ConvertMatrixToStringTable at tool/viz/lib/formatmatrix-double.c:40 + // 413a5f: FormatMatrixDouble at tool/viz/lib/formatmatrix-double.c:55 + // 413b13: StringifyMatrixDouble at tool/viz/lib/formatmatrix-double.c:65 + // 464923: GetChromaticAdaptationMatrix_testD65ToD50_soWeCanCieLab at test/dsp/core/illumination_test.c:39 + // 4650c2: testlib_runtestcases at libc/testlib/testrunner.c:94 + // 464676: testlib_runalltests at libc/testlib/runner.c:37 + // 46455e: main at libc/testlib/testmain.c:84 + // 401d30: cosmo at libc/runtime/cosmo.S:65 + // 401173: _start at libc/crt/crt.S:67 + word0(&d) += (unsigned)(be + bbits - 1) << Exp_shift; #endif if (k >= 0 && k <= Ten_pmax) { if (dval(&d) < tens[k]) diff --git a/third_party/infozip/README.cosmo b/third_party/infozip/README.cosmo new file mode 100644 index 000000000..dc39f4c45 --- /dev/null +++ b/third_party/infozip/README.cosmo @@ -0,0 +1,7 @@ +zip utility from Info-ZIP: + +http://infozip.sourceforge.net/Zip.html + +source code obtained as zip30.tar.gz from: + +https://sourceforge.net/projects/infozip/files/ diff --git a/third_party/infozip/infozip.mk b/third_party/infozip/infozip.mk new file mode 100644 index 000000000..74a572f12 --- /dev/null +++ b/third_party/infozip/infozip.mk @@ -0,0 +1,134 @@ +#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ +#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ +PKGS += THIRD_PARTY_ZIP + +THIRD_PARTY_ZIP_FILES = $(wildcard third_party/infozip/zip/*) third_party/infozip/zip/unix/unix.c +THIRD_PARTY_ZIP_SRCS_C = $(filter %.c,$(THIRD_PARTY_ZIP_FILES)) +THIRD_PARTY_ZIP = $(THIRD_PARTY_ZIP_COMS) $(THIRD_PARTY_ZIP_COMS:%=%.dbg) + +THIRD_PARTY_ZIP_COMS = \ + o/$(MODE)/third_party/infozip/zip.com \ + o/$(MODE)/third_party/infozip/zipsplit.com \ + o/$(MODE)/third_party/infozip/zipnote.com \ + o/$(MODE)/third_party/infozip/zipcloak.com + +THIRD_PARTY_ZIP_OBJS = $(sort \ + $(THIRD_PARTY_ZIP_COM_OBJS) \ + $(THIRD_PARTY_ZIPCLOAK_OBJS) \ + $(THIRD_PARTY_ZIPNOTE_OBJS) \ + $(THIRD_PARTY_ZIPSPLIT_OBJS) \ + ) + +THIRD_PARTY_ZIP_UTIL_OBJS1 = \ + o/$(MODE)/third_party/infozip/zip/globals.o \ + o/$(MODE)/third_party/infozip/zip/unix/unix_.o \ + o/$(MODE)/third_party/infozip/zip/zipfile_.o \ + o/$(MODE)/third_party/infozip/zip/fileio_.o \ + o/$(MODE)/third_party/infozip/zip/util_.o + +THIRD_PARTY_ZIP_UTIL_OBJS2 = \ + o/$(MODE)/third_party/infozip/zip/crypt_.o \ + o/$(MODE)/third_party/infozip/zip/crc32_.o + +THIRD_PARTY_ZIP_UTIL_OBJS = \ + $(THIRD_PARTY_ZIP_UTIL_OBJS1) \ + $(THIRD_PARTY_ZIP_UTIL_OBJS2) + +THIRD_PARTY_ZIP_COM_OBJS = \ + o/$(MODE)/third_party/infozip/zip/zip.o \ + o/$(MODE)/third_party/infozip/zip/zipfile.o \ + o/$(MODE)/third_party/infozip/zip/zipup.o \ + o/$(MODE)/third_party/infozip/zip/fileio.o \ + o/$(MODE)/third_party/infozip/zip/util.o \ + o/$(MODE)/third_party/infozip/zip/globals.o \ + o/$(MODE)/third_party/infozip/zip/crypt.o \ + o/$(MODE)/third_party/infozip/zip/ttyio.o \ + o/$(MODE)/third_party/infozip/zip/unix/unix.o \ + o/$(MODE)/third_party/infozip/zip/crc32.o \ + o/$(MODE)/third_party/infozip/zip/zbz2err.o \ + o/$(MODE)/third_party/infozip/zip/deflate.o \ + o/$(MODE)/third_party/infozip/zip/trees.o + +THIRD_PARTY_ZIPSPLIT_OBJS = \ + o/$(MODE)/third_party/infozip/zip/zipsplit.o \ + $(THIRD_PARTY_ZIP_UTIL_OBJS1) + +THIRD_PARTY_ZIPNOTE_OBJS = \ + o/$(MODE)/third_party/infozip/zip/zipnote.o \ + $(THIRD_PARTY_ZIP_UTIL_OBJS1) + +THIRD_PARTY_ZIPCLOAK_OBJS = \ + o/$(MODE)/third_party/infozip/zip/zipcloak.o \ + o/$(MODE)/third_party/infozip/zip/ttyio.o \ + $(THIRD_PARTY_ZIP_UTIL_OBJS1) \ + $(THIRD_PARTY_ZIP_UTIL_OBJS2) + +THIRD_PARTY_ZIP_LARGE_OBJS = \ + o/$(MODE)/third_party/infozip/zip/zip.o \ + o/$(MODE)/third_party/infozip/zip/zipsplit.o \ + o/$(MODE)/third_party/infozip/zip/fileio.o \ + o/$(MODE)/third_party/infozip/zip/fileio_.o + +o/$(MODE)/third_party/infozip/zip/%_.o: third_party/infozip/zip/%.c + @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) -DUTIL $< + +$(THIRD_PARTY_ZIP_OBJS): \ + OVERRIDE_CFLAGS += \ + -DUNIX \ + -DMMAP \ + -DUNICODE_SUPPORT \ + -DUSE_EF_UT_TIME \ + -DLARGE_FILE_SUPPORT \ + -DHAVE_DIRENT_H \ + -DHAVE_TERMIOS_H \ + -DNO_BZIP2_SUPPORT + +$(THIRD_PARTY_ZIP_LARGE_OBJS): \ + OVERRIDE_CFLAGS += -DSTACK_FRAME_UNLIMITED + +THIRD_PARTY_ZIP_DIRECTDEPS = \ + LIBC_ERRNO \ + LIBC_LIMITS \ + LIBC_ALG \ + LIBC_FMT \ + LIBC_STR \ + LIBC_MEM \ + LIBC_LOG \ + LIBC_CALLS \ + LIBC_STDIO \ + LIBC_TIME \ + LIBC_UNICODE + +THIRD_PARTY_ZIP_DEPS := \ + $(call uniq,$(foreach x,$(THIRD_PARTY_ZIP_DIRECTDEPS),$($(x)))) + +o/$(MODE)/third_party/infozip/zip.com.dbg: \ + $(THIRD_PARTY_ZIP_DEPS) \ + $(THIRD_PARTY_ZIP_COM_OBJS) \ + $(CRT) \ + $(APE) + @$(APELINK) + +o/$(MODE)/third_party/infozip/zipsplit.com.dbg: \ + $(THIRD_PARTY_ZIP_DEPS) \ + $(THIRD_PARTY_ZIPSPLIT_OBJS) \ + $(CRT) \ + $(APE) + @$(APELINK) + +o/$(MODE)/third_party/infozip/zipnote.com.dbg: \ + $(THIRD_PARTY_ZIP_DEPS) \ + $(THIRD_PARTY_ZIPNOTE_OBJS) \ + $(CRT) \ + $(APE) + @$(APELINK) + +o/$(MODE)/third_party/infozip/zipcloak.com.dbg: \ + $(THIRD_PARTY_ZIP_DEPS) \ + $(THIRD_PARTY_ZIPCLOAK_OBJS) \ + $(CRT) \ + $(APE) + @$(APELINK) + +.PHONY: o/$(MODE)/third_party/infozip +o/$(MODE)/third_party/infozip: $(THIRD_PARTY_ZIP) diff --git a/third_party/infozip/zip/BUGS b/third_party/infozip/zip/BUGS new file mode 100644 index 000000000..21a9013f6 --- /dev/null +++ b/third_party/infozip/zip/BUGS @@ -0,0 +1,6 @@ +- zip sometimes crashes on some versions of NetBSD (0.8, 0.9 and early + 0.9-current), FreeBSD (<= 1.1) and BSDI (< 1.1) . This is due to a + bug in stdio. + Upgrading the stdio package in /usr/src/lib/libc/stdio should + fix the problem. See *BSD mirrors in src/lib/libc/stdio + You must at least replace setvbuf.o in all the libc's with a newer version. diff --git a/third_party/infozip/zip/CHANGES b/third_party/infozip/zip/CHANGES new file mode 100644 index 000000000..751695f2c --- /dev/null +++ b/third_party/infozip/zip/CHANGES @@ -0,0 +1,3460 @@ +------------------------- August 7th 1996 version 2.2a ------------------ + 1. QDOS port (Jonathan Hudson) + 2. win32 volumelabel handling (Paul) + 3. VM/CMS clean up (Greg Hartwig) + 4. leading "../" in internal filenames are allowed (Paul) + 5. System V packages support (John Bush) + 6. Fix handling of atx in zipup() (Onno, Greg) + 7. Fixed typo that caused zip -R to dump core (Onno) + 8. msdos/makefile.dj2: fix for command line too long when linking zip.exe + 9. win95 long filename support with djgpp v2 (Onno, Kimio Itoh) +------------------------- August 9th 1996 version 2.2b ------------------ + 1. windll: use wiz instead of wizip (Mike) + 2. use z->name NOT z->zname to open files (Onno, Mike) +------------------------ September 1st 1996 version 2.2c ------------------ + 1. windll: use fprintf instead of putc to send data to std{out,err} (Mike) + 2. os2: make borlandc version detection equal to unzip 5.30d (Kai Uwe) + 3. use #elif constructions for msdos,os2 and win32 compiler detection (Onno) + 4. fix for incorrect free in zip.c (Onno, Mike, Steve) + 5. BeBox port from Chris + 6. unix/{configure,Makefile} fixes for SCO Xenix 286 (Tom Schmidt) + 7. remove zilog entry from unix/Makefile (Onno) + 8. man page fixes (Tom Schmidt) + 9. SCO ODT {3,5} fixes (Bill Davidsen) +------------------------ October 8th 1996 version 2.2d ------------------ + 1. Fix bug in QDOS patch that broke zipsplit.c (Onno, Paul) + 2. Fix a couple of warnings from BorlandC (Mike) + 3. msdos/makefile.wat: Delete some more files when cleaning up (Paul) + 4. store msdos volumelabels without a dot in them (Paul) + 5. clean up of unix/{Makefile,configure,packaging} (Tom Schmidt) + 6. make QDOS port case independent (Jonathan Hudson) + 7. new amiga SASC makefile (Walter Haidinger) + 8. don't truncate filenames in win32's in2ex() (Paul) + 9. os2/makefile.os2 update for emx 0.9c (Kai Uwe) +10. password() function for QDOS (Jonathan) +11. fix the last(?) free() related bug (Mike) +12. win32: security descriptors operations (Scott Field) +13. win32: FILE_SHARE_DELETE is not defined in some win32 compilers (Onno) +14. win32: fix makefile.wat to include nt.c (Onno) +------------------------ January 17th 1997 version 2.2e ------------------ + 1. define USE_CASE_MAP in osdep.h for those ports that need it (Onno) + 2. define PROCNAME in osdep.h for those ports that need it (Onno) + 3. wild() prototype decl only if PROCNAME defined => delete MSVMS define (Onno) + 4. add DOS EMX makefile (E-Yen Tan) + 5. include a little earlier in qdos/qdos.c (Jonathan) + 6. add ttyio.o to OBJZ in qdos/Makefile.qdos (Jonathan) + 7. remove unused fprintebc define from zip.c (Onno) + 8. use the right password routine in ttyio.c for unzip (Mike) + 9. BeOS update from Chris +10. Fix for 'zip -r foo x:' (Paul) +11. Fix library bug on beos (Chris) +12. Fix calculating version number (kitoh_@mix.or.jp, Walter Haidinger) +13. IsWinNT always returned TRUE (Mike) +14. Windll update from Mike +15. Improved crc routines for x86 from Scott Field +16. Detect in unix/configure if we can use crc_i386.S (Onno) +17. Fix spurious internal logic error (Paul) +18. Fix to include directory names on the Acorn when needed (Sergio) +19. include zip.h in mvs.h (Onno, George Carr) +20. add workaround for AZTEC C compiler bug to revision.h (Paul, Walter) +21. MVS doesn't have rmdir (George Carr) +22. define and use USE_ZIPMAIN for WINDLL en VM_CMS (Onno) +23. Fixes from Greg Hartwig to make CMS standalone versions possible. +24. Move OS specific encryption stuff to the os specific directories (Christian) +25. Change password fetching interface in ttyio and crypt (Christian) +26. Update emx support for 0.9c (Christian) +27. Define WINDLL instead of MSWIN (Christian) +28. Extended time stamp extra field format support (Christian) +29. Support for rsxnt-emx 0.9c win32 compiler (Christian) +30. Use izshr017b (Christian) +------------------------ March 11th 1997 version 2.2f ------------------ + 1. Move makefile.emx, rsxwinnt.h and zip.def to win32 subdir (Kai Uwe) + 2. Add win32 target to makefile.os2 to allow cross compilation (Kai Uwe) + 3. Fix NTSD_EAS link time failures with win32 (Paul) + 4. Fix buffer freed too early in password verification code (Mike) + 5. Remove unix/zipgrep and man/zipgrep.1 (sanvila@ctv.es) + 6. Only use crc_i386.o when we're using an x86 (Onno, Mark) + 7. Remove carriage returns from amiga/crc_68.a (Paul) + 8. New windll from Mike + 9. Fix typo in os2/os2zip.c (Kai Uwe) +10. Don't use ctime (last file status change) for unix and qdos cross compile + (Greg) +11. added gccwin32 crosscompilation target (RSXNT) to os2/makefile.os2 (Kai Uwe) +12. fixed the OS/2 file attribute and time stamp generation for zipping + stdin ("-") (Kai Uwe) +13. fixed the atime and ctime stat fields for the OS/2 Watcom C library + (Kai Uwe) +14. added atime and ctime support for the UT extra field when generated under + OS/2, the atime and ctime values are only stored when zipping (Kai Uwe) +15. qdos patches from Jonathan Hudson mainly for extended time flag handling +16. amiga aztec compiler bug workaround (Paul) +17. fix -v output of zipcloak, zipnote and zipsplit (Paul) +18. new amiga/makefile.azt with targets for debug versions (Paul) +------------------------ March 31st 1997 version 2.2g ------------------ + 1. remove -I/usr/local/include from unix/Makefile (Chris) + 2. Update versinfolines in revision.h (Greg) + 3. change 1U to 0x1 to accomodate non ANSI compilers (Onno, Rodney Brown) + 4. win32zip.c: cast buffer parameter in memcompress() to char * (Mike) + 5. remove beos/zipgrep (Chris) + 6. correct the -e password verification check in zip.c (Christian) + 7. use ZCONST instead of const in the generic code. (Christian) + 8. fix mktime timezone correction when time is near to daylight/nodaylight + switch points. (Christian) + 9. correct dependencies in makefile.os2 (Christian) +10. use a more sensible default for iztime.ctime than "0" when system does not + not support creation time stamps. (Christian) +11. fix VMS_PK_EXTRA function interface declarations. (Christian) +12. implement atime/ctime support in win32. (Christian) +13. win32/win32.c: replacement getch() for Watcom. (Paul) +14. win32/makefile.wat: debug object files kept separate. (Paul) +15. msdos/makefile.wat: debug object files kept separate. (Paul) +16. Fix extended time defines for the acorn. (Sergio) +17. Define PROCNAME() in acorn/osdep.h (Sergio) +18. Ignore exit status of ${INSTALL_D} in unix/Makefile (Chris) +19. Add Metroworks and BEOS info to version() in several files (Chris) +20. Move defines for the password fetch to zip.h (Christian) +21. Support the obsolete version rsxnt 1.1 / emx 0.9b (Christian) +22. Remove obsolete "#define PROCNAME ..." from cmsmvs/cmsmvs.h (Christian) +23. Fix extended time defines for qdos (Jonathan Hudson) +24. Use watcom getch() from unz530q in win32/win32.c (Onno) +25. Don't install zipgrep via the unix package tools (John Bush) +26. use izshr021 (Onno) +27. Fix zipnote: use iname not zname in zipnote.c (Onno) +28. Create proginfo directory (Christian) +------------------------ May 5th 1997 version 2.2h -------------------- + 1. Fix vms/zipup.h: iztime --> iztimes (Onno, Mike Freeman) + 2. Remove windll/wizdll.def (Mike) + 3. Add a couple of external variable declaration to windll.h (Mike) + 4. Remove zipgrep from install in unix/Makefile (Onno) + 5. Make updating .zip files with extended time fields possible (Kai Uwe) + 6. Delete beos/Makefile.gcc, beos/Makefiles handles both compilers (Chris) + 7. Fixes for unused variables (Chris) + 8. Added very simplistic example how to load and call the windll (Mike) + 9. Updated windll documentation to note this example (Mike) +10. Removed an unused memeber of a structure in windll (Mike) +11. Add BUGS instead of infozip.who and algorith.doc with the packaging + tools (John Bush) +12. tailor.h: increment NUM_HOSTS to keep in sync with UnZip (Christian) +13. win32/osdep.h: remove NO_SECURE_TESTS define (Christian) +14. zip.h: add declaration for free_crc_table() (Christian) +15. windll: move everything that's not windows specific into api.* (Mike) +16. use iname when checking for directory names in zipfile.c (Sergio) +17. improved mktime.c with better error checking (Christian) +18. improved crc routines (Christian, Rodney Brown) +19. get the -z option working again (Onno, Brad Clarke) +20. define BROKEN_FSEEK and seekable() for those systems where fseek() + always returns 0 (== OK) (Onno, Jeffrey Altman) +------------------------ May 10th 1997 version 2.2i -------------------- + 1. win32's seekable should only check for FILE_TYPE_DISK (Onno, Jeffrey Altman) + 2. add (ulg) cast to zipbeg = ~0 in zipfile.c (Steve) + 3. seekable() *really* belongs in flush_block, keep it there (Onno) + 4. seekable() calls fseekable(FILE *) (Onno) + 5. define HAVE_FSEEKABLE if a port has their own fseekable (Onno) + 6. WatCom doesn't have _get_osfhandle, use _os_handle instead (Paul) + 7. upgrade to Mike's latest windll sources (Mike) + 8. add -P option so you can specify a password on the commandline (Onno) + 9. Get -@ working again (Onno) +10. emx+RSXNT doesn't know about _get_osfhandle() (Kai Uwe) +11. fix a couple of typos in the OS/2 makefiles (Kai Uwe) +12. fix initialization bug in windll code (Mike) +13. tweak deletedir for RISC OS (Sergio) +14. RISCOS doesn't know about fstat() (Sergio) +15. Remove acorn/acorn (Sergio) +16. Delete debugging statements from version_local() in msdos.c (Greg) +17. Fix huge bug in readzipfile() (Onno) +------------------------ May 18th 1997 version 2.2j -------------------- + 1. Add missing ';' after return ZE_PARMS in zip.c (Mike) + 2. Remove obsolete 'struct stat st' in zipfile.c (Onno) + 3. Get Amiga SFX handling working again (Paul) + 4. Get zip -A working again (Onno) + 5. Change an && to & in zipfile.c (Johnny) + 6. Fix handling of empty sfx archives (Onno, Mike) + 7. Remove experimental entries from the makefiles (Jean-loup) + 8. Add exit codes to the manual page (Onno) + 9. Remove lines from the help screen that contain lesser used options (Onno) +------------------------ June 8th 1997 version 2.2k -------------------- + 1. use zip -t ddmmyyyy for year 2000 stuff (Greg) + 2. zip -@ only handles ONE filename per line (Jean-loup) + 3. beos support for DR9 filesystem and symlinks (Chris) + 4. VB support for windll (Mike) +------------------------ June 10th 1997 version 2.2l ------------------- + 1. beos filetype support (Chris) + 2. fill the buffer in getnam() to get it working again (Onno) + 3. implement -x@filename and -i@filename (Onno) +------------------------ June 22nd 1997 version 2.2m ------------------- + 1. Add a ; after de nextarg label in main() (Onno, Erik Baatz) + 2. Initialize p to NULL in get_filters() (Onno, Frank Donahoe) + 3. Fix typo in first if statement in filetypes() (Johnny Lee) + 4. zip -A works again (Onno, Greg) + 5. don't free zipbuf for VMS and CMS_MVS in main() (Onno, Mike Freeman) + 6. fix make_zip.com, link_zip.com and vmsdefs.h for gcc 2.6.3 on VMS (Onno) + 7. clarify -g option in the man page (Jean-loup) +------------------------ July 6th 1997 version 2.2n ------------------- + 1. use local in readzipfile2() declaration (Onno, Mike Freeman) + 2. return values with windll in get_filters() (Mike) + 3. a couple of minor patches for BEOS (Chris) + 4. zip -g works again (Onno, Chris) + 5. Some more Visual Basic dll support (Mike) + 6. Fix stack overflow in readzipfile() for DOS (Onno, Michael Mauch) +------------------------ August 19th 1997 version 2.2o ------------------- + 1. beos README and Makefile tweaks from Chris. + 2. Syntax corrections for README and man/zip.1 (Frank Donahoe) + 3. Use name not iname when deleting directories in trash() (Christian) + 4. change several wkuvx1 to lists in e-mail addresses (Christian) + 5. default to PK style extra fields for VMS (Christian) + 6. use izshr023 (Christian) + 7. replace buggy time library functions (Walter Haidinger, Paul, Christian) + 8. in2ex() and stat() are needed also when UTIL isn't defined (Greg Hartwig) + 9. don't use type=record in fopen() for MVS and CMS (Greg Hartwig) +10. Change P and K literals to hex for EBCDIC systems (Greg Hartwig) +11. Add output path support for CMS and MVS (Greg Hartwig) +12. Add memtoasc and memtoebc for EBCDIC systems (Greg Hartwig) +13. Handle comments correctly to fix zipnote for CMS and MVS (Greg Hartwig) +14. Add -tt option (do not operate on files after date mmddyy) (Christian) +15. move alloc routines for DOS into the !UTIL block (Christian) +16. move UTIL blocks and version_local() functions to a more logical place + (Christian) +17. Handle -P, -R, -x@, -i@ and -tt for the VMS CLI (Christian) +18. Update VMS help file with the new options (Christian) +19. Use iname in MATCH, not zname (Jonathan Hudson) +20. windll: more Visual Basic support (Mike) +21. windll: more project makefiles (Mike) +22. windll: insert Zip in front of global variable names (Mike) +------------------------ August 25th 1997 version 2.2p ------------------- + 1. Remove unused flags from LFLAGS2 in unix/Makefile (Onno) + 2. SunOS make bug: change unix_.o rule in unix/Makefile (Onno, Mike Freeman) + 3. ZipIsWinNT() instead of IsWinNT() in zip.h (Mike) + 4. Fix -t and -tt behaviour for windll (Mike) + 5. Remove windll makefiles that are now elsewhere (Mike) + 6. BEOS: preserve file attributes associated with symbolic links (Chris) + 7. No need to use in2ex() for ziputils (Christian) + 8. Fix comment handling for EBCDIC systems (Christian) + 9. EBCDIC conversion for entry names read from zipfile in UTIL mode (Christian) +10. Fix "fatal" error messages on EBCDIC systems (Christian) +11. zipnote.c: Fix handling of entry name changes for EBCDIC systems (Christian) +12. removed a large part of "dead" code from ziputils version (Christian) +13. use z->iname in comparison functions for sorting (Christian) +14. new installation utils for the acorn (Sergio) +15. use LSSTAT in set_extra_field for unix and beos (Onno) +16. perror(z->zname) instead of perror("zip warning") (Onno, Geoff Pennington) +17. Amiga SFX should work again (Paul) +18. refer to zip22 in install.doc (Frank Donahoe) +------------------------ September 10th 1997 version 2.2q ------------------- + 1. Change .doc to .txt, these aren't MS-Word documents (John D. Mitchell) + 2. Change msdos$_(OBJ) to msdos_$(OBJ) (Kai Uwe) + 3. Fix a couple of amiga related glitches (Paul) + 4. Support for DOS packed .exe files in makefile.dj2 (Frank Donahoe) + 5. Change warning message for zip -A (Greg) +------------------------ September 29th 1997 version 2.2r ------------------- + 1. Fix make svr4package (Eric Baatz) + 2. Fix VMS warning (Mike Freeman, Christian) + 3. Clean up beos gcc port and beos README (Chris) +-------------------------- October 6th 1997 version 2.2s -------------------- + 1. Change lpPrint to lpZipPrint for windll (Mike) + 2. Change lpPassword to lpZipPassword for windll (Mike) + 3. Amiga timezone fixes (Paul) + 4. WatCom C 11.0 makefile fixes (Paul) + 5. Tandem port from Dave Smith + 6. Corrections and updates for install.txt (Christian) + 7. Minor VMS README update (Christian) +-------------------------- October 12th 1997 version 2.2t -------------------- + 1. qdos compiler bug workaround (Jonathan) + 2. prevent storing qdos specific filenames that exceed filesystem limits + (Jonathan) + 3. fix undelimited comment in fileio.c (Frank Donahoe) + 4. disable storing of symlinks in BEOS until OS support is available (Chris) + 5. Init hash_head to 0 in amiga/deflate.a (Paul) + 6. Upgrade to izshr025 (Christian) + 7. don't add ".zip" to ZIP name for TANDEM (Dave Smith) + 8. use zipup.h not tandem.h in zipup.c (Dave Smith) + 9. rename history to CHANGES (Onno) +10. rename install.txt to INSTALL (Onno) +11. rename zip.txt to ZIPMAN (Onno) +12. create WHATSNEW (Onno) +-------------------------- October 15th 1997 version 2.2u -------------------- + 1. Use Info-ZIP instead of Info-Zip (Christian) + 2. Note recent filename changes in several files (Christian) + 3. Remove a couple of items from the TODO list (Christian, Onno) + 4. Add windll port, zip -t yyyymmdd and zip -R to WHATSNEW (Christian) + 5. VMS documentation cleanups and clarifications (Christian) + 6. dist entry in unix/Makefile (Onno) + 7. remove duplicate amiga/timezone.txt (Christian) + 8. rename ZIPMAN to MANUAL and update a couple of files regarding this (Onno) +-------------------------- October 24th 1997 version 2.2v -------------------- + 1. izshr026: in WHERE wiz40 instead of wiz30 (Christian) + 2. izshr026: another couple of Info-ZIP spelling fixes (Christian) + 3. Remove zipgrep from the makefiles that still had it (Christian) + 4. Update makefiles to handle the MANUAL renaming change (Christian) + 5. Fix the last daylight savings bug on the Amiga (Paul) + 6. Fix the SCO Unix specialty detection in unix/configure (Onno, + bug reported by Bo Kullmar for Solaris 2.6 and with uname -X output + for SCO Unix from ken@apisys.com and dgsmith@vnet.ibm.com) + 7. Update WHERE and amiga/time_lib.c from unzip 5.32g (Greg) +-------------------------- October 26th 1997 version 2.2w -------------------- + 1. Additional +Onolimit check in unix/configure (Onno, Peter Jones) + 2. Use ZIPERR macro instead of ziperr (Christian) + 3. initialize z->lflg for zip entries without extra field (Christian) + 4. "local (+ locextend)" vs. "central" header consistency check (Christian) + 5. Override local header values with central header values with -A + and differences between these headers (Christain) + 6. made "deltaoff" signed long; offset adjustment may be negative (Christian) + 7. fix a number of "wild" deallocation bugs (Christian) + 8. When zipping from a FAT drive (only 8.3 DOS names) under OS/2 or + WIN32, set z->vem to "OS_DOS | ". + Mark as "made by DOS PKZIP 2.0" only when dosify was requested. (Christian) + 9. DOS port should not store fake unix style external attributes. (Christian) +10. amiga/time_lib.c from izshr028 (Christian) +-------------------------- October 31st 1997 version 2.2y -------------------- + 1. amiga/time_lib.c from izshr029 (Christian) + 2. Turbo C++ version code clarification (E-Yen Tan) + 3. Fix spelling in cmsvms/zipname.conven (Rodney Brown) + 4. Fix memset check in unix/configure for Unixware 2.1.1 (Rodney Brown) + 5. Forward declaration fixes for HP-UX bundled compiler (Rodney Brown) +-------------------------- November 3rd 1997 version 2.2 -------------------- + 1. Update WHERE (Greg). +-------------------------- January 4th 1998 version 2.21a ------------------- + 1. BSD friendly version of version_local() in unix/unix.c (Onno) + 2. No NT versions in DOS version_local() (Steve Salisbury) + 3. -t mmddyyyy instead of -t ddmmyyyy in WHATSNEW (Walter Haidinger) + 4. use generic fseekable() for rsxnt (Christian) + 5. Fix MSC 8.x warnings (Christian, Steve Salisbury) + 6. win32 Borland C++ makefile (E-Yen Tan) + 7. Tandem doesn't know about extensions like .zip,.arj, ... (Dave Smith) + 8. Use dosmatch for EMX and DJGPP too (Christian) + 9. dummy djgpp startup functions to remove command line globbing and + recognition of environment variables from djgpp.env (Christian) +10. include DJGPP_MINOR in DOS version_local() (Christian) +11. TC 2.0 doesn't have mktime() (Christian, mmp@earthling.net) +12. VMS: rename opendir() to zopendir() so avoiding name clash with + VMS 7.x POSIX libraries (Christian, Martin Zinser) +13. Add support for VMS DEC C V 5.6 features (Christian) +14. Use iname for comparison in check_dup (Christian Spieler, Christian Michel) +15. Fix access to uninitialized ioctx records in vms_get_attributes() + Christian, Robert Nielsen) +16. Parenthesis around MAX_MATCH>>1 in match.S (Greg) +17. Use strchr() not strrchr() for -i and -x to get -i@ and -x@ really + working (Onno, Kai Uwe) +18. add chmod statements to unix/Makefile (Quentin Barnes) +19. Windll: handle both -r and -R (Mike) +20. Windll: general error handler in main() via setjmp/longjmp (Mike) +21. Don't allow zip -i@x.lst foo.zip (Onno) +22. vms/link_zip.com: use .eqs. not .nes. when checking with f$search + for the zip AXP object library (David Dachtera) +23. rsxnt 1.3.1 fixes (E-Yen Tan) +-------------------------- January 20th 1998 version 2.21b ------------------- + 1. Bigger PATH_MAX for win32's windll (Mike) + 2. Update windll.txt w.r.t. PATH_MAX (Mike) + 3. Amiga SAS/C fixes (Walter, Paul) + 4. zip -i@ and -x@ should *really* work now ...... (Onno) +-------------------------- February 20th 1998 version 2.21c ------------------- + 1. make -f unix/Makefile qnx needs LN=ln in its options (Chris) + 2. Support Metroworks Codewarrior/x86 on BEOS (Chris) + 3. Add Norbert Pueschel to proginfo/infozip.who (Walter) + 4. Use big endian for Be types (Chris) + 5. zip -i and -x were broken by the -i@ fix last time around (Christian) + 6. win32 stat bandaid (Paul) + 7. acorn filetype and timestamp fixes (Sergio, D. Krumbholz) + 8. update to izshr30 (Christian) + 9. Support for NTSD in the RSXNT environment (Christian) +10. restructure readzipfile() (Christian) +11. Where needed define MATCH in osdep.h (Christian) +12. version_local() fixes for RSXNT (Christian) +13. New vmsmunch.c (Christian) +-------------------------- March 15th 1998 version 2.3a ------------------- + 1. Fixes for the windll API (Mike) + 2. Use CPUTYPE in BorlandC Makefile for DOS (E-Yen Tan) + 3. BEOS: -rostr not available for the x86 compiler (Chris) + 4. preserve file attributes of a symlink on BEOS (Chris) + 5. New VM/CMS README.CMS and version_local() (Ian Gorman) + 6. INSTALL fixes from Takahiro Watanabe + 7. OS/390 port from Paul von Behren + 8. new api.h from Mike +-------------------------- April 19th 1998 version 2.3b ------------------- + 1. Improve Tandem file I/O performance (Dave Smith) + 2. New VM/CMS README.CMS and version_local() (Ian Gorman) + 3. cygwin32 port from Cosmin Truta + 4. Workaround for tasm32 5.0 bug in win32/crc_i386.asm (Cosmin Truta) + 5. win32/match32.asm fixes for tasm 5.0 (Cosmin Truta) + 6. simplify OS/390 port (Christian) + 7. win32 timezone handling fixes (Christian) + 8. fix 40-bit time conversion on the acorn (Sergio and Christian) + 9. strip network part from UNC type filenames (Christian) +10. Makefile for OpenMVS (Ian Gorman) +11. Use the Watcom getch() for cygwin32 (Christian) +12. Borland C++ 5.x added to win32's version_local() (Cosmin Truta) +13. Borland C++ needs tzset() in win32 (Christian, Cosmin Truta) +-------------------------- May 21st 1998 version 2.3c ------------------- + 1. Better error messages for -i and -x (Christian) + 2. Win32 stat() wrapper needs dos2unixtime (Christian,Paul,Mike) + 3. DJGPP: use _chmod to handle LFN attributes correctly (Michael Mauch) + 4. Fix Borlandc warnings (Mike) + 5. win32/makefile.bor fixes from Michael Mauch + 6. win32/makefile.{dj,emx} fixes from E-Yen Tan + 7. Use izshr031 (Christian) + 8. CMS: use RECFM=V LRECL=32760 by adding "byteseek" (Greg Hartwig) + 9. Check external name for trailing "/" (Greg Hartwig) +10. More specific info in CMS version_local() (Greg Hartwig) +11. Changed usage info to refer to "fm" rather than "path" on CMS (Greg Hartwig) +12. No more "extra data" messages when using the same OS (Greg Hartwig) +13. Rewritten README.CMS, one version for ZIP and UNZIP (Greg Hartwig) +14. DOS/OS2/WIN32/UNIX: ex2in() strips off "//host/share/" from UNC names (SPC) +-------------------------- June 23rd 1998 version 2.3d ------------------- + 1. Fixed Win32's stat() bandaid handling of time stamps (SPC) + 2. General fix of file selections for DELETE and FRESHEN action (SPC) + 3. CMS_MVS: Use ASCII coding for TIME extra field ID (SPC) + 4. EBCDIC: Repaired bogus CMS_MVS fix in zipup.c; check the internal + name for trailing (ASCII) '/' to detect directory entries (SPC) + 5. Use explicit ASCII coding when comparing or setting chars in iname (SPC) + 6. Fixed win32/makefile.bor, win32/makefile.dj (support NTSD), + win32/makefile.emx (SPC) + 7. Replaced win32/makefile.cyg by win32/makefile.gcc, containing new + support for mingw32 GCC environment (SPC) + 8. Use izshr032 (SPC) + 9. Modified zipup.c to hold (un)compressed lengths in "ulg" variables, in + an attempt to support handling of huge (>2GByte) files. (SPC) +10. Removed some duplicate #defines from api.h, they are now in crypt.h (SPC) +11. Reenabled "extra data size" info messages in noisy mode for all systems + except RISCOS and CMS_MVS (SPC) +12. For EMX 0.9c, the runtime lib contains a working mktime(), use it (SPC) +13. Miscellanous cosmetic changes (SPC) +14. Move win32/makefile.emx to msdos (E-Yen Tan) +15. make api.h work with zcrypt2.8 (Mike) +16. define ydays differently in api.h to avoid linking problems (Mike) +17. New windll.txt (Mike) +18. win32 lcc patches (E-Yen Tan) +19. win32 lcc makefile (E-Yen Tan) +20. Multiple inclusion bug: no malloc.h when using lcc-win32 (E-Yen Tan) +21. New VB support files for windll (Mike Le Voi, Raymond King) +22. MacOS port by Dirk Haase +-------------------------- August 1st 1998 version 2.3e ------------------- + 1. Generalized check for validy of TZ timezone setup info, similar to + UnZip; use it on AMIGA and MSDOS, as before. (SPC) + 2. Apply TZ validy check on OS/2 and enable creation of UT e.f. (SPC) + 3. BEOS: New Makefile, updates for README and Contents (Chris Herborth) + 4. beos/beos.c: declare some private functions as "local" (SPC) + 5. Include memcompress() code only for ports that make use of it, controlled + by preprocessor symbol ZP_NEED_MEMCOMPR (SPC) + 6. cmsmvs/README.CMS fix: Zip archive entries to be extracted into var-length + records CMS files should >>NOT<< contain binary data ... (SPC) + 7. crc32.c, crctab.c: the crc polynom table is ZCONST (SPC) + 8. trees.c: fixed a bug in the deflate algorithm that limited the compressed + size of an archive member to 512 MByte (SPC) + 9. deflate.c: Integrated the changes found in zlib that are neccessary to make + the deflate algorithm deterministic; modified msdos/match.asm to take + care of the "nice_match" global no longer being constant. (SPC) +10. deflate.c, trees.c, zipup.c: Reorganized and simplified deflate's + compressed output buffer handling. I/O and compression code are now + separated more cleanly. (SPC) +11. Killed bits.c by moving its contents into trees.c resp. zipup.c; + synchronized all Makefiles and Make procedures with this change. (SPC) +12. Integrated support for optionally replacement of deflate and crc32 by + public domain zlib code. (SPC) +13. Synchronize the different variants (UNIX/GNU C, OS/2, WIN32) of i386 + assembler replacement for deflate's longest_match() (SPC) +14. Moved the EMX+rsxnt Makefile.emx from msdos/ back into win32/ (SPC) +15. Restored a separate Makefile.emx for DOS; on DOS, some make programs may + have difficulties with recursive invokation (SPC) +16. Fixed the "include header mess" of the new MACOS port and removed the + "work-around hacks" caused by these bad MACOS .h-file includes (SPC) +17. Integrated Dirk Haase's beta4 (27-Jun-98) release of MacZIP (Dirk Haase) +18. Added support for MS Quick C in the MSDOS version_local() report (SPC) +19. Added WIN32 rsxnt targets linking against the emx crtl DLL to Makefile.emx + in os2/ and win32/ (SPC) +20. Fixed typo in os2/os2.c wild() function. (Kai Uwe Rommel) +21. Removed ChangeNameForFAT() from os2/os2.c in2ex() to fix problem with + long filename support. (Kai Uwe Rommel) +22. os2/os2zip.[ch]: correct type of DOS-style timestamp data is "ulg" (SPC) +23. vms/cmdline.c: Removed wrong ';' behind if condition (Johnny Lee) +24. VMS: Preliminary preparations in C code for supporting GNU C on OpenVMS + Alpha (Onno van der Linden, Christian Spieler) +25. VMS: Fixed check against adding zipfile to itself in fileio.c (SPC) +26. WIN32: Added lcc-Win32 variants of i386 assembler code for crc32() and + longest_match(). (SPC) +27. WIN32: Removed bogus type-cast in assignment to statb st_mode member (SPC) +28. zip.c: Fixed MACOS-related typo that broke "-@" command option (SPC) +29. zipup.c: Fixed messed-up expression for assignment to z->ver (SPC) +30. MACOS extra fields: check realloc return values (Onno, Johnny Lee) +31. Fix the PUTBYTE macro in trees.c: >= instead of < (Onno) +-------------------------- September 6th 1998 version 2.3f ------------------- + 1. Add zp_tz_is_valid to globals.c (Onno, Frank Donahoe) + 2. Updated tandem files from Dave Smith + 3. Windll: allow comments to zip archive with VB (Mike) + 4. Windll: add support for -b and update the documentation (Mike) + 5. win32: use wbS for FOPW to handle large zip files better (Steve Miller) + 6. MVS fix: use fseek();clearerr() instead of rewind() (Onno, Lee Burton) + 7. Updated VB examples for windll (Mike) + 8. Tandem: use UTC timestamps and GID/UID in extra field (Dave Smith) + 9. Tandem: handle -o option (Dave Smith) +10. default for ZCONST is const in tailor.h, override in osdep.h (Onno) +11. additional Macintosh options in zip.c (Dirk Haase) +12. additional Macintosh options in zip.1 and MANUAL (Onno, Dirk Haase) +13. Integrate Beta 5 of the Macintosh Port (Dirk Haase) +-------------------------- October 27th 1998 version 2.3g ------------------- + 1. zip_tz_is_valid should be zp_tz_is_valid (Kai Uwe) + 2. MVS native (not OE) beta fixes (Keith Owens) + 3. LynxOS support from Giuseppe Guerrini + 4. MVS already has stat() and fstat() so use 'em (Keith Owens) + 5. MVS fix in readzipfile() for new, unopened dataset without EOF marker + (Keith Owens) + 6. Remove 16-bit stuff from windll/windll.rc (Mike) + 7. Windll: Use hCurrentInst not hInst (Mike) + 8. In util.c compare strchr() return value with NULL (Onno, Frank Donahoe) + 9. unix/unix.c: initialize variable t in ex2in() (Onno, Frank Danahoe) +10. Remove windll/borland subdirectory (Mike) +11. Really fix extra field realloc() for BeOS and MacOS (Christian) +12. Fix the dj2 LFN related access violation bug (Christian, Joe Forster) +13. proginfo/3rdparty.bug: Added more info about other Zip clone's bugs. +14. The global copyright definitions in revision.h now depend on DEFCPYRT + (Christian). +15. tandem/macros: removed obsolete object file references (Christian) +16. fix memory leak with the "filter" patterns (Christian, Leah Kramer) +17. zip.c: completed the support for MacOS specific -N (Christian) +18. reorganized the Mac specific help screen code (Christian) +19. zipup.c: corrected the USE_ZLIB code to emit "stored" entries under + the same conditions as the "native deflate" code (Christian) +20. A couple of vars that will never be negative should be unsigned (Christian) +-------------------------- November 18th 1998 version 2.3h ------------------- + 1. DJGPP: When compressing from stdin don't set binary mode if stdin is + a terminal (E-Yen Tan) + 2. Fix signed/unsigned comparisons in fileio.c, util.c and zipcloak.c + (Frank Donahoe) + 3. Move macgetch() prototype from macos/source/macos.c to macos/osdep.h + (Christian) + 4. _doserrno should have type int, not unsigned int (Christian) + 5. In zipfile.c init a file pointer with NULL to fix gcc warning (Christian) + 6. Upgrade to MacOS beta 7 (Dirk Haase) + 7. Move the #pragma statements from generic sources to cmsmvs.h (Christian) + 8. Support for QNX/Neutrino 2.0 (Chris) + 9. Default to -r in help screen add -R at the bottom (Chris) +10. Clean up Makefile for BeOS R4 on x86 (Chris) +11. Beos: If not storing symlinks store attributes of symlink target (Chris) +12. Use izshr037 (Christian) +13. Remove ZIPERR() macro from in {msdos,win32}/osdep.h (Christian) +14. win32/win32.c: Fix 1-day offset in non-64bit FileTime2utime() (Christian) +15. win32: enable 64-bit FileTime2utime() for MS VC++ >= 5.0 (Christian) +16. cygwin32 only has _P_WAIT (Thomas Klausner) +17. msname() should *really* ignore illegal characters (Thomas Klausner) +18. Fix a missing ')' in Opendir() from win32zip.c (Thomas Klausner) +-------------------------- December 5th 1998 version 2.3i ------------------- + 1. Remove the #pragma statements that were forgotten the first time (Ian) + 2. Remove obsolete macos/source/CharMap.h (Steve Salisbury) + 3. isatty(fileno(zstdin)) in zipup.c should be isatty(zstdin) + (Onno, E-Yen Tan) + 4. several "shut up warnings from compiler" fixes (Christian) + 5. several cosmetic source changes (Christian) + 6. win32: make NTSD handling to be robust against alignment and structure + padding problems (Christian) + 7. Apply don't set binary mode when stdin is a terminal in zipup.c for + MSDOS and human68k (Christian) + 8. Upgrade to MacOS beta 8 (Dirk Haase) + 9. Add callback for WINDLL to handle user termination (Mike) +10. Fix typo in acornzip.c (Darren Salt) +11. acorn/sendbits.s: pass correct parameters to flush_outbuf() (Darren Salt) +12. Fixes for IBM C/C++ 3.6 where time_t is a double (Kai Uwe) +13. Fixes for IBM Visual Age C++ for win32 (Douglas Hendrix) +14. man/zip.1: some version numbers in the text were still "2.2" (Christian) +15. win32/makefile.emx: added a compilation variant that generates + standalone executables (Christian) +16. change __CYGWIN32__ into __CYGWIN__ and add compatiblity definition for + B19 and older (Cosmin Truta) +17. create uniform win32 getch() replacement (Christian) +18. put back in define of USE_EF_UT_TIME in tandem.h (Dave Smith) +19. put back in define of USE_CASE_MAP in tandem.h (Dave Smith) +20. updates to make/macros to allow the object to be licensed (Dave Smith) +21. updates to macros/doit to remove mktime.c (Dave Smith) +22. updates to tandem.c for in2ex/mapname/chmod amendments to match Unzip + (Dave Smith) +23. Use izshr039.zip (Christian) +24. Init filenotes to 0 for the amiga too (Onno) +25. get_filters(): remove one flag=0 statement to make -R work again (Onno) +-------------------------- December 17th 1998 version 2.3j ------------------ + 1. FOPWT defines opening a temp file for writing (Ian) + 2. Remove handling of bits.c from a couple of tandem files (Christian) + 3. A couple of "shut up warnings from compiler" fixes (Christian) + 4. win32/osdep.h: removed duplicate "IZ_PACKED" definition (Christian) + 5. win32/zipup.h: remove invalid "elseif" preprocessor token (Christian) + 6. sync MacOS help screen with other ports (Christian) + 7. get_filters(): set flag to 0 when -R isn't used (Christian) + 8. "local extra != central extra" now has "info" status (Christian) + 9. use windll directory as "home" directory for builds (Mike) +10. CMS/MVS: define FOPWT (Ian) +11. Upgrade to MacOS beta 9 (Dirk Haase) +-------------------------- January 17th 1999 version 2.3k ------------------ + 1. Change FOPW into FOPW_TMP (Christian) + 2. win32: #include uses paths relative to the parent directory (Christian) + 3. Use forward slashes as path separator in #include statements (Christian) + 4. windll: fix descriptions of f{In,Ex}cludeDate (Christian) + 5. win32/makefile.lcc: add some -I options to find files in the + right places (Christian) + 6. Supply default empty IZ_PACKED define (Christian) + 7. windll: Fix some typos, descriptions (Christian) + 8. windll project files: use relative paths, no specific root directory + (Christian) + 9. windll project files: remove link references to import libraries that + are not used by the zip library (Christian) +10. windll: fix potential infinite loop in a VB sample (Mike) +11. windll/windll.txt: remove "may not work with VB" statement (Mike) +12. Multibyte character set support from Yoshioka Tsuneo +13. Theos port from Jean-Michel Dubois +14. Tandem: added simple handling of Enscribe files by converting them into + text type files (Dave Smith) +15. Tandem Extra Field ("TA") containing Tandem File Attributes (Dave Smith) +16. Tandem history file showing background info to (UN)ZIP ports (Dave Smith) +17. create ZIP file on tandem with special file code (1001) (Dave Smith) +18. made tandem.c & tandem.h code completely the same as UNZIP (Dave Smith) +19. unix/configure: move +Onolimit and -Olimit into the machine specific + section (Onno, John Wiersba) +-------------------------- February 21st 1999 version 2.3l ------------------ + 1. Fix qdos Makefile (Jonathan Hudson) + 2. fgets instead of gets in zipnote to fix linker warnings (Jonathan Hudson) + 3. Theos: remove _setargv.c and a reference in zip.c (Jean-Michel Dubois) + 4. Theos README (Jean-Michel Dubois) + 5. interchanged the fRecurse flag values for "-R" and "-r" (Christian) + 6. add "z" pr prefix to MBCS functions to avoid name clashes (Christian) + 7. Whenever the position of the increment operator does not matter, the + INCSTR variant is used, which has been mapped to the {PRE|POS}INCSTR + variant that is more efficient. (Christian) + 8. fixed the "-R" handling in fileio.c, filter() function (Christian) + 9. simplified some THEOS specific code additions (Christian) +10. changed the line break of the compiler version message in version_local() + for MSDOS and Win32 to take into account some verbose compilers (Christian) +11. removed the THEOS changes from ttyio.c. Instead, a THEOS specific + setup was added to ttyio.h (Christian) +12. sync vms/link_zip.com with the corresponding make_zip.com (Christian) +13. added compatibility settings for support of MBCS on Win32 with all tested + compilers to win32/osdep.h +14. added type-casts to isalpha() macro calls (Christian) +15. fixed win32's wild_match which was clobbered by the MBCS addition + (Christian) +16. finished up the "potential infinite loop" problems in the VB sample + that Mike started to repair (Christian) +17. in ziperr.h, AZTEK C might require the false comma that was removed + to satisfy THEOS C (Christian) +18. removed the bogus THEOS specific isdir check in zipup.c (Christian) +19. modified the code for line ending translation to be independent + of the local system's convention for '\n' and '\r'; this allowed + the removal of the THEOS specialities (Christian) +20. Tandem: -B option to zip Enscribe files with no record delimiters + (Dave Smith) +21. Tandem: attempt to catch Large Transfer mode failure (Dave Smith) +22. Theos: Fixed keyboard entry functions. (Jean-Michel Dubois) +23. Theos: workaround for the argument wild card expansion that is bugged + in the standard library. Managed by MAINWA_BUG flag. (Jean-Michel Dubois) +24. Theos: support for filenames and notes with accented characters. + (Jean-Michel Dubois) +25. Upgrade to MacOS final (Dirk Haase) +-------------------------- March 31st 1999 version 2.3m ------------------- + 1. Theos: for relative paths to root directory cause open, fopen and stat + failure, workaround this. (Jean-Michel Dubois) + 2. Theos: when no path is indicated in a file or directory name and the + file or directory doesn't exist in the current directory it looks for + the file or directory in the root directory, workaround this. + (Jean-Michel Dubois) + 3. Corrected some typos and spelling error in macos/HISTORY.TXT; skipped + off invisible trailing whitespace (Christian) + 4. proginfo/extra.fld: added documentation for Tandem and Theos extra + field layout (Christian with Dave D Smith resp. Jean-Michel Dubois) + 5. qdos/Makefile.qdos: The build of ZipCloak requires inclusion of + the crctab object module; qfileio_.o compilation requires the -DUTIL + flag (Christian) + 6. win32: fix incorrect MB_CUR_MAX macro for mingw32 and lcc (Christian) + 7. theos/_fprintf.c, theos/_rename.c, theos/osdep.h: Some function + parameters require the "const" attribute to achieve compatibility + with ANSI C requirements (Christian) + 8. theos/theos.c: map Theos' (No)Hidden file attribute to MSDOS Hidden + bit in the MSDOS part of zipentry header's external attribute field; + 9. theos/stat.h: prevent multiple inclusions +10. Theos: Fixed wild card management for options other than adding + (Jean-Michel Dubois) +11. Theos: Removed modifications of const strings (Jean-Michel Dubois) +12. Split tandem.c up into separate zip/unzip parts (Dave Smith, Christian) +13. Move inclusion of OS specific zipup.h files to tailor.h (Onno) +-------------------------- August 14th 1999 version 2.3n ------------------- + 1. Move inclusion of OS specific zipup.h files back to zipup.c (Onno) + 2. Remove getline() from zipnote.c and use gets() again (Onno) + 3. BeOS PowerPC R4.1 support (Chris) + 4. New DOIT and MACROS files for the tandem port (Dave Smith) + 5. Don't switch the console to binary mode (Michel de Ruiter) + 6. In some circumstances undosm could be freed twice (Mike) + 7. Also define const in tailor.h for ultrix (Onno, Foppa Uberti Massimo) + 8. Tandem: Change zopen in TANZIPC to allow opening of files with missing + alt keys (err 4) (Dave Smith) + 9. Tandem: Assume not DST if can't resolve time (no DST table available) + (Dave Smith) +10. WIN32: skip trailing dots and spaces in getnam (Onno, Dan Kegel) +11. Use ZE_NONE when nothing to freshen or update (Onno, Yuri Sidorenko) +12. Remove tabs from files that don't need them (Onno) +13. Remove tabs and spaces from the end of a text line (Onno) +14. Upgrade macos to 1.04b2 (Dirk) +15. Add -Q documentation to manual page (Jonathan Hudson) +16. Copy hiperspace files instead of renaming them (Keith Owens) +17. Disallow some more characters to appear in DOS filenames when using -k + (Onno, Thomas Klausner) +18. Document missing options and environment variables in the manual (Onno) +19. New acorn/GMakefile to compile with gcc on RISCOS (Darren Salt) +20. ISO 8601 date format support for -t and -tt (Rodney Brown) +-------------------------- September 21st 1999 version 2.3o ------------------- + 1. Sync zip.h license with LICENSE (Onno) + 2. Add copyright notice to README, os2zip.c and os2.zip.h (Onno, Greg) + 3. Fix the ASM variable in acorn/GMakefile (Darren Salt) + 4. Add another requirement to acorn/ReadMe.GMakefile (Darren Salt) + 5. Fix unbalanced parenthesis in vms_get_attributes declaration in zip.h + and move it to vms/zipup.h (Onno, Mike Freeman) + 6. Make a couple of os2 files public domain (Kai Uwe) + 7. Change and rename disclaimer array in revision.h (Onno) + 8. Change copyright array in revision.h (Onno) + 9. macstuff.c copyright is the same as macstuff.h (Christian) +10. WHATSNEW: add ISO 8601 dates supported (Christian) +11. fileio.c - msname(): strip off leading dots, these are illegal for + MSDOS compatible names (Christian) +13. fileio.c - replace(): deactivate "dead" code for CMS_MVS (Christian) +14. man/zip.1: "-$" option is also used for WIN32 ports +15. msdos/msdos.c - version_local(): break the version line for + GNU compilers too (Christian) +16. tailor.h: added typecasts to MBCS macros, to suppress "type mismatch" + warnings (Christian) +17. util.c, zip.h, zipfile.c: ZCONSTify several pointers (Christian) +18. util.c - recmatch(), zip.c - version_info(): add compile time option + WILD_STOP_AT_DIR (Christian, Darren Salt) +19. util.c - envargs(): MBCS related fixes (Christian) +20. win32/lm32_lcc.asm: add TAB characters that are required by the lcc + assembler source parser (Christian) +21. zip.c: fix the "is a console" check (Christian) +22. zipnote.c: use getline() (Christian) +23. zipup.c: use zclose() in case of I/O errors (Christian) +24. zipup.c: use ZE_WRITE when a write error occurs (Christian) +25. win32/win32.c: HAVE_INT64 is used by mingw32 (Cosmin Truta) +26. update shared sources to match izshr041 (Christian) +-------------------------- November 29th 1999 version 2.3 ------------------ + 1. Missing parenthesis in win32/win32.c (Steve Salisbury) + 2. Add Cosmin Truta to proginfo/infozip.who (Onno) + 3. Remove one parenthesis pair too many from vms_get_attributes() declaration + in vms/zipup.h (Mike Freeman) + 4. qdos .s are expected to start with a #, work around it (Jonathan Hudson) + 5. tandem: -B0 should be deflating not storing (Dave Smith) + 6. human68k updates from Shimazaki Ryo + 7. beos Makefile cleanup (Chris) + 8. workaround for fseek to negativate offset behaviour of the RISC OS + SharedCLibrary (Darren Salt) + 9. set file type for RISC OS in zipcloak.c (Darren Salt) +10. change tandem zgetch() to allow crypt version to work (Dave Smith) +11. fix a comment typo in acorn/riscos.c (Christian) +12. fileio.c: two type-cast to shut up noisy compilers (Christian) +13. human68k: fix missing case_flag argmument (Christian) +14. win32/win32.c: remove HAVE_INT64 completely (Christian) +15. zip.c: raise "cannot zip to console" error when stdout IS a tty (Christian) +16. zip.h: don't use dummy argument names in declarations (Christian) +17. Add missing semicolon in fileio.c (Shimazaki Ryo) +18. win32.c: IBMC compiler >= 3.50 have int64 (Kai Uwe) +19. Handle initialization error return value from MVS stat() in procname() + (Keith Owens) +20. Use RISC OS instead of RiscOS in the manual (Darren Salt) +21. Use # instead of ? as single character wildcard on RISC OS (Darren Salt) +22. New windll example.c (Mike) +23. Correct storage of 8-bit char filenames with RSXNT (Burkhard Hirzinger) +24. fix install in unix/Makefile (Santiago Vila, Onno) +25. Fix zip -L output (Santiago Vila, Onno) +26. Ignore unix special files (Jonathan O'Brien) +27. Upgrade to izshr042 (Onno) +28. Make copyright notice the same as in izshr042 (Onno) +29. Make copyright notice in zip.h the same as LICENSE (Christian) +30. Set tempzf to NULL _after_ it has been closed (Chris Kacher) +31. Change email address for Jonathan Hudson (Jonathan Hudson) +32. Remove win32/winzip.c.orig (Steve Salisbury) +33. Use 'Steve Salisbury' throughout the documentation (Steve Salisbury) +34. Change email address for Steve Salisbury (Steve Salisbury) +35. Change email address for Chris Herborth (Chris Herborth) +36. Use zip23 in INSTALL (Roger Cornelius) +37. Use zcrypt28 in INSTALL (Onno) +38. New acorn/srcrename (Darren Salt) +39. amiga/makefile.azt: make clean should remove some more items (Paul) +40. Change email address for Cosmin Truta (Cosmin Truta) +-------------------------- February 11th 2001 version 2.4a ------------------ + 1. Identify newer Borland compilers (Brad Clarke) + 2. Detect Turbo C 2.01 which doesn't have mktime (Brian Lindholm) + 3. Fix the use of -@ together with -i -x (Christian) + 4. Update msdos/README.DOS to match reality (Christian) + 5. win32: use assembler crc32 code (Christian) + 6. windll: _CRTIMP is needed in several function declarations (Christian) + 7. back to zip 2.2 memcompress() behaviour (Kelly Anderson) + 8. new amiga time code based on nih public domain code (Paul Kienitz) + 9. Detect some more Borland C++ builder versions (Brad Clarke) +10. Fix OS/2's extended file attributes compression code (Christian, Kai Uwe) +11. Correct translation of EBCDIC passwords to ASCII (Christian) +12. Attempt at integrating novell patches from Roger Foss (Onno) +13. Use izshr043 (Christian) +-------------------------- July 3rd 2001 version 2.4b ------------------ + 1. Fix OS/2's ACL compression code (Christian, Kai Uwe) + 2. Rename netware subdir to novell (Christian) + 3. Remove -dNETWARE -dDOS from novell Makefile (Christian) + 4. Remove defined(NETWARE) from the sources (Christian) + 5. printf is a macro in glibc 2.2, fix version_local function + (Christian, Matthew Wilcox) +-------------------------- January 13th 2002 version 2.4c ------------------ + 1. Use klist_items when initilizating koff[] in tandem.c (Dave Smith) + 2. Only call NLMsignals() in zip.c when NLM is defined (Mike, Onno) + 3. include riscos.h instead of acorn/riscos.h in acorn/osdep.h (Andy Wingate) + 4. Use izshr044 (Christian) +-------------------------- January 13th 2002 version 2.4d ------------------ + 1. Don't use mmap for stored entries (Christian) + 2. BIG_MEM and MMAP cannot be defined at the same time (Christian) + 3. Allow redirection of version screen to file (Christian) + 4. Fix for OS/2 output redirection bug (Christian, Kai Uwe) + 5. Acorn script for creating self extracting zips (Darren Salt) + 6. Update amiga makefiles to support revised timezone routines (Christian) + 7. Correct memcompress calculation for allocation size (Christian) + 8. Fix FORCE_METHOD debug option for level 1 and 2 (Christian) + 9. Whitespace cleanup in man/zip.1 (Christian) +10. Define IZ_IMP to specify compiler declaration prefixes (Christian) +11. make win32 and msdos version_local() "stdio-macro-safe" (Christian) +12. move tandem's zip specific zipopen to tanzip.c (Christian) +13. first parm is void * in external scope of vms_get_attributes() (Christian) +14. use right novell subdirectory in zipup.c (Christian) +15. update copyright for files modified in 2002 (Onno) +-------------------------- January 19th 2002 version 2.4e ------------------ + 1. Add MacOS X to version_local() (Mark) + 2. unix/configure: Init LFLAGS1 to "", MacOS X doesn't like -s (Onno, Mark) + 3. rename errors array to ziperrors to avoid MacOS X library clash (Mark) + 4. Support for the upx executable packer in DOS makefiles (Christian) + 5. remove obsolete -m486 switch from dos djgpp makefile (Christian) + 6. When using DOS, force the use of msdos style external attributes when + updating zip entries created under another OS (Christian) + 7. os2/makefile.os2: fixed ASFLAGS for watcom16dos (Christian) + 8. Update copyright and ftp address in several files (Christian) + 9. The RISCOS port uses '.' as directory separator, not '/' (Christian) +10. win32/makefile.bor: more options to compile the asm CRC code (Christian) +11. win32: use registry to handle timezones with MS C rtl (Christian) +12. acorn: use recommended practice for calling the linker (Andy Wingate) +13. unix/configure: check if CPP works else use ${CC} -E (Onno, Mark) +14. update versioninfolines in revision.h to match reality (Onno) +-------------------------- February 10th 2002 version 2.4f ------------------ + 1. vms: Zip -V is now able to handle file sizes up to 4Gb (Christian) + 2. vms: Include target environment detection for MMS/MMK (Christian) + 3. Change dummy message from zipcloak (Christian) + 4. acorn: add riscos specific -/ option (Darren) + 5. Update acorn's WILD_STOP_AT_DIR feature (Christian) + 6. acorn: Fix buffer allocation for -/ option (Christian, Darren) + 7. acorn: fix make clean (Andy Wingate) + 8. acorn: use tabs for GMakefile to make GNU make happy (Andy Wingate) + 9. tandem: use nskopen not zipopen (Dave Smith) +10. tandem: allow passing of CRYPT define (Dave Smith) +11. use izshr045 (Christian) +-------------------------- April 1st 2002 version 2.4g ------------------ + 1. acorn: fix assembler and compiler options in makefile (Darren) + 2. use izshr046 (Christian) + 3. MVS: define isatty to 1 to fix screen output (Christian) + 4. tandem: encryption really works now (Dave Smith) + 5. win32: detect Borland C++ builder 6 (Brad Clarke) +-------------------------- April 30th 2003 version 2.4h ------------------ + 1. tandem: fix temporary file contention (Dave Smith) + 2. cmsmvs: generate better filenames with -j (Owen Leibman) + 3. tandem: fix temporary file leftovers (Dave Smith) + 4. solaris: enable large file I/O to break 2G barrier (Rick Moakley, Onno) + +Note: Zip 2.4 was never released. That code was the start of the Zip 3.0 +effort below. Some changes and fixes also made it to the Zip 2.3x releases. + +---------------------- January 21st 2004 version 3.0a ---------------------- +Initial work on Zip 3.0 by Ed Gordon and Rainer Nausedat + 1. Changed some comments to update copyrights (Ed) + 2. Changed text in command line messages from zip 2.4 to zip 3.0 (Ed) + 3. Changes to many files for Zip64 wrapped in ifdef ZIP64_SUPPORT (Rainer) + 4. Attempt to fix buggy Win32 buffered 64-bit calls (Ed) + 5. Add functions to zipfile.c for Little-Endian memory writes (Rainer) + 6. Add functions to zipfile.c for writing Zip64 extra fields (Rainer) + 7. Major changes to putlocal, putcentral, and putend (Rainer) + 8. Fixing -F and -FF for Zip64 postponed (Ed and Rainer) + 9. Command line code replaced. Global table sets options, long options now + supported. Permutes so order of arguments can vary (Ed) +10. Fix bug where not allowed to use -@ with stdout but was with stdin. + Now can read filenames from stdin using -@ and output to stdout and + no longer am allowed to use -@ if reading from stdin (Ed) +11. Replace stat() with zstat(), fstat() with zfstat() and struct + stat with z_stat in Zip64 blocks. Put 64-bit file calls in ifdef + LARGE_FILE_SUPPORT blocks. Can implement Zip64 without > 4 GB + file support but for now need large files for Zip64 support (Ed) +12. Move port-specific code to osdep.h and win32.c (port specific) and + tailor.h (generic) and remove temporary os_io.c. As OF() is + not defined until after osdep.h includes in tailor.h function + prototypes for zfseeko, zftello, and zstat after that in tailor.h (Ed) +13. Settings of ZIP64_SUPPORT and LARGE_FILE_SUPPORT automatic based on + port and version of compiler. Defining NO_ZIP64_SUPPORT or + NO_LARGE_FILE_SUPPORT overrides this (Ed) +14. Bugs compiling scanzipf_fix(...) in zipfile.c and the fix functions could + use rewrite (Rainer and Ed) +15. Add prototype for zfopen for mapping to 64-bit fopen on ports using + inodes but not implemented (Ed) +16. More work on extended local headers and encypted archives (Rainer) +17. Fix DLL files so now compiles (Ed) +18. File size in dll limited to 32-bit in structure. A new DLL api is needed + to return 64-bit file sizes. Current api fixed to return max 32-bit if + more than that (Ed) +19. Add local header Zip64 support and local extra field. Fixed cast + to ulg missed previously that forced zstat to return value mod 4 GB in + zipup.c which kept local header code from seeing actual file size (Ed) +20. Add new option --force-zip64 to force use of zip64 fields. Could + be temporary (Ed) +21. Fix for VB added to api.c that just store the passed strings internally. + Should update api to optionally return file sizes as 64-bit in call back + and to accept RootDir and other strings in same call that zips (Ed) +22. Readme updated to describe new features and mention updated mail group + web links (Ed) +23. Minor bugs in output format found and fixed. Now can add + files > 4 GB to archive and unzip using major unzippers (Ed) +24. If zip used as filter (zip - -) and sizes exceed limits of extended + local header (data descriptor) then set max 32-bit values there. Major + unzippers ignore and use central directory values which are correct. Can + create Zip64 data descriptor using --force-zip64 option but seems no need + for it (Ed) +25. A few bugs in how headers are handled prevented zipping large numbers + of files. Fixed (Rainer) +26. A bit of an attempt to fix -F and -FF. Seems to work but not that + robust. More work needed (Ed) +27. After some cast and other fixes zip compiles on Linux Red Hat 9 using Unix + generic. Added automatic detection of fseeko64 and if detected + sets LARGE_FILE_SUPPORT and setting that sets ZIP64_SUPPORT. Works but + could not test large files on the small system (Ed) +28. Tried to fix bug that prevents zipnotes from compiling when ZIP64_SUPPORT + is set. Still broke. This crashes the Unix Makefile but after + zip is compiled (Ed) +---------------------- May 8th 2004 version 3.0b ---------------------- + 1. Update license headers on more files (Ed) + 2. Change many ZIP64_SUPPORT ifdefs to LARGE_FILE_SUPPORT where appropriate. + Now can test ports using three stages, compile with NO_LARGE_FILE_SUPPORT + (which disables ZIP64_SUPPORT) to test base code, compile with + NO_ZIP64_SUPPORT to test the 64-bit file calls (assuming port sets + LARGE_FILE_SUPPORT) but otherwise use the base code, and without either + to test Zip64 if enabled on port (Ed) + 3. Fix zipnotes bug by moving a ZIP64_SUPPORT block in zipfile.c (Ed) + 4. Add Large File Summit (LFS) code to Unix port to enable 64-bit calls. + Update configure to include test for all needed 64-bit file calls before + enabling LARGE_FILE_SUPPORT for unix port (Ed) + 5. Merge encryption code from zcrypt29 (files from unzip) into zip and + enable by default (Ed) + 6. New man pages for zipnote, zipsplit, and zipcloak (Greg, Ed) + 7. Add encryption notice to crypt.c comments and to version information + in zip.c (Greg, Ed) + 8. Add Russian OEM EBCDIC support when OEM_RUSS defined in ebcdic.h but + Dmitri reports that 0x2F not '/' so make recommended change in cutpath + call in zipfile.c used by -D option (Dmitri - Nov 10 2003 email) + 9. ToDo30 file added to list what's left to do in this release (Ed) +10. Change fopen to zfopen for large file code and map to fopen64 for + Unix (Ed) +11. ftello64 seems broken in zipup.c on Linux (kernel 2.4), returning + negatives past the 2 GB barrier, though ftello64 works in a test program. + Likely error in defines. For now skip ftello64 check for Unix with + LARGE_FILE_SUPPORT. +12. A few updates in Readme. Needs overhaul likely. Also verified mxserver + is gone and replaced with list addresses (Ed) +13. First iterations at updating WinDLL for Zip64 (Mike) +14. Decide to drop backward dll compatibility in favor of a cleaner + dll interface. Decide to add string interfaces for VB (Ed, Mike) +15. Add string interfaces to dll interface to bypass array limitations + imposed by VB and add -x and -i to interface (Mike) +16. Create new VB example using new Zip64 dll interface (Ed) +17. Add O_LARGEFILE define for zopen in unix/zipup.h to enable reading + large files in unix (Ed) +18. Combine ZpSetOptions and ZpArchive dll calls to allow removing all VB kluges + in api.c to work around VB garbage collecting passed strings (Mike) +19. Change new VBz64 example to use updated interface. All works without + kluges (Ed) +---------------------- August 15th 2004 version 3.0c ---------------------- + 1. Add date formats in -t and -tt date errors (Ed) + 2. Add -so to display all available options (Ed) + 3. Many fixes from Dan Nelson to fix some large file support problems and + add large file support to a few ports. Main change is rather than use + explicit 64-bit calls like fopen64 now set 64-bit environment and use + standard calls. Also add a define for 64-bit printf format used to + print 64-bit stats (Dan, Ed) + 4. Changes to Unix config based on suggestions from Dan Nelson. Check + if off_t is at least 64 bit (Dan, Ed) + 5. Add -- to get_option. Any arguments after -- on command line now + read as paths and not options (Ed) + 6. Add extended help (Ed) + 7. Change add_filter flag parameter from char to int as some compilers have + problems with char arguments (Ed) + 8. Changed filter() to do R and i separately so i has precedence over R (Ed) + 9. Split variable t in zip.c into t (off_t) and tf (ulg) (Ed) +10. Add quotes to zipname in check_zipfile for MSDOS to allow spaces in + archive path given to unzip to test ( , Ed) +11. Move zip.h include before ctype.h include in trees.c and zipup.c as + when ctype.h is first and using 64-bit environment at least on unix port + found it defines off_t as 4 bytes in those files as off_t is defined as + 8 bytes in other files and this changes the size of the zlist structure + which is not good (Ed) +12. Add default 64-bit file environment to tailor.h if LARGE_FILE_SUPPORT + is set but no port 64-bit file defines are set up earlier in the file. + Should allow other ports to set LARGE_FILE_SUPPORT on the compiler + command line to test if the standard defines work (Ed) +13. Adjust binary detection in trees.c by changing 20% binary (4 out of 5 + ascii) that used >> 2 to 2% (64 out of 65) using >> 6 instead. + trees.c (Ed) +---------------------- November 12th 2004 version 3.0d ---------------------- + 1. Add global variable for EncryptionPassword in VBz64 example and + some other password callback cleanup (Ed) + 2. Add -W option to turn on WILD_STOP_AT_DIR where wildcards will not + include directory boundaries in matches (Ed) + 3. Add -nw option "no wild" to completely disable wildcards in MATCH + function. Allows a list of files to be read in without worrying about + wildcards or escapes (Ed) + 4. Add -s option split-size but not implemented (Ed) + 5. Add -sp option split-pause but not implemented (Ed) + 6. Add changes for WiZ including moving Win32 64-bit wrappers into + win32i64.c to avoid naming conflict between libraries in WiZ (Mike, Ed) + 7. Some large file fixes in crypt.c (Ed) + 8. Add new error code ZE_UNSUP for unsupported compiler options. Add + check of size of zoff_t in zip.c when LARGE_FILE_SUPPORT enabled (Ed) + 9. Changed ZE_UNSUP to ZE_COMPERR to avoid conflict with unzip (Ed) +10. On VMS (sufficiently recent, non-VAX), DECC$ARGV_PARSE_STYLE is set + automatically to preserve case of the command line if the user has + SET PROCESS /PARSE = EXTEND. This obviates quoting upper-case + options, like -V, when enabled. VMS.C (Steven Schweda (SMS)) +11. On VMS, building with macro VMS_PRESERVE_CASE defined preserves case + of names in archive, instead of forcing lower-case (the former and + current default behavior). VMSZIP.C (SMS) +12. On VMS, in some of the simplest cases, ODS5 extended file name + escape characters ("^") are removed from names in archive. + VMSZIP.C (SMS) +13. On VMS, fixed a problem in some cases with mixed-case directory + names, where too much of the directory hierarchy was included in the + path names in the archive. VMSZIP.C (SMS) +14. On VMS, minor changes for large file support (long -> zoff_t). + VMSZIP.C (SMS) +15. On VMS, changed some structure declarations to typedefs, and + rearranged to simplify #if's and reduce potential name conflicts. + VMS.H, VMS_IM.C, VMS_PK.C (SMS) +16. On VMS, reformed -V (/VMS) processing. Added -VV (/VMS=ALL). + Removed some sign bits to accomodate files bigger than 2GB. + CMDLINE.C, VMS_IM.C, VMS_PK.C, ZIP.C, ZIP_CLI.CLD, ZIP_CLI.HELP, + ZIPUP.H (SMS) +17. Update command line options to support -VV as distinct option (Ed) +18. More VMS changes (SMS) +19. Add zoff_t format function (SMS) +20. On VMS, when -b was not used, temporary archive files were always + created in the current default directory, rather than in the archive + file destination directory. VMS now uses its own tempname() + function. FILEIO.C, VMS.C (SMS) +21. Remove using FNMAX for path size in a few places including filetime.c + to avoid exceeding limit (based on fixes from Greg and others) (Ed) +22. Add port atheos (Ruslan Nickolaev, Ed) +23. Bug fix adds different extra fields for local and central in VMS (SMS) +24. Now short options also take optional values as next argument (Ed) +25. Change -dd to control -v dots (SMS, Ed) +26. On VMS, a new open callback function senses (where supported) the + process RMS_DEFAULT values for file extend quantity (deq), + multi-block count (mbc), and multi-buffer count (mbf), and sets the + FAB/RAB parameters accordingly. The default deq is now much larger + than before (16384, was none), and the default mbc is now 127 + (up from 64), speeding creation of a large archive file. Explicitly + set RMS_DEFAULT values override built-in defaults. OSDEP.H, VMS.C + (SMS) +27. VMS CLI definitions and CLI help have been updated, and may be + approximately correct. CMDLINE.C, ZIP_CLI.CLD, ZIP_CLI.HELP (SMS) +28. The man file zip.1 updated and Makefile updated to generate manual + pages for zipcloak.1, zipnote.1, and zipsplit.1 (Ed) +---------------------- July 23rd 2005 version 3.0e ---------------------- + 1. Debian patch 004 - apply 2.4i configure changes from Onno to remove + need for -fno-builtin in unix/configure (Onno, Ed) + 2. Debian patch 005 for bug 279867 - fix bug that could crash on large paths + and create security problem. Apply patch changes from Greg (Greg, Ed) + 3. SourceForge patch 1074363 - add win32i64.c to win32/makefile.w32 (Ed) + 4. Add check when not ZIP64_SUPPORT in scanzipf_reg() in zipfile.c if + Zip64 archive being read (Ed) + 5. Renamed fzofft() used to format zoff_t values to zip_fzofft() to remove + conflict when combined with UnZip in WiZ (Mike) + 6. Add check in scanzipf_reg() in zipfile.c if Zip64 archive being read (Ed) + 7. Fixes for amiga/makefile.azt to define directory for object files (Paul) + 8. Define prototypes for local functions optionerr, get_shortopt and + get_longopt in fileio.c. Define err argument of optionerr as ZCONST (Paul) + 9. Add help_extended and DisplayRunningStats prototypes, fix other prototypes + in zip.c (Paul) +10. Split int kk off of k for argument types (Paul) +11. Aztec #endif quirk fix in zip.c for Amiga (Paul) +12. Add detection of binary in first buffer read from file in zipup.c to avoid + a -l or -ll translation on binary file. Not perfect but at least should + catch some binary files (Ed) +13. Remove check for >= 128 from binary check in zipup.c as <= 6 enough for + signed char (SMS, Ed) +14. SF Bug 1074368 - check for empty zip file in readzipfile() in zipfile.c + (Christian d'Heureuse, Ed) +15. Add error exit to prevent archive corruption when updating a large-file + archive with a small-file program. Add ZE_ZIP64 error. + ziperr.h, zipfile.c (SMS) +16. Change percent() in zipup.c to do rounding better, handle cases near limits + while rounding, and allow negative percent returns (SMS, Ed) +17. Add function ffile_size() in zipfile.c but in #if 0 block until determine + if works on all ports under all conditions. Currently only used for size + check for Zip64 archive detection if compiled without ZIP64_SUPPORT and + this check may already be handled in scanzipf_reg() and should be added to + scanzipf_fix() when that is updated (SMS, Ed) +18. Change >>1 to /2 in zipsplit.c to allow for negative percent returns (SMS) +19. Add type uzoff_t for unsigned zoff_t things. Should clean up some casting + (Ed) +20. Based on discussions with other development groups, when data descriptors + (extended local headers) are used, force to Zip64. This is compatible + with other unzips and does not require a change of the AppNote, but the + resulting archive requires Zip64 to read. Using standard data descriptors + would mean that the zip operation would fail if a Zip64 entry was + encountered. See zipfile.c (Ed) +21. Add define SPLIT_SUPPORT to enable splits. The command line options are + done and the globals are set up but nothing more. globals.c, zip.h, and + zip.c mainly (Ed) +22. Create spanning signature at beginning of archive when splitting enabled. + If reading a split archive skip the spanning signature unless creating a + split archive. zip.c, globals.c (Ed) +23. Start implementing split archives. Define two methods. split_method = 1 + updates local headers and is the most compatible but requires updating + previous splits. split_method = 2 uses data descriptors and should work + for streams and removable media but may not be as compatible with other + zip applications. (In part based on previous discussions with Rainer.) + Updated global variables to include bytes written to just the current + entry in the current split. zipfile.c (Ed) +24. Add note about output redirection to zip.1 (?, Ed) +25. Remove num < 0 check as num now unsigned. util.c (SMS, Ed) +26. Change lastchar to lastchr in fileio.c in places to avoid function by same + name (SMS, Ed) +27. Moved #endif /* !WINDLL */ in zip.c (Mike) +28. Account for vms directory version being ;1. vmszip.c (SMS) +29. Fix Zip64 check in scanzipf_reg to use the buffer. zipfile.c (Ed) +30. Default define size_t (for use by Steve's ffile_size() function). tailor.h (Ed) +31. Enable Steve's ffile_size() function and enable large file check. It + currently does not allow file sizes over 2 GB but the code is not supporting + it anyway without large file support. Should remove that part of the check + when the casts are fixed. zipfile.c (Ed) +32. Fixes for djgpp. Now compiles with djgpp 2 (Ed) +33. Add new VC6 projects for win32 and windll (Cosmin) +34. Convert some variables in zipsplit.c from ulg to zoff_t so compiles (Ed) +35. Add wildcards to extended help. zip.c (Ed) +36. For optional option value now '-' is same as missing value. fileio.c (Ed) +37. Remove extra free() from -dd option switch. zip.c (Ed) +38. Change write_unsigned_to_mem() to write_ulong_to_mem() and write_short_to_mem() + to write_ushort_to_mem(). zipfile.c (Ed) +39. Create new append to mem functions. zipfile.c (Ed) +40. Change zlist nam and ext from extent to ushort as that is what gets written. + zipfile.c (Ed) +41. Change GetSD to use ush instead of size_t. win32/win32zip.c (Ed) +42. Change PutLocal(), PutExtended(), PutCentral(), and PutEnd() to write to + memory and then write the block at once to the file. zipfile.c (Ed) +43. Change zcomlen from extent to ush, other extent conversions. zipfile.c, + globals.c, zip.h (Ed) +44. Add is_seekable() and global output_is_seekable. Do seekable check + when output file is opened. zipup.c, globals.c, zip.h, zip.c (Ed) +45. Do not increment files_so_far and bytes_so_far if file could not be read. + zip.c (Ed) +46. If force_zip64 set, only force compressed size in central directory to Zip64 + instead of all entries (csize, usize, off, disk) in Zip64 extra field. This + fixes inconsistent handling of disk numbers. zipfile.c (Ed) +47. Add end status if displaying running stats and not all files were read. + zip.c (Ed) +48. Change force_zip64 to zip64_archive in putend(). zipfile.c (Ed) +49. Enable the i686-optimized code by default. crc_i386.S, + win32/crc_i386.asm, win32/crc_i386.c (Cosmin) +50. Document and implement a new text detection scheme provided by Cosmin in + set_file_type(). Should be able to handle UTF-8 and some other character sets. + proginfo/txtvsbin.txt, trees.c (Cosmin, Johnny, Christian) +51. Update binary detection for -l and -ll to use Cosmin black list. zipup.c (Ed) +52. Change ZE_BIG to include read and write. ziperr.h (Ed) +53. If archive not seekable then use data descriptors. If ZIP64_SUPPORT always + create Zip64 data descriptors and add a Zip64 extra field to flag it is + a Zip64 data descriptor. This is klugy but should be compatible with other + unzips. See the note in zipfile.c for details. (Ed) +54. Use ush for comment length in putend(). Instead of extent use ush for + zcount and fcount same as in zip file. zip.h (Ed) +55. Update VB readme. windll/VB/readmeVB.txt (Ed) +56. Change (INSTALL) to (INSTALL_PROGRAM). unix/Makefile (, Ed) +57. During update the file and byte status counts were off. Fixed by not coun- + ting files copied from old to new as those are not in totals. zip.c (Ed) +58. Change from -b to -bx for nroff of manuals to text files. unix/Makefile (Ed) +59. Add cygwin to makefile. unix/Makefile (, Ed) +60. Fix bug where files to delete not added to list. zip.c (Ed) +61. Fix delete stats. zip.c (Ed) +62. Increment version of crypt to 2.10. Update default behavior notes. + crypt.c, crypt.h (Paul, Christian) +63. Format changes, add parentheses to zfseeko(), fix output bytes, add ifdef + blocks for ZIP10, fzofft formatting, casts. crypt.c (Christian) +64. Cast block_start to unsigned. deflate.c (Christian) +65. Let -R patterns match in subdirectories. Update filter() to use switch, + use global icount and Rcount, handle subdirectories, update icount and + RCount in filterlist_to_patterns(). fileio.c, zip.c, zip.h, globals.c + (Christian) +66. Enclose option -! and use_privileges under NTSD_EAS guard. globals.c, + zip.c, zip.h (Cosmin) +67. Updates to version, copyright, license. [I did not split the copyright + to 2 lines as it already takes up space on the help screen. Ed] + revision.h (Christian) +68. Add ZCONST to some read-only string pointer arguments in function + declarations. zipcloak.c, zipnote.c, zipsplit.c, zip.c, zip.h (Christian) +69. Fix byte counts on exit in zipcloak() and zipbare() to fix zipcloak bug + (Christian) +70. Modified zipnote.c to use WRBUFSIZ to handle line widths of at least 2047 + characters in write mode (Christian) +71. Change simple() and greedy() from zoff_t to uzoff_t. zipsplit.c (Christian) +72. Remove duplicate copyright notices. zipsplit.c (Christian) +73. Remove export notice from help page. Move notice to bottom of license + page. zipcloak.c (Ed) +74. File USexport.msg export history added. (Greg) +75. Added support for VMS ODS5 extended file names. (Eight-bit only, no + Unicode.) VMS name character "/" is mapped to Zip name character + "?". New command-line options -C[2|5][-] (/PRESERVE_CASE[=opts]) + control name case preservation and/or down-casing. globals.c, + zip.c, zip.h, vms/cmdline.c, vms/vms_im.c, vms/vms_pk.c, vms/vms.c, + vms/vmszip.c, vms/vms.h (SMS) +76. New VMS option -ww (/DOT_VERSION) stores version numbers as ".nnn" + instead of ";nnn" [changed from -Y to -ww (Ed)]. zip.c (SMS) +77. Changes to vms_open(). vms/vms_im.c, vms/vms_pk.c +78. Changes to vms_read(). vms/vms_pk.c (SMS) +79. Documentation updates. vms/vms_zip.rnh (SMS) +80. Minor updates. vms/zip_cli.help, vms/cmdline.c, vms/vms_zip.rnh (Ed) +81. Changes to vmsmunch(). vms/vmsmunch.c (SMS) +82. Do some updating of VMS options. vms/zip_cli.cld (SMS) +83. Moved the VMS-specific ziptyp() function from zipfile.c to vms/vms.c + to segregate better the RMS stuff. (SMS) +84. Put 64-bit calls in ZIP64_SUPPORT ifdef blocks, change some long parameters + for append to memory block functions to ulg, remove redundant includes, + add OFT protos to some functions with parameter types that get promoted + like ush to avoid warnings in VMS. zipfile.c (SMS) +85. Use zip_fzofft() to format number. zipsplit.c (SMS) +86. Add file_id.diz from Zip 2.31 (?, Ed) +87. Update install from Zip 2.31 (?, Ed) +88. Update license from Zip 2.31. License (?, Ed) +89. Update Readme.cr from Zip 2.31 (?, Ed) +90. Add 64-bit assembler for Win32 from Zip 2.31. win32/makefile.a64, + win32/readme.a64, win32/gvmat64.asm (?, Ed) +91. Update Readme (Ed) +92. Update headers. crctab.c, crc32.c, deflate.c, ebcdic.h, fileio.h (Ed) +93. Option for extra verbose VMS, change DIAG_FLAG from verbose to + (verbose >= 2). vms/vms.c (SMS) +94. Update copyright header. qdos/qdos.c (Christian, Ed) +95. Change exit(0) to exit(ZE_OK). qdos/qdos.c (Christian) +96. Change ulg to unsigned long. tailor.h (, Christian) +97. Default uzoff_t to unsigned long long if LARGE_FILE_SUPPORT manually + enabled for an otherwise unsupported port. tailor.h (Ed) +98. Update copyright header. tailor.h (Ed) +99. Change EXIT(0) to EXIT(ZE_LOGIC) for ziperr recursion. zip.c (Christian) +100. Change EXIT(0) to EXIT(ZE_OK) for successful returns. zip.c, + zipcloak.c (Christian) +101. Update license. zip.h (Christian) +102. Initialized mesg in zipcloak.c, zipnote.c, zipsplit.c to fix access + violation crashes. (Christian) +103. Added -q (Quiet mode) option to zipcloak, zipnote, zipsplit. (Christian) +104. Add proto of mb_clen(). fileio.c (Cosmin) +105. Synchronize ttyio.c and ttyio.h with the unzip-5.52 source. (Cosmin) +106. Control the POSIX emulation provided by some Unix-on-Windows compiler + distributions, such as Cygwin, via the FORCE_WIN32_OVER_UNIX macro. + tailor.h, win32/Makefile.gcc (Cosmin) +107. Remove getenv() declaration. util.c (Cosmin) +108. Fix definitions of zopen and zstdin. unix/zipup.h (Cosmin) +109. Enable binary file operations for DJGPP and Cygwin. unix/osdep.h (Cosmin) +110. Remove -DMSDOS from CFLAGS; use correct dependency in target crc_i386.obj. + win32/makefile.w32, win32/makenoas.w32 (Cosmin) +111. Update win32/makefile.bor and win32/makefile.gcc (Cosmin) +112. Put mktemp() declaration inside the NO_PROTO guard. tailor.h (Cosmin) +113. Use the right type (DWORD) for volSerNo, maxCompLen and fileSysFlags + in FSusesLocalTime(). win32/win32.c (Cosmin) +114. Set the "zip Debug" configuration as default. win32/vc6/zip.dsp (Cosmin) +115. Define ASM_CRC by default. win32/osdep.h (Cosmin) +116. Avoid using file names that are distinguished solely by letter case; + e.g. crc_i386.S and crc_i386.s. unix/Makefile (Cosmin) +117. Stylistic fix inside ex2in(). unix/unix.c (Cosmin) +118. Change zlist dsk from ush to ulg to support Zip64 and added casts in + zipfile.c to write ush. zip.h, zipfile.c (Christian, Ed) +119. Conditionally apply S_IFLNK to support DJGPP. unix/unix.c (Cosmin) +120. Change -dd [siz] (display dots, set optional dot size) to the options + -dd (turn dots on, use 10 MB default) and -ds siz (set dot size). + Found that using -dd with an optional value got confusing as detection + of an optional argument, when the next argument was not either an option + or the end of the line, was easy to overlook. Easier to avoid optional + values. zip.c (Ed) +121. Change text output of manual pages to zip.txt, zip.txt, zipcloak.txt, + zipnote.txt, zipsplit.txt. unix/Makefile (Christian, Ed) +122. Change comments using // to /* */ format. api.c, zip.c (Christian) +123. Add support for signals SIGABRT, SIGBREAK, SIGBUS, SIGILL, and SIGSEGV + to utilities. zipcloak.c, zipnote.c, zipsplit.c (Christian) +124. Update ToDo30.txt file (Ed) +125. Delete old Manual file (Ed) +126. Update WHERE from Zip 2.32 (Ed) +127. Change description of dot-size. zip.c (Ed) +128. Change VMS to use -ds to set dotsize. vms/cmdline.c (Ed) +129. Update manuals. man/zip.1, man/zipsplit.1, man/zipnote.1, + man/zipcloak.1 (Ed) +130. Detect i586, i686 and Cygwin in version_local(). unix/unix.c (Cosmin) +131. Add clean target. win32/makefile.w32, win32/makenoas.w32 (Cosmin) +132. Changed most 64-bit size/offset variable declarations (like zoff_t) + into "unsigned" type (like uzoff_t), for better backward compatibility + with non-ZIP64_SUPPORT setups where "ulg" was used for these variables. + deflate.c, fileio.c, globals.c, trees.c, vms/vms_pk.c, win32zip.c, + zip.c, zip.h, zipfile.c, zipup.c (Christian) +133. Add (ulg) cast to strstart in flush_block. deflate.c (Christian) +134. Updated Win32 LARGE_FILE_SUPPORT setup for Watcom and MinGW. + tailor.h, win32/osdep.h (Christian) +135. Add attempt count to tempname(). fileio.c (Christian) +136. Fixed size counter handling in debug code for Zip64. trees.c (Christian) +137. Moved cryptnote display text definition into revision.h, like was done + in Zip 2.31. zip.c, revision.h (Christian) +138. Add ZCONST. fileio.c (Christian) +139. Removed earlier change in trash() where ASCII-containing iname was + searched for native-coded '/' characters. [Added note but left as + changed 5/20/05 EG] zipfile.c (Christian) +140. Change zipup size error message to use zip_fzofft(). zipup.c (Christian) +141. Updated win32/makefile.wat to enable Zip64 support and use directory + for intermediate files. (Christian) +142. Change fcount and zcount from ulg to extent as extent is used internally, + but Zip64 standard supports up to ulg. Add note to zip.h. globals.c, + zip.h (Christian) +143. Define NO_W32TIMES_IZFIX in compile options when appropriate. Add + version information for USE_ZLIB compiler option. zip.c (Christian) +144. Add support for SIGABRT, SIGBREAK, SIGBUS, SIGILL, and SIGSEGV signals. + zip.c (Christian) +145. Add display-usize option to show uncompressed size. zip.c (Ed) +146. Add many descriptions to options table. zip.c (Ed) +147. Remove -R from help screen as on extended help screen. zip.c (Ed) +148. Add basics to extended help. zip.c (Ed) +149. Fix checks in scanzipf_reg() for empty file since cenbeg now unsigned. + Change buffer from t to b in small big check. Back up after small + zip big archive check. zipfile.c (Ed) +150. Change Zip64 not supported warning in scanzipf_reg(). zipfile.c (Ed) +151. Fix bug where local and central headers were not matching when compiled + with NO_LARGE_FILE_SUPPORT. Restored order of zlist structure elements + to match order of local header as scanzipf_reg() compares it as an + array of bytes to the local header. Gag. It needs fixing but at least + it works as intended now. zip.h, zipfile.c (Ed) +152. Minor fix from 10000 to 10 K for WriteNumString(). util.c (Ed) +153. Add overflow check to file_read(). zipup.c (SMS) +154. Add parameter p1 product specification. vms/collect_deps.com (SMS) +155. VMS changes. vms/descrip_mkdeps.mms (SMS) +156. Change zoff_t to uzoff_t and unsigned int to size_t. vms/vms_im.c, + vms/vms_pk.c (SMS) +157. Fix ; that was : at end of line. Fix DisplayNumString() prototype. + zip.h (Ed) +158. Get rid of leading blanks in DisplayNumString(). util.c (Ed) +159. Reset dot_count each file. zipup.c (Ed) +160. Minor changes to extended help. zip.c (Ed) +161. Move defines into DEFINED_ONCE block. api.h (Mike) +162. Add Still Remaining And Planned For Zip 3.0 section. WhatsNew (Ed) +163. Delete quotes around CHANGES. Readme (Ed) +164. Add -lf, open file at path and use for logging, -la, append to + existing logfile, and -li, include informational messages, options. + globals.c, zip.h, zip.c (Ed) +165. Update extended help to include logging. zip.c (Ed) +166. Add support for required short option value in form -o=value as optional + does. fileio.c (Ed) +167. If bytes_total is smaller than bytes_so_far for some reason then display + negative of bytes_to_go. This can happen if files grow in size after all + the sizes are initially added up. zip.c (Ed) +168. Use usize from filetime for adding to bytes_total when updating instead + of size in old entry. zip.c (Ed) +169. Change status counts files_so_far and bytes_so_far to include bad files + so the status counts end at the end but add bad_files_so_far and + bad_bytes_so_far to track bad files. After minor fixes it looks like + the counts remaining at the end are correct, even when some files are + not readable. Update bad file warnings. zip.c, zip.h, globals.c, + zipup.c (Ed) +170. Add uq for unsigned q in zipup(). Initialize z->len in case an error + later so have a valid size. zipup.c (Ed) +171. Check noisy in DisplayRunningStats() so logging is independent of it. + zip.c (Ed) +172. Add check in DOS for windows and if running DOS version on Windows warn + user. zip.c, msdos/msdos.c, msdos/osdep.h (Johnny) +173. Add errno.h for strerror(errno) call. zip.c, zipup.c (SMS) +174. Fix log problem if using -q option. zipup.c (Ed) +175. Change "Far char" to "char Far" as Far is a qualifier not for the char + type but the storage allocation of the array. fileio.c (Christian) +176. Update note on extent. globals.c (Christian, Ed) +177. Remove extra USE_ZLIB. zip.c (Christian) +178. Add note for the OEM_RUSS '/' bug. Need to look at later as it seems + the Russian bug remains unfixed. zipfile.c (Christian, Ed) +180. So byte counts always come out even, create good_bytes_so_far to + count bytes read in and convert bytes_so_far to use the counts + from the initial scan. If files change during the zip operation + good_bytes_so_far will change and not match bytes_so_far. + zip.h, globals.c, zip.c (Ed) +181. Changes to extended help. zip.c (Ed) +182. Update WhatsNew (Ed) +183. Update DLL resource copyright. windll.rc, windll.aps (Ed) +184. Add directory search improvements to Win32 (within recursion, reuse + attribs from directory lookup to avoid calling stat()). Add + getd_attribs(), procname_win32(). win32/win32zip.c (Johnny) +185. Cache result of IsFileSystemOldFAT() to avoid repetitive system calls + for identical information. win32/win32.c (Johnny) +186. Add optimization to dosmatch(): apply alternate shortcut code when the + pattern to match consists of one multichar wildcard ('*') followed + by a fixed string. util.c (Johnny) +187. Move DOS check_for_windows() checks to Help and Version and errors + only. Shorten message to one line. zip.c, msdos/msdos.c (Ed) +188. Define WIN32_OEM to enable oem ansi conversions for more than RSXNT. + Not yet fully implemented. win32/win32.c, win32zip.c, zip.c, + zipfile.c (Ed) +189. Directory search improvements for MSDOS. msdos/msdos.c (Johnny) +190. Add caching of directory information. If pattern is just *string no + need to recurse. win32/win32.c (Johnny) +191. If wild_stop_at_dir then do recurse to handle cases like a/b/*.txt. + win32/win32.c (Ed) +192. Additional improvements to directory search speedups, including + a) MSDOS port fixes for Turbo C++ compiler + b) In both Win32 and MSDOS, change getDirEntryAttr() into macro, + saving one function call overhead + e) Add explaining comment to optimized procname_{local} code + f) In util.c, move "*literal" pattern-matching optimization from + dosmatch() to recmatch(). Advantages: + - optimization used for all systems + - optimization applied to all occurences where a "*" is last wildcard + in pattern + - "dosmatch()" only preconditoning wrapper for matching workhorse + "recmatch()", it should not implement matching algorithms itself + - optimization not applied for WILD_STOP_AT_DIR option + g) >>>disabled<<< "*literal" optimization for all MBCS-aware environments, + because suspect that supplied optimization code is not MBCS-clean + (for details see the comment within the patch), so IS NOT USED for + win32 port! Can force activation of match optimization by specifying + conditional compilation symbol TEST_FOR_MBCS_CLEAN. + (Christian) +193. Add and move comments, implement changes for directory search improvements + in Zip 3.0 util.c (Ed) +194. In win32/win32.c, IsFileSystemOldFAT(), add declarations of static caching + variables where missing to fix win32 port compilation bug (Christian) +195. Correct changed arguments in RSXNT-only character set conversion + call. win32/win32zip.c (Christian) +196. Implement Directory Search improvements from Zip 2.32. win32/win32zip.c + (Johnny, Ed) +197. Debian Bug #312090 fix. Reworded man page to give multiple examples of + recursion, not just zip -r foo foo. man/zip.1 (Ed) +198. Change "-Aa -D_HPUX_SOURCE +e" to -Ae for HP. "HP-UX with the HP compiler + and on AIX 4.2.0. AIX 5.1 with gcc-3.4.3 (32-bit) and Darwin built fine + - though AIX 5.1 needed CC=gcc make -e ... to find gcc. According to the + HP-UX man page -Ae is equivalent to -Aa -D_HPUX_SOURCE +e it seems the + +e is needed and -Ae is more terse anyway." Expression generated before + was too long. unix/configure (Rodney Brown) +199. Add support for osf4.0f that does not have fseeko or ftello but has 64-bit + fseek and ftell though. tailor.h (Rodney) +200. Fix unsigned char to char in recmatch(), add casts for compares. util.c + (Ed) +201. Fix for alpha off_t long long. unix/osdep.h (Rodney) +202. Change shmatch() from uch to char and change parameters to recmatch(). + Change dosmatch(). util.c (SMS, Rodney, Ed) +203. Add local for DisplayRunningStats(). zip.c (Rodney, Ed) +204. Disable unused append_ubyte_to_mem(). Fix error messages in other append. + zipfile.c (Rodney, Ed) +205. Delete unused getDirEntryAttribs(). msdos/msdos.c (Christian) +206. Change warning when running msdos version on Windows. msdos/msdos.c (Ed) +207. Change recmatch() to support MBCS matching. util.c (Christian) +208. Update WhatsNew (Ed) +209. Update Readme (Ed) +210. Format Readme to fit in 80 character lines (SMS, Ed) +211. Rename install.vms to install_vms.txt. vms/install_vms.txt (SMS) +212. Add reference to vms/install_vms.txt in INSTALL (SMS) +213. Update INSTALL (Ed) +214. Remove ALT_NEXTBYTE and Building UnZip sections as no longer needed. + vms/notes.txt (SMS, Ed) +215. Add note to TODO (Ed) +216. Update Makefile message to suggest using generic. unix/Makefile (Ed) +217. Update text output of manual. zip.txt (Ed) +218. Update VMS section. INSTALL (SMS, Ed) +219. Minor changes in vms/install_vms.txt (SMS, Ed) +220. Update VMS install information. INSTALL, vms/install_vms.txt (SMS, Ed) +221. Do not use _stati64 under Cygwin. win32/osdep.h (Cosmin) +222. Add note to Makefile to use generic first. unix/Makefile (Ed) +223. Add Test option for VMS CLI. vms/cmdline.c (SMS, ?) +224. Add noconfirm to deletes, define symbol edit. vms/descrip.mms (SMS) +225. Changes to vms/install_vms.txt (SMS) +226. Add note on symbols to VMS. INSTALL (SMS) +227. Update license headers. vms/osdep.h, vms/vms.h, vms/vmsmunch.c, + vms/zipup.h, vms/vmszip.c, vms/vms.c, vms/vms_im.c, vms/vms_pk.c, + vms/command.c (Ed) +228. Add stsdef.h include for VMS and convert unzip test return to VMS + result for VMS. zip.c (SMS) +229. Add const to ziperr(). amiga/amiga.c (Paul) +230. Clean up makefile. amiga/makefile.azt (Paul) +231. Don't try Amiga large file support. amiga/osdep.h (Paul) +232. Add note on -V and -VV. vms/notes.txt (SMS) +233. Small update. vms/zip_cli.help (SMS) +234. Format Windows warning message. msdos/msdos.c (Christian) +235. Format changes. util.c (Christian) +236. Update VMS. INSTALL (SMS) +237. Add creation of intermediate object directories. msdos/makefile.wat + (Christian) +238. Add void * cast. msdos/msdos.c (Christian) +239. Add include for mktemp(). msdos/osdep.h (Christian) +240. Fix __RSXNT__ and WIN32_OEM define blocks. win32/win32.c (Christian) +241. Fix __RSXNT__ and WIN32_OEM define blocks. win32/win32zip.c (Christian) +242. Add != NULL to check. zip.c (Christian) +243. Fix WIN32_OEM. zipfile.c (Christian) +---------------------- October 11th 2005 version 3.0f01 ---------------------- +(the internal betas may be merged later) + 1. Add DSEG for Watcom data segment. msdos/makefile.wat (Christian) + 2. Add -zq and use assembler. os2/makefile.os2 (Christian) + 3. Update header. os2/match32.asm (Christian) + 4. Change len from int to unsigned int. os2/os2.c (Christian) + 5. In GetLongPathEA() limit tempbuf to CCHMAXPATH. os2/os2.c (Christian) + 6. Add DWATCOM_DSEG to use data segment. win32/makefile.wat (Christian) + 7. Update header and add DGROUP. win32/match32.asm (Christian) + 8. Add UNICODE_SUPPORT define. zip.h, zip.c (Ed) + 9. Add oname to f and z structs for the display name to use in messages. + Change z->zname to z->oname in messages. fileio.c, zip.c, win32zip.c, + zipup.c, zipfile.c, zip.h (Ed) +10. Move multi-byte defines to make global (they were needed with wide + characters but that was taken out and left them where they are). + fileio.c, zip.h +11. Add copy_args(), free_args(), and insert_arg() to create copy of argv + that can free() and to support inserting "@" in get_option for lists. + fileio.c, zip.h +12. Insert arg "@" after list if not followed by option. fileio.c +13. Add args variable and copy argv to args so can use insert_arg(). zip.c +14. Add MKS Korn Shell note. zip.c +15. Change cast of option in add_filter() calls from char to int. zip.c +16. Implement multi-byte version of Unicode support. To support Win32 NT + wide calls will require additional work not planned for this release. + Changes include (Ed): + - Add use_wide_to_mb_default flag. globals.c, zip.h + - Add compiler UNICODE_SUPPORT version information. zip.c + - Add uname to f and z structs for UTF-8 name. zip.c + - Moved some defines out of ZIP64 section. zipfile.c + - Add define UTF8_PATH_EF_TAG for Unicode Path extra field. Currently + the tag is 0x7075 which is 'u' 'p' for Unicode path and seems + free according to the AppNote. The extra field is + tag (2 bytes 'u' 'p') + size (2 bytes) + Unicode Path size (2 bytes) + unused (2 bytes set to 0) + unused (2 bytes set to 0) + Unicode path (variable) + The unused locations also serve as a check in case the tag is in + use already. + - Add add_Unicode_Path_local_extra_field() and + add_Unicode_Path_cen_extra_field() functions. zipfile.c + - Add read_Unicode_Path_entry() function. zipfile.c + - Set uname and oname in scanzipf_ref(). zipfile.c + - Add define wide_to_mb_default. Add zchar but not used. win32/osdep.h + - Add wide command line reading but don't use. win32/win32.c + - Add port functions for Unicode, including local_to_utf8_string(), + wide_to_escape_string() (for converting a wide character that can't be + converted to mb in the local character set to a reversable escape string), + escape_string_to_wide(), wide_to_local_string(), local_to_display_string() + (for creating the display version of name), utf8_to_local_string(), + local_to_wide_string(), wide_to_utf8_string() (NOT IMPLEMENTED), and + utf8_to_wide_string() (NOT IMPLEMENTED). win32/win32.c + - Implement attempt at escape function. Whenever a wide character can't + be mapped to the local character set, this function gets called. + Currently the wide character is converted to a string of hex digits. + If the wide can fit in 2 bytes then the form #1234 is used. If not, + the 4-byte form #L12345678 is used. + It compiles but needs the utf8 functions implemented. Also needs testing + in a multi-byte environment and only Windows is implemented so need to at + least do Unix. (Ed) +17. Update freeup() to include uname and oname. zip.c +18. Move define wide_to_mb_default so default for all is '_'. zip.h (Ed) +19. No changes needed to osdep.h and update unix/unix.c but not tested. (Ed) +---------------------- October 19th 2005 version 3.0f02 ---------------------- + 1. Remove null value check for split_size as get_option() already checks. + zip.c (Ed) + 2. Update f$search(). vms/descrip.mms (SMS) + 3. Save parse name before search and use that on failure. Change name parsing + in ziptyp() to solve a problem with search-list logical name device directory + specs. vms/vms.c (SMS) + 4. Compile in UNICODE_SUPPORT if have wchar_t and mbstowcs(). unix/configure (Ed) + 5. Move Unicode defines to zip.h and functions to fileio.c so generic. Create + a new OEM function for Windows. fileio.c, zip.h, tailor.h, win32/win32.c (Ed) + 6. Add UTF-8 functions. fileio.c (Paul) + 7. Convert Unicode functions to use zwchar defined as unsigned long for wide + char. fileio.c, zip.h (Ed) + 8. Add wchar_t check for Unix. unix/configure (Ed) + 9. Add default when zwchar (4 bytes) is too big for wchar_t (2 bytes). zip.h (Ed) +10. Allow for states for wide characters but surrogates not done. fileio.c (Ed) +11. Update WhatsNew (Ed) +---------------------- December 16th 2005 version 3.0f03 ---------------------- + 1. Fix broke encryption when ZIP64_SUPPORT enabled by accounting for need for + data description when encrypting. Data description is not required for + encryption (WinZip does not use one) but seems needed by Zip for some reason. + zipfile.c (Ed) + 2. Add function bfwrite() to do buffered fwrite(). Most output already is + written by zfwrite used by crypt.c which now calls bfwrite. All splitting + and new byte counts are done in bfwrite. fileio.c (Ed) + 3. Move some functions out of ZIP64_SUPPORT defines for use with UNICODE_SUPPORT. + zipfile.c, zip.h (Ed) + 4. Add is_ascii_string() and only create Unicode extra field if z->iname is + not ascii. zipfile.c, zip.h, fileio.c, (Ed) + 5. Add parameter rewrite to putlocal() to note when rewriting bytes so the bytes + rewritten are not counted in output totals. zipfile.c, zip.h (Ed) + 6. Handle VMS ... wildcard. util.c (SMS) + 7. Make tempzip file name global. zip.c, globals.c, zip.h (Ed) + 8. Add out_path global and -O path option to allow the output archive to have a + different name than the input archive, if there is one. This allows + updating a split archive, since output to the same split name would otherwise + be complicated and not user friendly. Use out_path for output. zip.h, + zip.c, globals.c (Ed) + 9. Many output functions that had output file y as parameter, such as zipup(), + zipcopy(), putlocal(), putcentral(), and putend(), now do not as y is + now global. This allows changing y as splits are created. zip.c (Ed) +10. Add function zipmessage() for writing messages like zipwarn() but are + informational. zip.c (Ed) +11. Minor changes to help. zip.c (Ed) +12. Add SPLIT_SUPPORT to version output. zip.c (Ed) +13. Add rename_split() to rename and set attributes for a split. zip.c (Ed) +14. Add set_filetype() to set attributes of split. zip.c (Ed) +15. Change variable a (holds attributes) to zipfile_attributes and make global. + zip.c, zip.h, globals.c (Ed) +16. Add key_needed flag for encryption and move setting key to after + command line processed. zip.c (SMS) +17. Initialize dot size using default only if dot_size not set. zip.c (Ed) +18. Change command line processing so that last -P or -e is used. zip.c + (Ed) +19. Fix broke writing of 4-byte spanning signature at the beginning of the archive + if splitting. zip.c (Ed) +20. Use bfcopy() instead of fcopy() to copy archive beginning. bfcopy() uses + global y. zip.c (Ed) +21. It looks like tempzf is no longer used. zip.c (Ed) +22. Account for SUNPRO_C and DECC_VER. Change SPARC to Sparc. unix/unix.c (SMS) +23. Remove GNUC. vms/cmdline.c (SMS) +24. Change case of system calls. vms/vms.c (SMS) +25. Add fix for VMS ... matching, but may change Zip to avoid ex2in() and in2ex() + for pattern matching in future. vms/vmszip.c (SMS) +26. Remove /NODIRNAMES and /DIRNAMES from VMS help. vms/zip_cli.help (SMS) +27. Define new globals zip64_eocd_disk, zip64_eocd_offset, current_local_tempname, + bytes_this_split, and bytes_this_entry for splits. globals.c, zip.h (Ed) +28. Add SUNPRO C and DEC C compile checks. unix/configure (SMS) +29. Add CFLAGS_NOOPT for removing optimization for configure. unix/Makefile (SMS) +30. Modify crypthead() to use bfwrite(). crypt.h, crypt.c (Ed) +31. Modify zfwrite() to use global output file. crypt.h, crypt.c (Ed) +32. Modify zfwrite() when no encryption to use bfwrite(). crypt.h (Ed) +33. Add bfcopy() to copy to y. fileio.c (Ed) +34. Add close_split() and bfwrite() for splits. fileio.c (Ed) +35. Add is_ascii_string() to check if UTF-8 extra field is needed. fileio.c (Ed) +36. Change Unicode escape of 2-byte wide from #1234 to #U1234. fileio.c (Ed) +37. Add read_Unicode_Path_entry() to read the UTF-8 path extra field. zipfile.c (Ed) +38. Latest Unicode Path extra field format is + 1 byte Version of Unicode Path Extra Field + 2 bytes Name Field Checksum + variable UTF-8 Version of Name +39. Use CRC-32 for Unicode Path Checksum and AND halves. zipfile.c (Paul) +40. Add Unicode Path Checksum check to make sure extra field applies to Name field + still. zipfile.c (Christian) +41. Move get_extra_field() out of Zip64 block and make available for splits. + zipfile.c (Ed) +42. Check in putlocal() using is_ascii_string() and don't create Unicode path + extra field if name is ASCII characters. zipfile.c (Ed) +43. If local header for split is on another disk and using split method 1, close + that split in putlocal() after rewrite local header. zipfile.c (Ed) +44. Fix data descriptor bug when encrypting where putextended() did not handle the + not Zip64 case, which mostly only happens now for encryption. zipfile.c (Ed) +45. Check for ASCII name using is_ascii_string() in putcentral() for Unicode path + extra field. zipfile.c (Ed) +46. Instead of single disk values, update putend() to use real split values for + current_disk, cd-start_disk, cd_entries_this_disk, cd_start_offset, + zip64_eocd_disk, zip64_eocd_offset, and current_disk and allow for + needing Zip64 if exceed standard max values for current_disk, cd_start_disk, + cd_entries_this_disk, total_cd_entries, and cd_start_offset. zipfile.c (Ed) +47. Use current_local_offset and current_local_disk for z->off and z->dsk in + zipup(). zipup.c (Ed) +48. Fix bug where force_zip64 was used to determine descriptor size but can have + Zip64 entry without force_zip64 so use zip64_entry. zipup.c (Ed) +49. Change the p - o != s compression size test for splits to bytes_this_entry + != (key ? s + 12 : s) and avoid ftell() in split. zipup.c (Ed) +50. If local header is on a previous split and split method 1 do the seek on that + split to update header. zipup.c (Ed) +51. For streaming, only force Zip64 if reading stdin and writing a non-seekable + device. In other cases can detect either the input file size and set Zip64 + if needed or seek in the output to update the local headers. zipup.c, + zipfile.c, zipup.c (Ed) +52. Allow creation of stored archives with descriptors for testing. Currently + they can't reliably be read but this is planned. zipup.c, zipfile.c, zip.c + (Ed) +53. Update help, adding in -e, -P, -s splitsize, -sp, and -sv options. zip.c (Ed) +54. Spelling fix in zipsplit man page. man/zipsplit.1, zipsplit.txt (Ed) +55. New option -sv and variable noisy_splits to enable verbose splitting. + Default is to quietly create splits, unless -sp set to pause between splits. + zip.h, zip.c, globals.c, fileio.c (Ed) +---------------------- December 23rd 2005 version 3.0f04 ---------------------- + 1. Move inlined text-vs-binary checks from file_read() into a separate, + new function named is_text_buf(). zipup.c, util.c, zip.h (Cosmin) + 2. Fix calls to putlocal to remove the removed dest parameter. crypt.c (Ed) + 3. Add get_split_path() to get the path for a split given the disk number. + fileio.c, zip.h (Ed) + 4. Change formatting of zipmessage() to remove tabbing and add formatting + to call to zipmessage(). fileio.c, zip.c (Ed) + 5. Initialize many variables such as y and tempzip. zip.c, fileio.c, + zipfile.c (Ed) + 6. Add loop to pause during split method 2 to allow changing disk or changing + the path for the next split. fileio.c (Ed) + 7. If after starting new split there is not enough room for the remaining buffer + for split method 1 display error and exit and for split method 2 we can + display a warning and user can replace disk or change path. fileio.c (Ed) + 8. Add list to store input file arguments using add_name() to add the name to + filelist_struc filelist and then process the names after the input archive + is read. zip.c (Ed) + 9. Fix infinite loop when opening a log file whose name contains multiple '/'. + zip.c (Cosmin) +10. Move split size message lower and only output if option sv sets + noisy splits. zip.c (Ed) +11. Set y to output file, remove output file from zipcopy(), putlocal(), + putcentral(), and putend(). zipsplit.c, zipnote.c, zipcloak.c (Ed) +12. Add code for not SPLIT_SUPPORT case. zipfile.c, zipup.c (Ed) +13. Prepend '-' to commands from target clean. + win32/makefile.w32, win32/makenoas.w32, win32/makefile.bor (Cosmin) +14. Must not call putenv() in iz_w32_prepareTZenv() under Cygwin. + win32/osdep.h (Cosmin) +15. Add browse info in Visual C++ 6 project. win32/vc6/zip*.dsp (Cosmin) +---------------------- December 27th 2005 version 3.0f05 ---------------------- + 1. Add proposed changes to License (Ed) + 2. Fix -l corruption bug by using memcpy() instead of wrongly changing the + buffer pointer. Fix was left out of last beta. zipup.c (Cosmin) + 3. Fix get_split_path() parameter. zip.h (SMS, Ed) + 4. Add -dg and display_globaldots to display dots globally for entire archive + instead of for each file. Is not affected by noisy flag. globals.c, + zip.h, zip.c, zipup.c, fileio.c (Ed) + 5. Make dot_count and dot_size uzoff_t, dot_count because with global dots + dot_count does not reset and with terabyte files the number of buffers + could exceed 2G, dot_size to allow use of ReadNumString() to read number. + zip.c, zip.h, globals.c (Ed) + 6. Add Deletion to help. zip.c (Ed) + 7. Fix delete date. zip.c (Ed) + 8. For streaming, need to assume Zip64 if writing a non-seekable device so + extra field for Zip64 is created if needed. zipup.c, zipfile.c, zipup.c (Ed) + 9. Add remove_local_extra_field() and remove_central_extra_field(). + zipfile.c (Ed) +10. Remove disabled copyright from license(). zip.c (Ed) +11. Clean up recent changes. zip.c, zipfile.c, fileio.c, zip.h, zipup.c (Ed) +12. Create scanzipf_regnew() for new file scan. zipfile.c (Ed) +---------------------- December 29th 2005 version 3.0f06 ---------------------- + 1. Change dot_size and dot_count from uzoff_t to zoff_t to allow use of + negative flag values. globals.c, zip.h (SMS, Ed) + 2. Remove file parameter to bfwrite() in putend(). zipfile.c (SMS, Ed) + 3. Add back code for not SPLIT_SUPPORT to putend(). zipfile.c (SMS, Ed) + 4. Change tag from ush to ulg in remove_local_extra_field() and + remove_central_extra_field() to avoid parameter problems. zipfile.c (Ed) + 5. Add allow_empty_archive to flag when want to create an empty archive. + globals.c, zip.h (Ed) + 6. Set allow_empty_archive when using -i and expecting an archive to be + created. This is in response to the 2/14/05 email. zip.c (Ed) + 7. Make before and after variables that hold the dates of files to + process or delete global so can use them in scanzipf_regnew(). zip.h, + zip.c, globals.c (Ed) + 8. Change scanzipf_regnew() to be based on scanzipf_fix() which seems closer. + Still have not coded the new regular zipfile reader. zipfile.c (Ed) + 9. For new reader first get add list and then read old archive and filter + as reading old entries. zip.c, zipfile.c (Ed) +10. Define USE_NEW_READ to turn on using new reader, which is being + created. This allows all to work while the new reader is being worked + on. zip.c, zipfile.c (Ed) +---------------------- January 9th 2006 version 3.0f07 ---------------------- + 1. Remove dest parameter from crypthead() and zipcopy(). crypt.c (SMS, Ed) + 2. Change -ds to handle dots for as small as every 32 KB. zip.c (Ed) + 3. Add ask_for_split_write_path() and ask_for_split_read_path() for + asking where to put the next write split and for locating the next + read split. zip.h, fileio.c (Ed) + 4. Add in_path to track where reading splits from. zip.h, globals.c, zip.c (Ed) + 5. Update copyright date on changed files to include 2006 (Ed) + 6. Replace stderr with mesg for most output messages. deflate.c, fileio.c, + trees.c, util.c, zip.c, zipcloak.c, zipfile.c, zipnote.c, zipsplit.c + 7. Add mesg_line_started to track if need new line on mesg output and update + zipmessage() and zipwarn() to use it. Set mesg_line_started to 1 when + newline not last character written to mesg and 0 when it is. deflate.c, + zip.h, zip.c, globals.c, zipup(), fileio.c (Ed) + 8. Include base_path as parameter for get_split_path(). fileio.c (Ed) + 9. Account for VMS version in split path. Add vms_file_version(). fileio.c, + zip.c, vms/vms.c, vms/vms.h (SMS) +10. Create crc16f() to create ANDed halves crc32 for Unicode using copy + of crc32() but may change to use main copy. zipfile.c, zip.h, + fileio.c (Ed) +11. Close in_path and out_path in finish() and ziperr(). zip.c (Ed) +12. Change perror() to strerror() and print to mesg in ziperr(). zip.c (Ed) +13. Add find_next_signature() to find the next signature when reading a + zip file. zipfile.c (Ed) +14. Add find_signature() to find a given signature from current point in + archive. zipfile.c (Ed) +15. Add at_signature() to check if at a given signature in archive. + zipfile.c (Ed) +16. Changes to scanzipf_regnew() but far from done. zipfile.c (Ed) +17. Changes to readzipfile() to close input archive file and allow new + zipfile reader to open and close files as goes through splits. + zipfile.c (Ed) +18. Change -s to default to MB and set minimum split size to 64k. + zip.c (Ed) +19. Add link to user32.lib for CharToOem(). makefile.w32, makenoas.w32 + (Cosmin) +20. Remove unused Z64_EFPos. globals.c (Ed) +---------------------- February 13th 2006 version 3.0f08 ---------------------- + 1. Move option checks before any work is done. zip.c (Ed) + 2. Update bfcopy() to handle reading splits and remove input file + parameter and use global in_file. fileio.c (Ed) + 3. Change ask_for_split_read_path() to allow user aborting. fileio.c (Ed) + 4. Change get_split_path() to use standard file extensions from most + recent AppNote of .z01, .z02, ..., .z99, .z100, .z101, ... fileio.c (Ed) + 5. Change is_ascii_string to use 0x7F for ASCII detection. fileio.c (Ed) + 6. Add copy_only global for when -O is used to change the format of an + archive without changing the contents. This allows for converting an + archive to a split archive for instance. The global copy_only is used + to output status information for copies when normally copied files + have no status messages. globals.c (Ed) + 7. Add in_file, split_path, total_disks, current_in_disk, and + current_in_offset as globals to track reading splits. zip.h, + globals.c (Ed) + 8. Update copyright date. revision.h (Ed) + 9. Close in_file if open in finish(). zip.c (Ed) +10. Add -O (big o) to extended help. zip.c (Ed) +11. Remove readzipfile() from zipstdout() and use main call later down. + zip.c (Ed) +12. Move archive reading and file scanning after command line checks. + zip.c (Ed) +13. If -O out_zip and so have_out is set then set copy_only and allow + copying instead of error message *Nothing to do*. zip.c (Ed) +14. If zipbeg is just 4 bytes and spanning then assume is spanning + signature and set zipbeg to 0 to ignore. zip.c (Ed) +15. Don't open initial write test as modify if have_out is set and so have + a separate output file. zip.c (Ed) +16. If zipbeg is 0 and nothing at beginning of archive to copy then don't + open input file until zipcopy() does. zip.c (Ed) +17. If stuff at beginning then copy and close input file. Should be able + to keep it open but easier to close it and let zipcopy() reopen it. + zip.c (Ed) +18. Add status message when copy_only set so something is displayed. + zip.c (Ed) +19. Instead of closing x at bottom close in_file. The variable x was used + inconsistently and it seemed easier to make in_file global instead. + Then again y remains the global output variable. zip.c (Ed) +20. Update copyright. zipnote.c, zipsplit.c, zipcloak.c (Ed) +21. Change adjust_zip_local_entry() to return 1 if the entry is Zip64 and + 0 if not. This is needed to know how large the extended local header + is later. zipfile.c (Ed) +22. Add read_Unicode_Path_local_entry() to read the local version of the + Unicode Path extra field. zipfile.c (Ed) +23. Handle disk in adjust_zip_central_entry(). zipfile.c (Ed) +24. Change USE_NEW_READ to SPLIT_SUPPORT as splits seems to be stable more + or less. zipfile.c (Ed) +25. Add is_signature() to compate signatures. zipfile.c (Ed) +26. Create scanzipf_fixnew(). It should look like scanzipf_regnew(). + zipfile.c (Ed) +27. Change scanzipf_regnew() to read the central directory and create zlist + and handle reading traditionally. Allows using central directory + information, in particular file sizes, in zipcopy() while reading + entries. Use global in_file instead of f for input file and set to NULL + when not a valid file so finish() only tries to close it if needed. + Check to make sure the End Of Central Directory record found is infact + the last one in case a stored archive is in the last 64 KB. Refuse + to update a split archive but recommend using -O instead. zipfile.c (Ed) +28. Change readable check in readzipfile() to require input archive to exist + if using -O out_archive. zipfile.c (Ed) +29. Change putlocal() to not create a Zip64 extra field unless needed and + on rewriting the local header to remove Zip64 extra field if was created + but not needed. Add check if assumed entry does not need Zip64 but does, + meaning probably the uncompressed size is less than 4 GB but the + compressed size is over 4 GB. zipfile.c (Ed) +30. Change zipcopy() to use the global in_file and y files and to open and + close read splits as needed. Checks the local header against the + central directory header to verify same file, which should be as using + the disk and offset values from the central directory. Update disk and + offset in central directory. zipfile.c (Ed) +31. Change out_path and out_len to base_path and base_len in + get_split_path(). fileio.c (SMS) +32. Update command line options for VMS to include verbose splitting. + vms/zip_cli.cmd, vms/cmdline.c (SMS) +33. Handle HP. unix/unix.c (SMS) +34. Add adler16() checksum function. util.c (Cosmin) +35. Use FILE_FLAG_BACKUP_SEMANTICS and a less demanding access mode + in CreateFile() when retrieving file attributes. Fixes a problem + when adding a directory entry from an NTFS or a CDFS partition + (i.e. one that stores timestamps using universal time), and the + directory timestamp is not the same daylight savings time setting. + The effect is an offset in the timestamp by one hour, if zip is + built using NT_TZBUG_WORKAROUND. The problem is not exposed, + however, if NO_W32TIMES_IZFIX is defined. win32/win32.c (Cosmin) +---------------------- March 19th 2006 version 3.0f09 ---------------------- + 1. Fix encryption problem where a large file with uncompressable data + can cause deflate to store bad data. See crypt.c for details. + Thanks to the nice people at WinZip for finding and providing the + details of this problem. crypt.c (Ed) + 2. Add note at top of Extended Help to refer to the Zip Manual. zip.c + (Ed) + 3. Update extended help for delete. zip.c (Ed) + 4. Change crypthead() to use buffer and bfwrite() which is split aware. + crypt.c (Ed) + 5. Create SPLIT_SUPPORT version of zipcloak() and zipbare() and read + local header rather than assume data using central header. crypt.c (Ed) + 6. Change zfwrite() to use global output file y. crypt.c (Ed) + 7. Remove in and out parameters from zipcloak() and zipbare() for + splits. crypt.h, zipcloak.c (Ed) + 8. Change get_split_path() to get_in_split_path() and get_out_split_path(). + zipfile.c, fileio.c, zip.h (Ed) + 9. Change crc32f() to crc32u(). fileio.c, zip.h (Ed) +10. Add encryption overwrite fix to copy_block() and remove from zfwrite(). + crypt.c, tree.c (Ed, Christian) +11. Add note on bug fix. WhatsNew (Ed) +12. Add copy_only mode. zip.c (Ed) +13. Make SPLIT_SUPPORT the default. zip.h (Ed) +14. Add set_filetype(), rename_split(), and zipmessage(). zipcloak.c, + zipnote.c, zipsplit.c (Ed) +15. Add long option support. zipcloak.c (Ed) +16. Set in_path. zipcloak.c, zipnote.c, zipsplit.c (Ed) +17. Use SPLIT_SUPPORT calls. zipcloak.c, zipnote.c, zipsplit.c (Ed) +18. Set current_disk, cd_start_disk, and cd_entries_this_disk for use + by putend() and bytes_this_split for putcentral(). zipsplit.c (Ed) +19. Include ctype.h for toupper(). zipfile.c (Ed) +20. Add readlocal() for utilities to read local header. zipfile.c (Ed) +21. Put Zip64 variables and code in ZIP64_SUPPORT ifdef in scanzipf_regnew(). + zipfile.c (Ed, SMS) +22. Use zip_fzofft() for converting offset. zipfile.c (Ed, SMS) +23. Add casts to many append to memory calls. zipfile.c (Ed) +24. Move handling of .zip split to get_in_split_path() and + get_out_split_path(). zipfile.c (Ed) +25. Handle fix = 3 case for ZipNote that renames entries in zipcopy(). + zipfile.c (Ed) +26. Restore clearing of extended local header bit when not encrypting. When + encrypting need to output extended local header using putextended() in + zipcopy(). zipfile.c (Ed) +27. Add notes on using file time for encrypting. zipup.c (Ed) +28. Remove extended local header bit separately for z->lflg (local flags) + and z->flg (central directory flags). These should be the same but + could be different. zipup.c (Ed) +29. Suppress command line globbing for MINGW. win32/win32.c (Christian) +30. Add EF UT time fix for delete. zip.c (Christian) +---------------------- April 28th 2006 version 3.0f10 ---------------------- + 1. Add note to extended help to escape [ as [[] or use -nw. zip.c (Ed) + 2. Remove local declaration of tempfile as now global. zipnote.c, + zipcloak.c (SMS) + 3. Add zip_fzofft() for outputting uzoff_t bin size c. zipsplit.c (SMS) + 4. Add only_archive_set and clear_archive_bits to do Window archive bit + selection and clearing. Add -AS option to require DOS Archive bit + be set and -AC to clear archive bits of included files. Add + ClearArchiveBit() to clear archive bits after archive created. + Only Win32. globals.c, zip.h, zip.c, win32zip.c, win32.c (Ed) + 5. Change procname_win32() and readd() to check archive bit. + win32/win32zip.c (Ed) + 6. Update copyright. win32/win32zip.h (Ed) + 7. Add mesg_line_started = 0 to stats to remove blank line when clearing + archive bits. zipup.c (Ed) + 8. Add zip_fzofft() to format split size. zipsplit.c (SMS) + 9. Update help for splits and archive bit and add note on escaping [. + zip.c (Ed) +10. Add -M option and bad_open_is_error to exit with error if any input + file unreadable. Also error if -M and would get "name not matched" + warning. zip.c (Ed) +11. Copy Zip 2.32 csharp example, though it is designed for zip32.dll and + not zip32z64.dll from Zip 3.0. Updated note. windll/csharp (Ed) +12. Change -M to -MM and define -mm to avoid accidental use of -m. + zip.c (Ed) +13. Define split_method -1 to not allow splitting, mainly used when reading + a split archive to stop automatic splitting of output with same + split size. Now -s=0 or -s- disables splitting. zip.h, globals.c, + zip.c (Ed) +14. Add fflush() after dots displayed. deflate.c, fileio.c, zipup.c (Ed) +15. Instead of assuming buffer size as 32 KB for dots, use WSIZE for + compressing and SBSZ for storing and calculate as dots are counted. + Now dot_count and dot_size are bytes instead of buffers. Add dots + to Delete and Archive modes. zip.c, zipup.c, deflate.c, fileio.c (Ed) +16. If reading a split archive and split size has not been given, get + size of first split read by zipcopy(), which should be the first + split, and set split size to that, making the output archive the same + split size as the input archive. Delay -sv split size message + if split size is 0 at first but then changed. zipfile.c (Ed) +17. Add proc_archive_name() for new archive mode to process names in old + archive only and skip looking on the file system. Easier than modifying + the various port codes. fileio.c (Ed) +18. Fix cd_start_offset bug. fileio.c (Ed) +19. Create new action ARCHIVE that looks for matches only in old archive + for Copy Mode. If no input paths and there is an output archive, + Copy Mode is assumed even without ARCHIVE. Old default Copy Mode + when no input files updated to work like -U mode and allow filters. + New global copy_only currently only used to control global dots. + zip.c, fileio.c, globals.c, zip.h (Ed) +20. Update help. Change extended help to more help. Update more help + to include internal modes delete and new Archive. Update help for + formatting options. Update help for wildcards. Remove streaming + examples from top basic section. Indent examples. Help for new + --out and Copy Mode. Add warnings that output using data descriptors + may not be compatible with some unzips. Update dots help and add + warning that dot size is approximate. Add help for new DOS archive + bit options. More help for -b and -MM. zip.c (Ed) +21. Add support for Unix FIFO (named pipe). Add set_extra_field() stat + name ending in '/' fix found in Zip 2.32. unix/unix.c (Ed) +22. Add check to not allow setting -U (internal copy) in similar cases to + -d (delete). zip.c (Ed) +23. Add counts for internal modes Delete and Archive. Byte counts for -db + remain uncompressed size for external modes, but internal modes Delete + and Archive now use compressed sizes as these copy that many bytes. + zip.c (Ed) +24. Add check for when ftell() wraps. zipup.c (Ed) +25. Add mesg_line_started = 0 to result percentage message. zipup.c (Ed) +26. Update contact information. unix/packaging/preinstall.in (SMS, Ed) +27. A few Zip64 fixes to set Zip64 correctly and fix disk and offset of + Zip64 End Of Central Directory. zipsplit.c (Ed) +28. Update comments for get_option(). fileio.c (Ed) +29. Update DLL version. windll/windll.rc (SMS, Ed) +30. New option -sf shows files that would be operated on. zip.c (Ed) +---------------------- May 5th 2006 version 3.0f11 ---------------------- + 1. Use C prototypes for Unicode functions. fileio.c (SMS) + 2. Change constant for mask in set_file_type from unsigned to signed. + trees.c (SMS) + 3. Use C prototypes for zip_fzofft() and zip_fuzofft() signed and + unsigned zoff_t formatting functions. util.c (SMS) + 4. Remove U from constants in Adler16 code. util.c, zip.h (SMS) + 5. Add spaces to VMS usage to avoid misinterpretation. zip.c (SMS) + 6. Add OF() to at_signature(). zipfile.c (SMS) + 7. Use zip_zofft() for entries error. zipfile.c (SMS) + 8. Remove U in constants in percent(). zipup.c (SMS) + 9. VMS command line updates. vms/cmdline.c, vms/descrip_deps.mms, + vms/vms_zip.rnh, zip_cli.cld, vms/zip_cli.help (SMS) +10. Update to VMS help. vms/zip_cli.help (Ed) +11. Check for memmove() and strerror(). Remove specific 64-bit support + for SunOS, as large file support now does. unix/configure (SMS) +12. Add emergency replacements for memmove() and strerror(). + unix/unix.c (SMS) +13. Remove old not SPLIT_SUPPORT code. globals.c, zipnote.c, fileio.c, + crypt.h, crypt.c, zipcloak.c, zip.h, zip.c, zipup.c, zipsplit.c, + zipfile.c (Ed) +---------------------- May 12th 2006 version 3.0f12 ---------------------- + 1. Add UNICODE_SUPPORT ifdef around uname in zipup(). zip.c (SMS) + 2. Change size from uzoff_t to zoff_t in zipcopy(). zipfile.c (SMS, Ed) + 3. Fix a bug where filetime() returns -1 for device but not handled in + byte counts. zip.c (Ed) + 4. Add check for UnZip version and exit if not 6.00 or later if + a Zip64 archive. Define popen() and pclose() in Win32 to native + _popen() and _pclose(). ziperr.h, zip.c, win32/osdep.h (Ed) + 5. Add -sb option to ring bell when pause to change disk. Use new + global split_bell. global.c, zip.h, zip.c, fileio.c (Ed) + 6. Enable crc32u() and use for Unicode extra field. fileio.c (Ed) + 7. Add -dv to display volume being written to. zip.c, zip.h, + globals.c (Ed) + 8. Update WhatsNew. WhatsNew (Ed) + 9. Help updates. zip.c (Ed) +10. Create option -X- (negated -X) to keep old extra fields and remove + -XX which is now -X. Make get_extra_field() global. Add + copy_nondup_extra_fields()to copy old extra fields not already + in new extra fields. zipup.c, zip.c, zipfile.c (Ed) +11. Use output name oname for -sf option to show files that would be + worked on. zip.c (Ed) +12. When updating or freshening old entries, read the old local header + with readlocal() to get local flags and extra fields. zip.c (Ed) +13. Add UNICODE_SUPPORT ifdefs around uname code. zip.c (SMS, Ed) +14. If WIN32_OEM set then on WIN32 store OEM name in archive. As read + DOS or WIN32 archives convert assumed OEM paths to ANSI. Remove old + WIN32_OEM code. Make oem_to_local_string() global for WIN32_OEM and + local_to_oem_string() global for WIN32_OEM and UNICODE_SUPPORT. + zip.h, zipfile.c, zipup.c, win32/win32.c, win32/win32zip.c (Ed) +15. Update error 8 to include wrong unzip. ziperr.h (Ed) +16. Change checksum for Unicode extra field to standard crc32 using + C version crc32u(). Add crctab.c. win32/vc6/zipnote.dsp, + win32/vc6/zipsplit.dsp, zipfile.c +17. Update readlocal() to handle multi-disk archives if not UTIL. + zipfile.c (Ed) +18. Convert size to signed zoff_t in zipcopy(). Update note. + zipfile.c (Ed) +19. Update Readme. Readme (Ed) +20. Add crctab.o to zipsplit and zipnote. unix/Makefile (Ed) +21. Proposed update to license. License (Ed) +---------------------- May 20th 2006 version 3.0f13 ---------------------- + 1. Reformat License file. License (Cosmin) + 2. Change %d to %lu for disk number and add cast. zip.c (Cosmin, Ed) + 3. Display Scanning files message after delay at start based on + suggestion from Greg. Currently the time is checked every 100 + entries processed. After 100 entries the start time is saved. + After 5 seconds or 100 entries after that, whichever takes + longer, the Scanning files message is displayed and every 2 seconds + or 100 entries, whichever takes longer, after that a dot is displayed. + fileio.c, zip.c, globals.c, zip.h (Greg, Ed) + 4. Add Unicode mismatch flag and option -UN. Default is now a Unicode + mismatch is an error. -UN=warn outputs warnings and continues, + -UN=ignore disables warnings, and -UN=no ignores the Unicode extra + fields. globals.c, zip.h, zipfile.c (Ed) + 5. Add options for VMS. vms/cmdline.c, vms/zip_cld.cld (SMS) + 6. Add casts to percent(). zipup.c (Ed) + 7. Minor changes to logfile formatting. zip.c (Ed) + 8. Update help. zip.c (Ed) + 9. Add -Z=compression-method option. zip.c (Ed) +10. Add sd: to -sd status messages. zip.c (Ed) +11. Instead of original argv[] use args[] for -sc show command line + to show final command line. zip.c (Ed) +12. Change argv[] to args[] for logfile. zip.c (Ed) +13. Put results of -sf show files in log file if open. zip.c (Ed) +14. Add Johnny's bzip2 patch but not tested. win32/makefile, zip.c, + zip.h, zipup.c (Johnny) +15. Minor tweeks to bzip2 to work with latest beta. zip.c, zipup.c (Ed) +16. Add -sf- to list files that would be included only in log file + and not on stdout as list can be long. Only list totals on stdout. + zip.c (Ed) +17. Create check_unzip_version(). Fix Unix check. Zip still creates + the temporary archive then does the check, and if it fails + the archive is deleted, even if the check fails because of the wrong + verion of UnZip. On Unix only 'unzip' the system version of UnZip + is checked, not './unzip' which would allow putting a local more + up to date version of UnZip in the current directory for the check. + There should be a way to override the system version of UnZip for + the -T test. zip.c (Ed) +---------------------- July 12th 2006 version 3.0f14 ---------------------- + 1. Change crypt version from 2.10 to 2.91 to match Zip 2.32 and avoid + confusion. crypt.h (Cosmin) + 2. Add abbrevmatch() to handle option values that can be abbreviated + like compression method. util.c, zip.h, zip.c (Ed) + 3. Change USE_BZIP2 to BZIP2_SUPPORT as USE_BZIP2 implies it replaces + deflation maybe. zip.c, zip.h, zipup.c (Ed) + 4. Update man page. man/zip.1, zip.txt (Ed) + 5. Add bzip2 to VMS. vms/build_zip.com, vms/bzlib.h, vms/cmdline.c, + vms/descrip.mms, vms/descrip_src.mms, vms/find_bzip2_lib.com, + vms/install_vms.txt, vms/zip_cli.cld (SMS) + 6. Remove zipfile parameter from bzfilecompress(). Add unsigned + cast for EOF in bzip2 code. Add bzip2 version information. + zipup.c, zip.c (SMS) + 7. Add bzip2 to Unix. unix/configure (SMS) + 8. Add and update bzip2 descriptions. INSTALL, README, WhatsNew, + bzip2/install.txt (SMS, Ed) + 9. Add vc6bz2 projects for compiling bzip2 code into zip (not the + best approach perhaps). win32/vc6/readmevc.txt, + win32/vc6bz2/readvcbz.txt, win32/vc6bz2/zip.dsp, win32/vc6bz2/zip.dsw, + win32/vc6bz2/zipcloak.dsp, win32/vc6bz2/zipnote.dsp, + win32/vc6bz2/zipsplit.dsp (Ed) +10. Add support for VC++ 2005 by disabling deprecation. win32/osdep.h + (Cosmin) +11. Update instructions for makefile. unix/Makefile (Ed) +12. Update todo list. todo30.txt (Ed) +13. Reduce #if 0 block to now allow extra data message. zipfile.c (Ed) +14. Add note that readlocal() reads local headers. zipfile.c (Ed) +15. Archive comment was not being read by new scanzipf_regew(). Added. + zipfile.c (Ed) +16. Handle reading and writing OEM comments. zipfile.c (Ed) +17. Update Zip64 data descriptor note. zipfile.c (Ed) +18. Format filetypes() check. zipup.c (Ed) +19. Update note to remember to force deflation for descriptors by + release. zipup.c (Ed) +20. In compression code using libraries, enable dots for noisy also. + zipup.c (Ed) +21. Update extended help to add more of the basic options and + compression method. zip.c (Ed) +22. Add additional lines bz_opt_ver2 and bz_opt_ver3 to bzip2 + version to give credit to bzip2. zip.c (Ed) +23. Add descriptions to version information for USE_EF_UT_TIME, + NTSD_EAS, WILD_STOP_AT_DIR, WIN32_OEM, LARGE_FILE_SUPPORT, + ZIP64_SUPPORT, and UNICODE_SUPPORT similar to how UnZip does. + zip.c (Ed) +24. Add note that crypt is modified in Zip 3. zip.c (Ed) +25. Use abbrevmatch() and update warnings for compression + method selection. zip.c (Ed) +26. Update config to handle either using IZ_BZIP2 to define + the location of the bzip2 files or the bzip2 directory. + unix/configure, zipup.c, zip.c (SMS, Ed) +---------------------- July 14th 2006 version 3.0f15 ---------------------- + 1. Change USE_BZIP2 to BZIP2_SUPPORT in VMS. vms/descrip_src.mms, + vms/build_zip.com (SMS) + 2. Add SYS$DISK:. vms/descrip.mms, vms/build_zip.com (SMS) + 3. Change vms/install.txt to [.vms]install.txt. bzip2/install.txt (SMS) + 4. Change VMS files to lower case. vms/mod_dep.com, vms/install_vms.txt, + vms/zip.opt, vms/hlp_lib_next.com, vms/notes.txt, vms/unixlib_gcc.h, + vms/unixio_gcc.h (SMS) + 5. Remove old VMS files. vms/descrip-old.mms (removed), + vms/link_zip.com (removed), vms/make_zip.com (removed), + vms/makefile.vms (removed) (SMS) +---------------------- July 24th 2006 version 3.0f16 ---------------------- + 1. Fix global dots so can set with dot size. deflate.c, fileio.c (Ed) + 2. Update License top line to refer only to license. License (Cosmin) + 3. Update License. License (Ed) + 4. Implement zero length UTF-8 path length as flag standard path is UTF-8 + and should use that. This allows Zip to use the standard path as + UTF-8 when the local character set is UTF-8. zipfile.c (Ed) + 5. Update WhatsNew. WhatsNew (Ed) + 6. Change case of bzip2/install.txt. INSTALL (Ed) + 7. Change MANUAL.txt to ZIP.txt and update ftp site. README (Ed) + 8. Update announcement. zip30f.ann (Ed) + 9. Now also check if OS has bzip2 library and can use that. + unix/configure, zip.c (Mark Adler, Ed) +10. Add fix from akt@m5.dion.ne.jp in Japan to recurse on doublebyte + characters without processing in recmatch(). This should not be needed + unless the rest of the code in there is broke for Japanese character + sets in some way. Need to test. util.c (Ed) +11. Add note for bzip2. zip.c (Ed) +12. Do not do seek wrap test if ftell() returns -1 as from a pipe. Add + output of last ftell() and current ftell() for zipfile too big seek + error. zipup.c (Ed) +13. Add version to the options table. Remove the check to display version + before the command line is processed. Add to option -v a check to + display the version if that is the only argument. Can still enable + verbose with piping by using zip -v - - format. zip.c (Ed) +14. Add abbrevmatch() for -UN option. zip.c (Ed) +---------------------- August 7th 2006 version 3.0f17 ---------------------- + 1. Change license modifications to retain intent of copyright holders, as + any major change in license conditions would require contacting all + copyright holders. LICENSE (Greg, Ed) + 2. Move debugging statement after zipstdout() where mesg is set to stderr. + Add mesg and fflush() to sd messages where needed so that messages go + to stderr when piping. zip.c (Ed) + 3. Update encryption comment. zipup.c (Ed) + 4. Do not use data descriptors for directories. zipup.c (Mark, Ed) + 5. Update Q & A to match license. README (Ed) + 6. Update WhatsNew. WHATSNEW (Ed) + 7. Add ifndef around version_info() for dll. zip.c (Ed) + 8. Add -TT (--unzip-path) to allow setting the unzip command to use with + -T to test the archive. zip.c (Ed) + 9. Add -DF (--difference-archive) which requires --out and turns off + copying unchanged entries to the new archive creating an archive with + just the changes and additions since the original archive was created. + zip.c, globals.c, zip.h (Ed) +10. Update help. zip.c (Ed) +---------------------- September 7th 2006 version 3.0f18 ---------------------- + 1. Split -t and -tt options and remove limitation that only one can be + used to allow setting a date range. zip.c, WhatsNew (Ed) + 2. Minor changes in comments. zipfile.c (Ed) + 3. Add entries for format of Unicode Path and Unicode Comment extra fields. + proginfo/extrafld.txt (Ed) + 4. Change note at top of infozip.who, but needs to be updated with all new + contributors. proginfo/infozip.who (Ed) + 5. Note Zip 3 and UnZip 6 now support Zip64. proginfo/ziplimit.txt (Ed) + 6. Add note on Unicode. README (Ed) + 7. Update WHATSNEW. WHATSNEW (Ed) + 8. Update help. zip.c (Ed) + 9. Add {} support to -TT option, allowing insertion of temp archive path + into the command string to -TT similar to Unix find does. zip.c (Ed) +10. Start changes for -F fix option. Add checks when reading input archive + and skip bad central directory entries and bad local entries. Currently + -F requires the central directory to be intact (except for bad CD entries + that will be skipped) and local entries and data to be where the + central directory say they are. This allows all recovered entries to + be complete with all central directory information. Calculate CRC of + input entry and compare to CRC from central directory. Allow skipping + split disks the user may not have. Store state of output archive + before each local entry and data are read, allowing seeking back and + restoring state to skip bad entries. fileio.c, global.c, zipfile.c, + zip.h (Ed) +11. Started changes for fixfix. fileio.c (Ed) +12. Update help on -t and -tt. zip.c (Ed) +13. Add note on Unicode support, but may change if add handling of names + with characters not supported in current character set. README (Ed) +14. Combined ToDo30.txt and ToDo but more to be done. TODO (Ed) +15. Update ToDo list. ToDo30.txt (Ed) +16. Add -F and -FF to help. zip.c (Ed) +17. Run fix mode in copy mode, as it is copying from one archive to + another, and use those checks. zip.c (Ed) +18. Add Try -F and Try -FF warnings in places. zipfile.c (Ed) +19. Allow reading version 4.6 (bzip2) archives. zipfile.c (Ed) +20. Add Unicode Path and Unicode Comment extra field descriptions. + proginfo/extrafld.txt (Ed) +21. First attempt at updating the Who file. proginfo/infozip.who (Ed) +22. Add note to top of ziplimit.txt. proginfo/ziplimit.txt (Ed) +23. Add possible fix for paths returned by the Win32 directory scan with + '?' in the name. These are characters in the Unicode name stored on + disk but not represented in the multi-byte character set used by zip + for the scan. In this case, return the short name in 8.3 format so + directory scan can continue. Could put the Unicode name in the Unicode + extra field, but not done. Add warning when long name is replaced + by short name. Not fully tested. win32/win32zip.c, zip.h, zip.c, + fileio.c (Ed) +24. If archive name and -sf are the only parameters, list archive contents. + zip.c (Ed) +---------------------- September 8th 2006 version 3.0f19 ---------------------- + 1. Fix error message. zipfile.c (SMS, Ed) + 2. Put crc32() in ifndef UTIL as only needed for fix. fileio.c (SMS, Ed) +---------------------- November 3rd 2006 version 3.0f20 ----------------------- + 1. Fix comment. vms/vmszip.c (SMS) + 2. Include oem_to_local_string() if UNICODE_SUPPORT. win32/win32.c, + zip.h (Ed) + 3. Modify procname_win32() to flag a path not supported by the local + character set so can get Unicode for it. Check Unicode names. + win32/win32zip.c (Ed) + 4. Add matching of escaped Unicode names to proc_archive_name() that + reads entries from an archive. Add sorted zlist zusort. + globals.c, fileio.c, zip.h, zipfile.c (Ed) + 5. Add support for non-local character set names and paths for WIN32, + getting and storing the UTF-8 path when needed. Use 8.3 name + when normal name has characters not supported in current local + character set. Note when short name used. zip.c, fileio.c (Ed) + 6. Add support for fix = 2 which reads local headers first to + bfcopy(). fileio.c, zip.h (Ed) + 7. Allow selection of .zip split in ask_for_split_read_path() when + reading a split archive that has no end records giving the total + split count. fileio.c (Ed) + 8. Add zoff_t casts to dot counts. fileio.c (Ed) + 9. Comment changes for Unicode. fileio.c (Ed) +10. Call wide_to_local_string() separately in utf8_to_local_string() + to free up temp value. fileio.c (Ed) +11. Support new AppNote bit 11 for forcing UTF-8, but needs finishing. + globals.c (Ed) +12. Add to zlist struct zuname for the escaped version of the UTF-8 + name in uname and add ouname for the display version of zuname. + zip.c, zip.h, zipfile.c (Ed) +13. Add zipmessage_nl() that can output to the screen and to the log + file like zipmessage(), but can write lines without a newline. + zip.c, zip.h, zipcloak.c, zipnote.c, zipsplit.c (Ed) +14. Update help for -FF and Unicode. zip.c (Ed) +15. Change > to >= for byte message check to avoid -0 (negative zero). + zip.c (Ed) +16. Add -su show unicode option which adds escaped unicode paths to + -sf. Also uses show_files = 3. zip.c (Ed) +17. Update comments for -UN and -X. zip.c (Ed) +18. Add support for new AppNote bit 11 that says standard path and + comment have UTF-8 when -UN=f is used. zip.c (Ed) +19. Fix zipfile name message by replacing value with zipfile. + zip.c (Ed) +20. Add new code for -FF, which processes archives by trying to read + the EOCDR to get split count, then starting with the local + entries. This option does not use the standard code but does + everything itself. Add scanzipf_fixnew(), which tries to read + the EOCDR, then the local entries, then the central directory. + zip.c, zipfile.c (Ed) +21. Update note for ZIP64_CENTRAL_DIR_TAIL_SIZE. zipfile.c (Ed) +22. Put read_Unicode_Path_entry() and read_Unicode_Path_local_entry() + into UNICODE_SUPPORT ifdef. zipfile.c (Ed) +23. Add zuqcmp() and zubcmp() to support Unicode sorted list of + paths. zipfile.c (Ed) +24. Update zsearch() to also search unicode paths. zipfile.c (Ed) +25. Split out iname in read_Unicode_Path_entry() for debugging. + Should put it back. Update Unicode mismatch warning. + zipfile.c (Ed) +26. Update Unicode in readlocal(). zipfile.c (Ed) +27. Add more Unicode support to scanzipf_regnew(). zipfile.c (Ed) +28. Add support for fix = 2 to zipcopy(). Add checks and warnings, + but allow scan to continue when can. Use local data to fill + in central directory fields in case no central directory entry + for local entry. zipfile.c (Ed) +29. Add get_win32_utf8path() to get UTF-8 from Windows if can. + zipfile.c (Ed) +---------------------- November 7th 2006 version 3.0f21 ----------------------- + 1. Add crude data descriptor support to -FF in bfcopy() that should be + updated by release. fileio.c (Ed) + 2. Change %d to %s and use zip_fzofft() to format zoff_t byte count. + zipfile.c (SMS, Ed) + 3. Call local_to_oem_string() for only WIN32 in zipcopy(). zipfile.c + (SMS, Ed) +---------------------- November 29th 2006 version 3.0f22 ----------------------- + 1. Change ' to " in extended help. zip.c (Ed) + 2. Change -dv disk number display to indisk>outdisk. zip.c (Ed) + 3. Finish -FF fix option. Move detailed output to require -v. zip.c (Ed) + 4. Add note to help to use -v with -FF to see details. zip.c (Ed) + 5. Add -sU option to view only Unicode names when exist. zip.c (Ed) + 6. Change default dot size in verbose from every buffer to 10 MB. zip.c (Ed) + 7. Exit if --out and in path same as out path. zip.c (Ed) + 8. Remove verbose information when fixing archive. zip.c (Ed) + 9. Initialize in disk to 0, but still problem with disk number of first entry + for each disk lagging by 1. zip.c (Ed) +10. Consistently use ZE error codes for exit from ask_for_split_read_path. + zipfile.c, zip.c (Ed) +11. Seek back when fix finds bad entries. Also skip last entry of split + if next split is missing. Should check if entry completed. zip.c (Ed) +12. Add messages to -sd for writing the central directory, replacing the old + zip file, and setting file type. zip.c (Ed) +13. Don't set file type on stdout. zip.c (Ed) +14. Increase errbuf from FNMAX + 81 to FNMAX + 4081. zip.h (Ed) +15. Add skip_this_disk, des_good, des_crc, des_csize, and des_usize globals + for -FF and reading data descriptors. Change note on display_volume. + Add global skip_current_disk. zip.h, globals.c (Ed) +16. BFWRITE_HEADER define now also does data descriptor. zip.h (Ed) +17. Skip zipoddities() if fix. Maybe can later add back. zipfile.c (Ed) +18. Update fix messages. zipfile.c (Ed) +19. Allow user to end archive early using ZE_EOF. zipfile.c, fileio.c (Ed) +20. Only show split numbers and offsets for -FF if verbose. zipfile.c (Ed) +21. Handle spanning signature at top of split archive. zipfile.c (Ed) +22. Only close in_file if open. zipfile.c (Ed) +23. Add note if no .zip and only splits suggest use -FF. zipfile.c (Ed) +24. In putlocal() and putcentral() only convert to OEM if z->vem == 20. + zipfile.c (Ed) +25. Do not OEM convert archive comment as PKWare says this should + be ASCII. zipfile.c (Ed) +26. Fix swap of siz and len and LOCSIZ and LOCLEN. zipfile.c (Ed) +27. Call read_Unicode_Path_local_entry() before OEM conversion so Unicode + checksum checks iname before conversion. zipfile.c (Ed) +28. Only check if local and central crc match if not stream entry. + zipfile.c (Ed) +29. Keep data descriptors if fix == 2, but need to look at this. + zipfile.c (Ed) +30. Fix bug adding up header bytes in n by adding 4 for signature. + zipfile.c (Ed) +31. If fix == 2 use local crc for central, otherwise use central crc + for local. zipfile.c (Ed) +32. In zipcopy(), check data descriptor and skip if not correct one. + zipfile.c (Ed) +33. Add SH, LG, and LLG macros from zipfile.c to allow reading the data in + the data descriptor. fileio.c (Ed) +34. In bfcopy(), read and check the data descriptor if n == -2. If + run out of bytes before find descriptor, return error. fileio.c (Ed) +35. In ask_for_split_read_path(), increase buf to SPLIT_MAX_PATH + 100, + fix bug by adding "- 1", set split_dir = "" if current directory, + and update prompts to add skip and end choices. Add skip and end + choices. fileio.c (Ed) +36. Increase buffer for fgets to SPLIT_MAXPATH. fileio.c (Ed) +37. Update WhatsNew. WhatsNew (Ed) +---------------------- December 10th 2006 version 3.0f23 ----------------------- + 1. Handle additional ODS5 issues by upper casing many symbols and file names. + vms/build_zip.com, vms/collect_deps.com, vms/descrip.mms, + vms/descrip_mkdeps.mms, vms/descrip_src.mms, vms/find_bzip2_lib.com (SMS) + 2. Update VMS Find Help Library code. vms/hlp_lib_next.com (SMS) + 3. Instead of tempname use temp_name as parameter to avoid function + tempname(). zipsplit.c, zipnote.c, zipcloak.c, zip.c (Ed) + 4. If fixing archive with -FF and no EOCDR to get disk count, see if top of + archive has spanning signature or local header and guess if it is + single-disk archive, then ask user to confirm. zipfile.c (Ed) + 5. For Unix where NO_MKSTEMP is not defined, replace mktemp() with mkstemp() + that avoids a race condition. zip.c, zipcloak.c, zipnote.c, fileio.c (Ed) + 6. Eliminate mkstemp() warning by using mkstemp() instead of mktemp() for + Unix. Only for UNIX and if NO_MKSTEMP is not defined. Many OS do not + have mkstemp(). zipcloak.c, zipnote.c, zip.c, fileio.c (Ed) + 7. If UNICODE_SUPPORT and UNIX then try to switch to UTF-8 locale to allow + displaying of Unicode, otherwise just get escapes. This results in some + characters displaying as whitespace if needed fonts, such as East Asian, + are not installed. zip.c (Ed) + 8. If new global unicode_escape_all is set, then escape all non-ASCII + characters when converting Unicode file path. This allows viewing paths + as escapes on Unix that would otherwise be white space. If not set, any + characters that map to the current locale are returned as is. Can only + display if either supported as base by the OS or fonts installed. Set + using -UN=escape option. zip.c, fileio.c, zip.h, globals.c (Ed) + 9. Update extended help for Unicode. zip.c (Ed) +10. All variables used by Win32 in global.c should now be initialized at + start so dll is initialized each call. zip.c (Ed) +---------------------- January 1st 2007 version 3.0f24 ----------------------- + 1. Fix a problem when building with (old, obsolete) IM attribute encoding + combined with bzip2 support. vms/descrip_src.mms (SMS) + 2. Update WHATSNEW. WhatsNew (Ed) + 3. Update README. ReadMe (Ed) + 4. Remove in_crc code. Too involved to implement but may look at later. + fileio.c, globals.c, zip.c (Ed) + 5. Use 0x50 and 0x4b for 'P' and 'K' in signatures to handle EBCDIC case. + zipfile.c, fileio.c (Ed) + 6. Implement new -FS file sync option that deletes entries missing on the + file system from an archive being updated. globals.c, zip.c (?, Ed) + 7. Update help. zip.h, zip.c (Ed) + 8. Include scanning files dots when update small but new file scan long. + zip.c (Ed) + 9. Ask if single-file archive when using -FF and can't tell. zipfile.c (Ed) +10. Display message when entry would be truncated. zipfile.c (Ed) +11. Check for VMS_IM_EXTRA. Update bzip2 support for VMS. Change + destination directory if large-file enabled. vms/build_zip.com, + vms/descrip_src.mms (SMS) +12. Change parameters for VMS bzip2 search. vms/find_bzip2_lib.com (SMS) +---------------------- January 12th 2007 version 3.0f25 ----------------------- + 1. Incorporate faster crc32.c including the Rodney Brown changes (originally + implemented in the zlib project) from UnZip, which includes the + IZ_CRC_BE_OPTIMIZ and IZ_CRC_LE_OPTIMIZ optimizations when those symbols + are defined. These modifications include: + - enlarge unrolling of loops to do 16 bytes per turn + - use offsets to access each item + - add support for "unfolded tables" optimization variant + crc32.c (Christian) + 2. As the crc32.c module now includes crc table generation, remove crctab.c. + crctab.c (remove) (Christian) + 3. Update crc i386 assembler code from UnZip (details see above). + win32/crc_lcc.asm, win32/crc_i386.asm, win32/crc_i386.c, crc_i386.S + (Christian) + 4. Guard against redefinition of symbols @CodeSize and @DataSize in memory + model setup section to work around Open Watcom (version 1.6) wasm + assembler problem. msdos/crc_i86.asm (Christian) + 5. Change type of keys[] array for new crc, add IZ_CRC_BE_OPTIMIZ, and + use new crypt crc table. Use header buffer instead of buf for header. + crypt.c, crypt.h (Christian) + 6. Update version and remove crc table. crypt.h (Christian) + 7. Add crc32.h, change sprintf() format for disk number from d to lu as + can go over 16-bit, remove crc32u(). fileio.c (Christian) + 8. Update to use new crc. msdos/makefile.bor, msdos/makefile.dj1, + msdos/makefile.dj2, msdos/makefile.emx, msdos/makefile.msc, + msdos/makefile.tc, msdos/makefile.wat, unix/Makefile, + vms/build_zip.com, vms/descrip_deps.mms, vms/descrip_src.mms, + vms/osdep.h, win32/makefile.bor, win32/makefile.dj, win32/makefile.emx, + win32/makefile.gcc, win32/makefile.ibm, win32/makefile.lcc, + win32/makefile.w10, win32/makefile.w32, win32/makefile.wat, + win32/makenoas.w32, win32/vc6/zip.dsp, + win32/vc6/zipcloak.dsp, win32/vc6/zipnote.dsp, win32/vc6/zipsplit.dsp, + win32/vc6bz2/zip.dsp, win32/vc6bz2/zipcloak.dsp, win32/vc6bz2/zipnote.dsp, + win32/vc6bz2/zipsplit.dsp, windll/visualc/dll/zip32.dsp, + windll/visualc/dll/zip32.mak, windll/visualc/lib/zip32.dsp, + win32/visualc/lib/zip32.mak (Christian) + 9. Include crc32.h. Make variable uname local in proc_archive_name(). + Remove unused num and new_base_path. Change %02d to %02l2 for + disk number in print format. Remove crc32u() as now use crc32(). + Add parentheses around conditions in loops. Use 0 instead of NULL + for zwchar. fileio.c (Christian) +10. Add z_uint4 defines from crypt.c to tailor.h. Move uch, ush, and ulg + typedefs before tailor.h include which needs them. tailor.h, zip.h (SMS) +11. Include crc32.h. change add_name() to return not int but long + since number of command line arguments can exceed 16 bits. Cast + variable option to (int) for subtraction. Change 0x400 to 0x400L. + Add braces to show_files print block. zip.c (Christian) +12. Add warning if use -F or -FF without --out. Change defined(NO_MKSTEMP) + to !defined(NO_MKSTEMP). zip.c (Ed) +13. Define EC64LOC and EC64REC for size of Zip64 EOCD Locator and Zip64 + EOCD Record. Add extern for crc_32_tab. Move crc32() to crc32.h. + zip.h (Christian) +14. Add crc.h. zipcloak.c (Christian) +15. Include crc32.h. Comment out scanzipf_reg() and scanzipf_fix() as + no longer used, which are left in for now for comparison. Cast + blocksize to extent for malloc(). Instead of 0x10000 malloc 0xFFFF for + extra field block so fits in 16 bits. Instead of crc32u() use crc32(). + Only do lflg != flg check for fix == 2. Add comments to various #endif. + Indent comment. Comment out copy_sig() which is not used. Reduce size + of SCAN_BUFSIZE to EC64REC for MEMORY16. Use ENDHEAD for EOCDR size. + Change %u to %lu in print formats for disk count. Use EC64LOC for size + of Zip64 EOCD Locator. Use EC64REC for size of Zip64 EOCD Record. + Add streaming and was_zip64 to ZIP64_SUPPORT. Remove lflg != flg check + in zipcopy(). zipfile.c (Christian) +16. Add note that z-flg & ~0xf check will fail if new bit 12 for UTF-8 paths + and comments is set. Update -FF warning. zipfile.c (Ed) +17. Include crc32.h. Modify tempzn update. Fix comment. Set + z->lflg = z->flg after deflate as deflate may have set bits in z->flg + [Ed, Christian]. Include BZIP2_SUPPORT block in !UTIL block. zipup.c + (Christian) +18. Changes to use crc32.c. acorn/gmakefile, acorn/makefile, amiga/lmkfile, + amiga/makefile.azt, amiga/smakefile, aosvs/make.cli, atari/makefile, + atheos/makefile, beos/makefile, cmsmvs/cczip.exec, cmsmvs/mvs.mki, + cmsmvs/zip.makefile, cmsmvs/zipmvsc.job, cmsmvs/zipvmc.exec, + human68k/makefile, human68k/makefile.gcc, novell/makefile, novell/zip.lnk, + os2/makefile.os2, qdos/makefile.qdos, qdos/makefile.qlzip, tandem/history, + tandem/macros, tandem/tandem.h, theos/makefile, tops20/make.mic, + unix/configure, unix/makefile, win32/makefile.a64 (Christian) +19. Add note to use BZ_NO_STDIO. bzip2/install.txt (Ed) +20. Remove crctab. cmsmvs/zipvmc.exec (Ed) +21. Update comment. macos/source/pathname.c (Christian) +22. Start of manual update. Zip.1 (Ed) +23. Changes to use crc32.c. vms/descrip.mms, vms/descrip_deps.mms, + vms/descrip_mkdeps.mms, vms/descrip_src.mms, vms/vms.c (SMS) +---------------------- January 17th 2007 version 3.0f26 ----------------------- + 1. Add note for UnZip. crypt.c (Christian) + 2. Change current_disk and disk_number from int to ulg. Change num from int + to unsigned int. [Even though a 16-bit system likely won't see more than + 64k disks, it probably should be ulg - Ed] Remove unused mbsize. Change + match from long to int as the number of possible options should always fit + in that. fileio.c, globals.c (Christian) + 3. Use -Gt to force some data into separate data segments so all data fits. + msdos/makefile.msc (Christian) + 4. Move some copyright constants to far to save near space. + revision.h (Christian) + 5. Change u for character from int to unsigned int. util.c (Christian) + 6. Move include of crc32.h from vms/vms.c to vms/vms_pk.c. vms/vms.c, + vms/vms_pk.c (Christian) + 7. Update crci386_.o. win32/makefile.gcc (Christian) + 8. Use NOASM=1 to disable assembler and clear variables when do not. + win32/makefile.w32 (Christian) + 9. Remove unused totalslashes and returnslashes from get_win32_utf8path(). + win32/win32zip.c (Christian) +10. Remove local versions of tempzip and tempzf. + zip.c (Christian) +11. Make options[] far. Change cd_start_disk from int to ulg. Cast -1 to + (ulg) for cd_start_disk. Put here = zftello() in DEBUG defines. + zip.h, zip.c (Christian) +12. Change length of zipfile comment parameter from ush to extent. Change + disk numbers from int to ulg in close_split(), ask_for_split_read_path(), + ask_for_split_write_path(), get_in_split_path(), find_in_split_path(), + get_out_split_path(). Add Far to longopt and name strings in + option_struct. zip.h (Christian) +13. Add far to options[]. zipcloak.c (Christian, Ed) +14. Define write_string_to_mem() only for UNICODE_SUPPORT. Change ulg to + extent for append to mem memory offset and blocksize parameters. Make + at_signature() local. Cast usValue to char. Remove unused oname in + read_Unicode_Path_local_entry(). Remove local definitions of zip64_entry + as Zip is always processing one entry at a time and this is a global + flag for the current entry. Make find_next_signature() and + find_signature() local. Add ZCONST to signature parameter. Make + is_signature() and at_signature() local. Change m, result of fread(), + from int to extent. Reduce SCAN_BUFSIZE from 0x40000 to the size of the + largest header being read. As find_next_signature() is used to scan for + the next signature and that reads a byte at a time, the scan buf is only + used to read in the found headers. Since we skip the extra parts of the + Zip64 EOCDR, all headers are a fixed size. Remove unused variables from + scanzipf_fixnew(). Use ENDCOM for end comment offset. Instead of 64 KB + seek back 128 KB to find EOCDR. Use ENDOFF and ENDTOT for offsets in + EOCDR. Remove tabs. Merge versions of putend(). Update Amiga SFX. + Remove unused offset in zipcopy(). Make size local in zipcopy(). + zipfile.c (Christian) +15. Update putend() comment. zipfile.c (Ed) +16. Add far to options[]. zipnote.c, zipsplit.c (Christian) +17. Add NO_ASM versions of Win32 zipnote, zipsplit, and zipcloak projects. + Add crc32.h and crc32.c to zipsplit and zipnote projects. + win32/vc6/zipsplit.dsp, win32/vc6/zipnote.dsp, win32/vc6/zipcloak.dsp (Ed) +18. Add NO_ASM versions of Win32 bzip2 zipnote, zipsplit, and Zipcloak + projects. Add crc32.h and crc32.c. win32/vc6bz2/zipsplit.dsp, + win32/vc6bz2/zipnote.dsp, win32/vc6bz2/zipcloak.dsp (Ed) +19. Update Win32 dll and lib projects and make files. + windll/visualc/lib/zip32.dsp, windll/visualc/lib/zip32.mak, + windll/visualc/dll/zip32.dsp, windll/visualc/dll/zip32.mak (Ed) +20. Remove space in front of #ifdef and other conditionals that slipped in. + zipfile.c, zipup.c (SMS) +21. Updates for bzip2. vms/bzlib.h, vms/install_vms.txt (SMS) +22. Updates. vms/notes.txt (SMS) +23. Update copyrights. crc32.c, deflate.c, globals.c, revision.h, ziperr.h, + trees.c, win32/nt.c, win32/win32.c, win32/win32i64.c, win32/win32zip.h, + win32/zipup.h (Ed) +24. Update WhatsNew. WHATSNEW (Ed) +---------------------- February 4th 2007 version 3.0f27 ----------------------- + 1. Fix array sizes and loop lengths in wide_to_escape_string(). fileio.c + (Johnny, Ed) + 2. Fix escape_string_to_wide() to handle hex strings, then comment out as + not used. zip.h, fileio.c (Ed) + 3. Use ZIPERRORS() macro instead of ziperrors[] array. zip.c, zipcloak.c, + zipnote.c, zipsplit.c (SMS) + 4. Add VMS-compatible "severity" values, add new ZE_SEV_PERR define to + set when perror() needs to be called, add ZIPERRORS() macro, change + PERR() to use ZE_SEV_PERR, change ziperrors[] to new structure array + to hold error strings, add new VMS facility names and severity codes + assigned by HP to ziperrors[] array, and add new official + VMS_MSG_IDENT. ziperr.h (SMS) + 5. Change ZE_SEV defines to ZE_S to save space and reformat ziperrors[]. + ziperr.h (Ed) + 6. Update install.txt to include generic Unix case. bzip2/install.txt (Ed) + 7. Add creation of message file and add NOMSG message. vms/build_zip.com, + vms/descrip.mms, vms/install_vms.txt (SMS) + 8. Update notes.txt to add changes to program exit status values and changes + to messages. vms/notes.txt (SMS) + 9. Include crc32.h, include ssdef.h, instead of FAB_OR_NAM use FAB_OR_NAML, + add status code summary note detailing old versus new error codes, and if + CTL_FAC_IZ_ZIP is 0x7FFF and OLD_STATUS is defined use old VMS error codes. + vms/vms.c (SMS) +10. Change FAB_OR_NAM to FAB_OR_NAML and remove NAME_DNA, NAME_DNS, NAME_FNA, + and NAME_FNS. vms/vms.h (SMS) +11. Change FAB_OR_NAM to FAB_OR_NAML. vms/vms_im.c, vms/vms_pk.c, + vms/vmszip.c (SMS) +12. Fix compile warning on VC 2005. win32/makefile.w32 (Johnny) +13. Update readmevb.txt and readvb64.txt. windll/vb/readmevb.txt, + windll/vbz64/readvb64.txt (Ed) +14. Change tch from int to ulg in utf8_from_ucs4_char(). Move comments to keep + line lengths to 80 characters. fileio.c (Christian) +15. Update comment for total_cd_entries. global.c, zip.c, zip.h (Christian) +16. Comment out unused Adler-16 code. util.c, zip.h (Christian) +17. Add InterlockedExchangePointer() macro if not defined. Update Initialize() + to use macro. nt.c (Christian) +18. Move zip64 eocd disk and offset variables next to input archive variables. + zip.c (Ed) +19. Remove zipbegset from scanzipf_fixnew() as offsets are ignored when this + is fixing archives. Add comment to cd_total_entries. Remove local + cd_start_disk and cd_start_offset as these are already global. Use + ZIP_UWORD16_MAX when disk number exceeds this to flag use of Zip64. + zipfile.c (Christian) +20. Some comment changes. zipfile.c (Ed) +21. Fix indentation in places. zipsplit.c (Christian) +22. Remove unused variable zfile. zipup.c (Christian) +23. Update manual. zip.1, zip.txt, zipsplit.txt (Ed) +---------------------- February 22nd 2007 version 3.0f28 ---------------------- + 1. Update notes. vms/notes.txt (SMS) + 2. Add stream_lf.fdl to specify carriage control. vms/stream_lf.fdl (SMS) + 3. Update License to also refer to www.info-zip.org and to hopefully provide + an example of misrepresentative use. LICENSE (Ed) + 4. Update Readme. README (Ed) + 5. Update WhatsNew. WHATSNEW (Ed) + 6. Change output archive cd_start_disk and cd_start_offset to input archive + local in_cd_start_disk and in_cd_start_offset in scanzipf_fixnew() and + scanzipf_regnew() to avoid mixing in and out. zipfile.c (Ed) + 7. Update copyright. Remove crc32.h include. vms/vms.c (Christian) + 8. Changes for new crc32. Remove CRC32. Add CRCA_0 and CRCAUO. Add + compiling of crc_i386.S. win32/makefile.emx. (Christian) + 9. Add handlers for better RSXNT and Windows OEM conversions. Add detailed + comments on conversions. win32/osdef.h (Christian) +10. Define CP_UTF8. win32/rsxntwin.h (Christian) +11. Define WIN32_LEAN_AND_MEAN to reduce size of Windows includes. + win32/win32.c, win32/win32zip.c, zip.c (Christian) +12. Use only standard FAT attributes if OEM. win32/win32zip.c (Christian) +13. Add use of INTERN_TO_OEM() and related OEM changes. Add console comment. + zip.c (Christian) +14. Change severity from char to int. Update macros. ziperror.h. (Christian) +15. Update Visual Basic project to clarify some of the code. + windll/vbz64/vbzip.vbp, windll/vbz64/vbzipbas.bas, + windll/vbz64/vbzipfrm.frm (Ed) +16. Update copyright. api.c (Ed) +17. Update format for duplicate entry warning. fileio.c (Ed) +18. Instead of ifdef __RSXNT__ use ifdef WIN32. Define WIN32_LEAN_AND_MEAN. + Use WIN32_CRT_OEM. Change OEM check from vem == 20 to vem & 0xff00 == 0 + and instead of local_to_oem_string() use _INTERN_OEM(). Remove unused + first_CD in scanzipf_fixnew(). Instead of oem_to_local_string() use + Ext_ASCII_TO_Native(). Instead of local_to_oem_string() use + INTERN_TO_OEM(). zipfile.c (Christian) +19. Replace escape from zipsplit man page with '. zipsplit.txt (Christian) +20. Instead of using 20 every time, account for dosify when setting vem. + Update FAT comment. zipup.c (Christian) +------------------------ March 3rd 2007 version 3.0f29 ------------------------- + 1. Remove crctab.c. vms/build_zip.com (SMS) + 2. Add LFLAGS_ARCH. vms/descrip.mms (SMS) + 3. Remove redundant includes descrip.h, rms.h, and atrdef.h. + vms/vmsmunch.c (SMS) + 4. Remove includes descrip.h and rms.h. vms/vmszip.c (SMS) + 5. Only define NO_UNISTD_H if __VAX defined or __CRTL_VER is + less than 70301000, allowing support of the new symbolic + links in VMS. Also use unlink instead of delete if version + above 70000000. vms/osdep.h (SMS) + 6. Formatting changes. vms/notes.txt, vms/install_vms.txt (Christian) + 7. Remove spaces before tabs. win32/makefile.emx (Christian) + 8. Formatting change. win32/osdep.h (Christian) + 9. If -y on VMS open the link not the target file. vms/vms_im.c (SMS) +10. If -y on VMS search for the link, not the target file. vms/vms_pk.c (SMS) +11. Change default for Unicode path mismatch from error to warning, so + processing will continue. zip.c, globals.c (Ed) +------------------------ March 12th 2007 version 3.0f30 ------------------------ + 1. Add bzip2 support for the reduced no stdio bzip2 library for VMS and Unix. + Use libbz2_ns_.olb for VMS bzip2 library which is compiled from the VMS + version of bzip2 with the BZ_NO_STDIO flag set. This flag removes most + standard bzip2 stdio support and enables using a callback routine for + errors. zbz2err.c, unix/Makefile, vms/build_zip.com, vms/descrip.mms, + vms/descrip_deps.mms, vms/descrip_src.mms (SMS) + 2. Add zbz2err.c to Win32 vc6bz2 project for support of BZ_NO_STDIO for bzip2. + Modify zbz2err.c to handle different ports. zbz2err.c (Ed) + 3. Update license. zip.h (Ed) + 4. Update copyright. zip.c, zipfile.c, zipup.c, zbz2err.c, revision.h (Ed) + 5. Fix bug where directories got set to ver 4.6 in local headers instead of + ver 1.0 when using bzip2. zipfile.c, zipup.c (Ed) + 6. Minor updates to INSTALL. INSTALL (Ed) + 7. Minor updates to README. README (Ed) + 8. Add BZ_NO_STDIO to vc6bz2 projects. Error routine seems to work. + win32/vc6bz2 (Ed) + 9. Set bit FAB$M_BIO (.fab$v_bio) in the FAB when using sys$open() on a + symlink. vms/vms_im.c (SMS) +10. Change sys$disk to SYS$DISK. vms/build_zip.com (SMS) +11. Update extended help. zip.c (Ed) +12. Update bzip2 install. bzip2/install.txt (Ed) +------------------------ March 19th 2007 version 3.0f31 ------------------------ + 1. Define bz2_olb as LIBBZ2_NS.OLB. Change LIBBZ2.OLB to bz2_olb. Use + ZZBZ2ERR.C error callback for bzip2. vms/build_zip.com (SMS) + 2. Change NO_SYMLINK to NO_SYMLINKS to be consistent with UnZip. tailor.h, + acorn/osdep.h, macos/osdep.h, tops20/osdep.h, vms/osdep.h (SMS) + 3. Minor note changes. Add section on Symbolic Links. vms/notes.txt (SMS) + 4. Update copyright. globals.c (Ed) + 5. Update License with official copy. LICENSE (Greg, Ed) + 6. Update Readme. README (Ed) + 7. Add support for NO_BZIP2_SUPPORT. tailor.h (Ed) + 8. Add common compiler flags to Install. INSTALL (Ed) + 9. Remove SPLIT_FILE define. zip.c (Ed) +10. Minor updates to extended help. zip.c (Ed) +11. Modify Makefile to also build bzip2 library if found. Split $MAKE + ("make -f unix/Makefile") into $MAKE and $MAKEF, leaving $MAKE as defined by + Make and defining $MAKEF to "-f unix/Makefile". Add clean_bzip2 target. + unix/Makefile (SMS) +12. Modify configure to handle compiling bzip2. unix/configure (SMS) +13. Remove linking bzip2 with utilities. Other changes. unix/Makefile (Ed) +14. Change bzip2 wrong library errors to warnings. Put back OS bzip2 library + check. Only compile bzip2 if in bzip2 directory. unix/configure (Ed) +15. More modifications to Makefile and configure to only allow compiling in + the bzip2 directory. unix/Makefile, unix/configure (Ed) +------------------------ March 27th 2007 version 3.0f32 ------------------------ + 1. Modify configure and Makefile to only allow compiling bzip2 in the Zip bzip2 + source directory. unix/Makefile, unix/configure (SMS, Ed) + 2. Update bzip2 installation instructions. bzip2/install.txt (SMS, Ed) + 3. Remove need for BZIP2_USEBZIP2DIR define by using an appropiate include dir + specification (-I ../../bzip2) when needed. zip.c, win32/vc6bz2/zip.dsp, + unix/configure (SMS, Ed, Christian) + 4. Update VC6 readme. win32/readmeVC.txt (Christian, Ed) + 5. Add crc32.h to VC projects. Add assembler group to zipcloak, zipnote, and + zipsplit projects. Add BZ_NO_STDIO to all configurations with bzip2 so + reduced bzip2 code is used. win32/vc6/zip.dsp, win32/vc6/zipcloak.dsp, + win32/vc6/zipnote.dsp, win32/vc6/zipsplit.dsp (Christian) + 6. Update VC6bz2 readme. win32/readVCBZ.txt (Christian, Ed) + 7. Modify bzip2 VC6 workspace to use standard zipcloak, zipnote, and zipsplit + projects as they don't need bzip2. win32/vc6bz2/zip.dsw (Christian) + 8. Fix zlib flag problem by properly setting and clearing deflInit flag to + initialize and release resources. zipup.c (Bill Brinzer, Christian) + 9. Update copyright. crypt.h, api.c, tailor.h, fileio.c, ziperr.h, + zipsplit.c, zipnote.c, zipcloak.c, util.c (Ed) +------------------------ April 25th 2007 version 3.0f33 ------------------------ + 1. Fix -dd display_dots option for VMS. Fix adding value for -ds to command + line. Fix /NAMES = AS_IS for older header files. cmdline.c (SMS) + 2. Add Win32 wide scan support. In fileio.c add Win32 wide functions lastw(), + msnamew(), newnamew(), wchar_to_wide_string(), is_ascii_stringw(), + wchar_to_local_string(), and wchar_to_utf8_string(). In globals.c + add no_win32_wide that is true if the wide versions of calls like + GetFileAttributesW() do not work as on Win9x without the Unicode kit. + In tailor.h define zwstat for stats that use wchar_t strings and + defines SSTATW and LSSTATW. In util.c add isshexpw() and recmatchw() + and dosmatchw() for matching using wchar_t strings. In win32.c add + FSusesLocalTimeW(), IsFileSystemOldFATW(), GetFileModeW(), GetLongPathEAW(), + and zstat_zipwin32w(). In win32zip.c add zdirscanw structure, + GetDirAttribsW(), zDIRSCANW, readdw(), wild_recursew(), procname_win32w(), + OpenDirScanW(), GetNextDirEntryW(), CloseDirScanW(), procnamew(), + local_to_wchar_string(), wchar_to_utf8_string(), in wild() code to + check if W versions are supported and send zip down byte or wide path, + ex2inw(), in2exw(), and filetimew(). In zipup.h define zwopen to use + wide paths. In zipup.c if supported use filetimew() and zwopen(). + In zip.h add namew, inamew, and znamew to zlist and flist. In zip.c + remove duplicate initialization of use_wide_to_mb_default, force_zip64, + zip64_entry, and zip64_archive. Use filetimew() if UNICODE_SUPPORT and + using wide paths for directory scan. Remove old 8.3 path Unicode fix as + now use wide paths and get all where the 8.3 kluge missed paths where + characters in path needed multiple code pages. Changes to bit 11 Unicode + but still not ready. fileio.c, globals.c, tailor.h, util.c, zipup.h, + win32/win32.c, win32/win32zip.c, win32/win32.h, zipup.c, zip.c (Ed) + 3. Update copyright. Don't define UNICODE_SUPPORT if already defined. + Define MATCHW and zstat_zipwin32w(). win32/osdep.h (Ed) +------------------------ April 29th 2007 version 3.0f34 ------------------------ + 1. Add temporary option -sC to test Unicode file creation enabled with + UNICODE_TEST define. zip.c, fileio.c (Ed) + 2. On Unix display control characters as ^X as UnZip. (SMS) fileio.c + 3. Update extended help. zip.c (Ed) + 4. Fix bugs in Unicode changes. zip.c, fileio.c (SMS, Ed) + 5. Add NAMES AS_IS support. Handle root dir [000000]. zip.h, + vms/install_vms.txt, vms/vmszip.c, vms/vmsmunch.c (SMS) + 6. Add global zipfile_exists to handle missing zipfile errors better. zip.h, + globals.c, zip.c (Ed) + 7. Add functions utf8_to_escape_string(), wide_to_escape_string(), + local_to_escape_string(), utf8_to_wchar_string(), and + rename wide_to_escape_string() to wide_char_to_escape_string(). fileio.c, + win32/win32zip.c, zip.h (Ed) + 9. Free f->inamew in fexpel(). Use zuname for matching. fileio.c (Ed) +10. Fix memory bug by setting z->namew, z->inamew, and z->znamew to NULL. + Set f->namew, f->inamew, and f->znamew to NULL for new file in newname(). + Free wide_string in local_to_utf8(). Other Unicode fixes. Add namew, + inamew, and znamew to freeup(). fileio.c, win32/win32zip.c, zip.h (Ed) +11. Move wchar functions only used by Windows to win32zip.c. fileio.c, + zip.h (Ed) +12. Fix spelling in manual. zip.1 (SMS, Ed) +13. Add zuebcmp() for Unicode. zipfile.c +14. Open files to read using wide name as input path. zipup.c (Ed) +15. Update help. zip.c (Ed) +16. Change -TT long option from --unzip-path to --unzip-command. zip.c (Ed) +17. Update Manual to include section on Unicode, add -TT option, make some + changes to Unicode in other sections, update copyright at bottom, and + some small changes to wording and examples. man/zip.1, zip.txt (Ed) +18. Put #ifdef WIN32 around WIN32 blocks. zipfile.c (Ed) +------------------------- May 14th 2007 version 3.0f35 ------------------------- + 1. Update VMS to include new options. vms/cmdline.c, vms/zip_cli.cld (SMS) + 2. Update VMS help. vms/vms_zip.rnh (SMS) + 3. Minor updates to VMS help. vms/vms_zip.rnh (Ed) + 4. Create global filter_match_case that defaults to 1 (case-sensitive). zip.c + zip.h, globals.c (Ed) + 5. Add option -fc to fold case for case-insensitive matching in filter(). + Currently enabled only for WIN32. zip.c, win32/osdep.h (Ed) + 6. Change (action == DELETE || action == FRESHEN) to filter_match_case in + PROCNAME() define. I just couldn't figure out what was going on here and + why the case flag was controlled by this. zip.c (Ed) + 7. Update WhatsNew. WHATSNEW (Ed) +------------------------- May 17th 2007 version 3.0f36 ------------------------- + 1. Touch date on generated file. vms/ZIP_MSG.MSG (SMS, Ed) + 2. Update Betas readme to include Release Candidates. Betas_Readme.txt (Ed) + 3. Update Zip 3.0f announcement. zip30f.ann (Ed) + 4. Minor updates to VMS help. vms/cvthelp.tpu, vms/vms_zip.rnh (SMS) + 5. Major changes to VMS CLI help. vms/zip_cli.help (SMS, Ed) + 6. Update license. revision.h (Ed) +------------------------- May 21st 2007 version 3.0f37 ------------------------- + 1. Rename -fc (fold case) to -ic (ignore case) which may be more intuitive. + zip.c (Ed) + 2. VMS CLI updates for new options. vms/cmdline.c, vms/vms_zip.rnh, + vms/zip_cli.cld, vms/zip_cli.help (SMS) + 3. Updates to support Watcom C, mingw, djgppv2 and msc-16-bit, including + supporting wide stat and compare calls and work-around for problem with + "no symlink support" detection. tailor.h, util.c, zip.c, win32/osdep.h, + win32/win32.c, win32/win32/zipup.h (Christian) +------------------------- May 29th 2007 version 3.0f38 ------------------------- + 1. Update description. file_id.diz (Ed) + 2. Handle better when not splitting and run out of disk space. Also, for split + method 1 (automatically write all splits to same place) exit if run out of + space instead of filling available space with near empty splits. For split + method 2 require splits to be at least 64K bytes (the minimum split size). + fileio.c (Ed) + 3. Add line break in ziperr() if message line has been started. zip.c (Ed) + 4. In ziperr() don't close output handle y if same as current_local_file handle + and just closed that. zip.c (Ed) + 5. Change default definition of PROCNAME() to handle new filter_match_case flag + and restore backward compatibility. zip.c (Christian, Ed) + 6. Add note detailing definition of PROCNAME(). zip.c (Ed) + 7. Remove nonlocalpath parameter from procname_win32() and procname_win32w() + and variables nonlocal_path and nonlocal_name as this is not used now that + unicode is implemented in WIN32 using the wide calls. + 8. Enable ignore case option for VMS. zip.c (SMS) + 9. Update -v and other updates in manual. man/zip.1 (Christian, Ed) +10. Updates for Watcom C and Win32 symlinks. win32/osdep.h (Christian) +11. Fix historic problem with VAX seeking. zipfile.c (SMS) +12. Add NAM_M_EXP_DEV. Add determination if device is in file specification. + If device name in file specification do ODS2 and ODS5 down-casing. + Define explicite_dev(). vms/vms.h, vms/vmszip.c (SMS) +------------------------- June 4th 2007 version 3.0f39 ------------------------- + 1. Update osdep.h to use new filter_match_case flag. vms/osdep.h (SMS) + 2. Fix unterminated string bug and trim extra allocated space in + local_to_display_string(). fileio.c (Ed) + 3. Updated extended help for -u and -ic options. zip.c (Ed) + 4. Update Manual. man/zip.1, zip.txt (Ed) +------------------------- June 15th 2007 version 3.0f40 ------------------------- + 1. Update Unicode Path and Unicode Comment descriptions based on suggestions + from WinZip. proginfo/extrafld.txt (Steve Gross, Ed) + 2. Update descriptions for Add, Update, and Freshen in the manual. man/zip.1 + (Christian) + 3. Update default definition of PROCNAME() to use filter_case_match flag to + turn off case matching in filter(). zip.c (Christian) + 4. Update WhatsNew. WHATSNEW (Ed) + 5. Update announcement. zip30f.ann (Ed) + 6. Update manual. man/zip.1, zip.txt (Ed) +------------------------- July 7th 2007 version 3.0f41 ------------------------- + 1. Use File Name as Unicode path if UTF-8 flag is set in header. zip.c, + globals.c, zipfile.c, zip.h (Ed) + 2. Update ToDo. TODO (Ed) + 3. Update WhatsNew. WHATSNEW (Ed) + 4. Update ReadMe. README (Ed) + 5. Fix problems with incompatible stat types on Win32. fileio.c, tailor.h, + zip.h, win32/win32.c, win32/win32zip.c, win32/osdep.h (Ed) + 6. Define NO_STREAMING_STORE to turn off storing while streaming. + INSTALL, zipup.c (Ed) + 7. Define UNICODE_ALLOW_FORCE to enable -UN=force option which is now + disabled and would need work. globals.c, zip.h (Ed) + 8. Add global using_utf8 to flag when OS current character set is UTF-8. + If an existing entry has the UTF-8 flag set the flag is kept. If a new + entry needs Unicode and on a UTF-8 system assume the strings are UTF-8 + and set the UTF-8 flag. globals.c, zip.h (Ed) + 9. Update Unicode extra field descriptions. proginfo/extrafld.txt (Ed) +10. Add include directory so can find bzip2 header file when using bzip2 + directory. unix/configure (Ed) +11. Fix wide character wild(), wild_recursew() and OpenDirScanW() for Win32 so + work like the regular versions. win32/win32zip.c (Ed) +12. Update Unicode in manual. Update -W description in manual zip.1 +13. Flush logfile writing. zip.c (Ed) +14. Update extended help for -UN option. Update help for Update to note it + updates files where the OS has a later date. Chance -UN=Exit to -UN=Quit + so can abbreviate to first letter. zip.c (Ed) +15. Fix a bug in readzipfile() when zip used in pipe. Other pipe fixes. zip.c, + zipfile.c (Ed) +------------------------ August 10th 2007 version 3.0f42 ----------------------- + 1. Update error message for -DF. zip.c (Ed) + 2. Add bzipped message to write to log file. zipup.c (Ed) + 3. Update bzip2 install instructions. bzip2/install.txt (Ed) + 4. Move local.h include to tailor.h to fix compiler multiple define. tailor.h, + zip.c (SMS) + 5. Add additional C compiler checks for GNU and HP. unix/configure (SMS) + 6. Fix to build libbz2.a. unix/Makefile (SMS) + 7. Update copyright. acorn/osdep.h, macos/osdep.h, tops20/osdep.h, + vms/vmszip.c, vms/vmsmunch.c, vms/vms_pk.c, vms/vms_im.c, vms/vms.h, + vms/vms.c, vms/osdep.h, win32/rsxntwin.h, win32/osdep.h, win32/nt.c (Ed) + 8. Change zfeeko(file, 0, SEEK_SET) to rewind(file) in ffile_size() so + EOF is always reset. This was creating problems in WIN32 when + NO_ZIP64_SUPPORT was set but LARGE_FILE_SUPPORT was set. zipfile.c (Ed) + 9. Update compile -v descriptions for LARGE_FILE_SUPPORT and ZIP64_SUPPORT to + be more specific as to what each does. zip.c (Ed) +10. Fix bug that added the local header size to the next entry compressed size + giving a wrong compressed size error if splitting and the split occurs when + writing a local header. fileio.c (Ed) +11. Remove UNICODE_TEST define from VC 6 projects. win32/vc6/zip.dsp, + win32/vc6/zipcloak.dsp, win32/vc6/zipnote.dsp, win32/vc6/zipsplit.dsp (Ed) +12. Update extended help. zip.c (Ed) +13. Only output -FF central directory messages in verbose mode. zipfile.c (Ed) +14. Add note about possible bug when copying entries from a split archive. + WHATSNEW (Ed) +------------------------ August 11th 2007 version 3.0f43 ----------------------- + 1. Display locale inside check to avoid NULL locale. zip.c (SMS, Ed) + 2. Add include wchar.h to tailor.h. tailor.h (SMS) +------------------------ August 21st 2007 version 3.0f44 ----------------------- + 1. Remove verbose messages when setting locale as verbose flag is not set yet. + zip.c (SMS, Ed) + 2. Change reading splits message "abort archive" to "abort archive - quit" and + change selection letter from a to q so q quits consistently. For quit, + don't confirm as more annoying than helpful. fileio.c (Ed) + 3. In bfwrite() handle case where a split ends at the end of one entry and + trying to write the next local header forces opening next split. This + caused copying entries from one archive to another to fail if this came up. + Also handle case where a new split is needed while writing central directory + entries. Now close last split and update pointers to point to the new + split. fileio.c (Ed) + 4. Update use of mesg_line_started and add new logfile_line_started to account + for line ends in logfile. fflush() output. zip.c, zip.h, globals.c (Ed) + 5. Move setting split size if input archive is split and split_size not set + to after archive is read. zipfile.c, zip.c (Ed) + 6. Update Manual to describe Unicode as implemented and note that old splits + are not automatically excluded. man/zip.1, zip.txt (Ed) + 7. Update WhatsNew to remove note that creating and copying split archives + is broke as it seems fully working now. WHATSNEW (Ed) + 8. Update announcement. zip30f.ann (Ed) +------------------------ August 31st 2007 version 3.0f45 ----------------------- + 1. Unicode fix for VMS. tailor.h (SMS) + 2. Add member current to zlist structure to flag when an archive entry is + current with the matching OS file using file time and size. This is used by + File Sync to copy current entries from archive. zip.h, zip.c (Ed) + 3. Comment out zip info verbose extra data message as this message does not + seem to add much. zipfile.c (Ed) + 4. Add local and central directory Version Needed To Extract to mismatch + warning. Update warning text. zipfile.c (Ed) + 5. Add function BlankRunningStats() to output blanks for the running stats + part of the line to use when displaying stats for entries not on the mark + list so all output lines up. zip.c + 6. Add -FS to extended help as new mode. zip.c (Ed) + 7. Update description of -FF to remove Assume Worst. zip.c (Ed) + 8. Add all_current flag that is set if all entries in archive are current and + skip updating archive if -FS and all entries are current. zip.c (Ed) + 9. Change argv[] to args[] for "try: zip" error message as message depends on + new argument order in args where options are now at beginning. zip.c (Ed) +10. For File Sync, copy entries to new archive if file time and size are the + same. If verbose, output ok when copying current entries, otherwise no + message when current_entry. Set all_current to 0 if an entry not marked or + a file not on OS as need to avoid the All Current message in these cases to + catch only deletions. zip.c (Ed) +11. Initialize variables excluding zipstate and setjmp() if USE_ZIPMAIN defined + to fix bug when recall zipmain(). zip.c (Ed) +12. Update Manual. zip.1, zip.txt (Ed) +13. Update WhatsNew. WHATSNEW (Ed) +14. Update announcement. zip30f.ann (Ed) +----------------------- September 5th 2007 version 3.0f46 ---------------------- + 1. Move write of local header after when GPB11 UTF-8 bit set in putlocal(). + zipfile.c (Ed) + 2. Change to uppercase for compatibility. vms/install_vms.txt (SMS) + 3. Set cenbeg and bytes_this_split to fix grow. Check if grow split archive. + zipfile.c, zip.c (Ed) +----------------------- September 14th 2007 version 3.0f47 -------------------- + 1. Include address for new Info-ZIP forum. Add note on 16-bit OS support. + Add note about text file line ends. README (Ed) + 2. Update WhatsNew to include latest on Unicode. Add section on plans for + Zip 3.1. WHATSNEW (Ed) + 3. Minor change in note for Unicode in extended help. zip.c (Ed) + 4. Modify definitions of Unicode extra fields based on discussions with PKWare + and WinZip. proginfo/extrafld.txt (Ed) + 5. Add note on UTF-8 flag. INSTALL (Ed) + 6. Minor updates to ToDo list. Needs more work. TODO (Ed) + 7. Update announcement. zip30f.ann (Ed) + 8. Change definition of IZ_OUR_BZIP2_DIR to be compatible with Configure and + to work with HP-UX. unix/Makefile (SMS) +------------------------ September 24th 2007 version 3.0f --------------------- + 1. Update extended help Unicode description. zip.c (Ed) + 2. Update Readme. README (Ed) + 3. Fix case of define identifying IA64. vms/vms.c (SMS) + 4. Update announcement date. zip30f.ann (Ed) + 5. Update Unicode extra field definitions based on changes proposed for + AppNote. extrafld.txt (Ed) +------------------------ October 17th 2007 version 3.0g01 --------------------- + 1. Can get stuck on open Unix FIFO so default to skip and add option -FI to + enable reading FIFO. Add global allow_fifo. zip.c, zip.h, globals.c + (Willus 0, Ed) + 2. As problems with MinGW with wide-character paths, disable wide-character + Unicode support. zip.c, unix/unix.c (Willus 0, Ed) + 3. Update manual installs to include zipcloak.1, zipnote.1, and zipsplit.1 + pages. unix/Makefile (Ed) + 4. Update Solaris packages. unix/Packaging/pkginfo.in, + unix/Packaging/postinstall, unix/Packaging/preinstall.in, + unix/Packaging/prototype (SMS) +------------------------ October 30th 2007 version 3.0g02 --------------------- + 1. Fix bug in get_in_split_path() where look for .zip split when attempting + to open archives without a .zip extension, even when a single file archive + like jar file. fileio.c (Gabriele (balducci@units.it), Ed) + 2. Fix bug where temp file got created in current working directory on Unix + by giving entire archive path to mkstemp() as template. fileio.c, zip.c + (Willus, Ed) + 3. Use 64-bit output functions for bits_sent. trees.c (SMS) + 4. Add -FF to fixfix -sd messages to make different from identical main + messages. zip.c (SMS, Ed) + 5. If quiet do not ask for splits and all splits must be in same location. + zipfile.c (Ed) + 6. Clean up making zip manuals. unix/Makefile (Ed, SMS) + 7. Add clean_exe to make. unix/Makefile (SMS) + 8. Update to VMS Notes, including adding details on symlinks, -V, and UTC + dates times. vms/notes.txt (SMS) + 9. Fix bug in wild() when calling wile_recursew() where qw should be + pointing inside pw. win32/win32zip.c (Willus, Ed) +10. Fix bug where is_ascii_string() fails when passed a NULL string. This + may fix problem where the CentOS mbstowcs() function is returning -1 when + trying to convert a file name with a bad character (0xe6), causing + local_to_wide_string() and then local_to_utf8_string() to return NULL, so + f->uname gets NULL and so is_ascii_string() fails with SIGSEGV. fileio.c + (Willus, Ed) +------------------------ October 31st 2007 version 3.0g03 --------------------- + 1. Add handling of -b temp directory when opening splits in bfwrite() using + mkstemp(). fileio.c (SMS, Ed) +------------------------ November 3rd 2007 version 3.0g04 --------------------- + 1. Move show_files to global so can avoid split warning for -sf. zip.c, + globals.c, zip.h, zipfile.c (Ed) + 2. Account for -b tempath when opening temp file. zip.c, zipnote.c, + zipcloak.c (SMS, Ed) +------------------------ November 4th 2007 version 3.0g05 --------------------- + 1. Minor fixes to fdopen calls. zipcloak.c, zipnote.c (SMS, Ed) +------------------------ November 4th 2007 version 3.0g06 --------------------- + 1. Add negation to -db, -dc, -dd, -dg, -du, -dv display options. zip.c (Ed) + 2. Put back UNICODE_SUPPORT no_win32_wide code left out in previous fix. + win32/win32zip.c (Willus, Ed) +------------------------ November 21st 2007 version 3.0g07 --------------------- + 1. Fix bug preventing newline in some cases in zipmessage(). zip.c (Ed) + 2. Update Unicode help. zip.c (Ed) + 3. Update -sd messages. zip.c (Ed) + 4. Add filetimew() for Unicode case. zip.c (Ed) + 5. Add ClearArchiveBitW() for Win32 wide. zip.c, zip.h, win32/win32.c (Ed) + 6. Only ask for .zip split if path ends in .znn or .znnn where n 0 to 9. This + allows -FF to work on .exe sfx files without adding .zip. zipfile.c (Ed) + 7. Fix bug where only backed up 20 bytes to find Z64 EOCD Locator. Now back + up 24 bytes to include size of Z64 EOCD Locator signature. This prevented + reading and updating archives greater than 4 GB. zipfile.c (Ed) + 8. If -FF on Win32 initialize wide strings namew, inamew, and znamew to NULL. + zipfile.c (Ed) + 9. Add #include to support towupper(). tailor.h (SMS) +------------------------ December 4th 2007 version 3.0g08 --------------------- + 1. Update dot_size comment. globals.c (Ed) + 2. Update Compression in extended help. zip.c (Ed) + 3. Add extended help on self extractor -A and -J. zip.c (Ed) + 4. Update VMS SYMLINK version information. zip.c (SMS) + 5. Remove not final from Unicode version information as final now. zip.c (Ed) + 6. Remove apparently not needed WINDLL variable retcode. zip.c (Ed) + 7. Fix -A to calculate sfx offset and adjust offsets as it should. zip.c (Ed) + 8. Split -F and -FF used with -A warning to separate warnings. zip.c (Ed) + 9. Add adjusting to can't to that to split archive error. zip.c (Ed) +10. Fix bug for -A that tries to open split by asking for disk 0 instead of + disk 1. Add adjust_offset and cd_total_size variables. Calculate + sfx offset by determining offset of start of central directory. Archives + larger than 4 GB are not supported as sfx archives but these don't seem + to work anyway. Add adjust_offset to Zip64 EOCDR offset and central + directory offsets. zip.c, zipfile.c (Ed) +11. Comment out here debug variable in find_next_signature(). zipfile.c (Ed) +12. Change %2x to %02x as format for parts of a signature in error messages. + zipfile.c (SMS) +13. Add warning adjusting split archives not yet supported. zipfile.c (Ed) +14. Add period to central directory comment. zipfile.c (Ed) +15. Update readme for vb Zip64 project. windll/vbz64/readvb64.txt (Ed) +16. Update comments of VB for Zip64 example. Add SplitSize to VB Zip64 + example. windll/vbz64/vbzipbas.bas, windll/vbz64/vbzipfrm.frm (Ed) +17. Add SourceForge to comment noting where can get the source code. + windll/vbz64/vbzipfrm.frm (Ed) +18. Update WhatsNew. WHATSNEW (Ed) +------------------------ December 12th 2007 version 3.0g09 -------------------- + 1. A few minor changes to extended help. zip.c (Ed) + 2. Uppercase beginning of most -sd messages. zip.c (Ed) + 3. Add spaces between options in some error messages. zip.c (Ed) + 4. Update comments in scanzipf_regnew(). zipfile.c (Ed) + 5. Update scanzipf_regnew() to figure out sfx offset. (Ed) + 6. Uppercase VMS RUNOFF file as apparently needed. VMS_ZIP.RNH (SMS) + 7. Add comments to zipmessage(). zip.c (Ed) + 8. Update extended help and option descriptions. zip.c (Ed) +------------------------ December 20th 2007 version 3.0g10 -------------------- + 1. Fix -F to include -A adjustment check. zipfile.c (Ed) + 2. Change -FF message when find EOCDR. zipfile.c (Ed) + 3. For -FF, reset first CD entry flag in_central_directory when a local entry + is found after CD entries so that another CD entry forces sorting of all + local entries to that point. This allows files with multiple archives in + them to be processed. zipfile.c (Ed) + 4. Add message when a local entry is found after a central directory. + zipfile.c (Ed) + 5. Remove word offset from disk offset location messages. zipfile.c (Ed) + 6. Make Adjust offset message more descriptive. zipfile.c (SMS, Ed) + 7. In scanzipf_regnew(), if adjustment to offsets, add it to + in_cd_start_offset. zipfile.c (Ed) + 8. Allocate cextra only if localz->ext not 0 in zipcopy(). zipfile.c (Ed) +------------------------ December 28th 2007 version 3.0g11 -------------------- + 1. Include definitions of zip64_eocdr_start and z64eocdl_offset in + ZIP64_SUPPORT ifdef block. Add comments for End Of CD Record (EOCDR). + Update comments for adjust offset detection. zipfile.c (Ed) + 2. Change ((uzoff_t)1 << 32) to 0xFFFFFFFF. zipfile.c (SMS, Ed) + 3. Leave off local header detection as not useful when searching for start + of central directory to get adjust offset. Looks like all expected cases + are now covered as long as archive is intact. zipfile.c (Ed) + 4. Update some warning messages. Simplify adjust offset information message. + zipfile.c (Ed) + 5. Add braces to unicode_mismatch if block. zipfile.c (Christian) + 6. Add (void *) cast in InterlockedExchangePointer() mutex calls to fix + compile warnings in MinGW (GCC 3.4.4). win32/nt.c (Christian) + 7. Remove unused nonlocalpath variable. win32/win32zip.c (Christian) + 8. Update betas readme file. betas_readme.txt (Ed) + 9. Partial update to Who list of contributors. proginfo/infozip.who (Ed) +10. Update ReadMe. Create Announcement. README, zip30g.ann (Ed) +11. Update WhatsNew. WHATSNEW (Ed) +------------------------ January 7th 2008 version 3.0g12 -------------------- + 1. Convert Scanning files message to use standard zipmessage_nl() so line + ends are generated when needed. fileio.c (Ed) + 2. Add line ends in DisplayRunningStats() if a display line has been + started. zip.c (Ed) + 3. For the command line listed at the top of the log file, add double + quotes around any arguments that have spaces in them. + zip.c (Ed) + 4. Instead of stdout use standard mesg output stream for show files. + Output new line for show files for display and log file if there was + output on the current line. zip.c (Ed) + 5. Comment out new line output code after zipup() and replace with + call to zipmessage_nl("", 1) to output new line if needed. + zip.c (Ed) + 6. In GetFileMode() and GetFileModeW() when get attributes fails + instead of fprintf(mesg, ...) use zipwarn() so error goes in + log file and new lines are displayed when needed. win32/win32.c (Ed) + 7. In GetSD(), change cbytes from long to ulg. Check cbytes (the + compressed size of the security descriptor) and issue warning if + the compressed security descriptor is greater than 0x7FFF (32k) + as the entire header this extra field is in needs to fit in the + 64k header. Should be a check on the running size of the header + so the actual space remaining is tracked. Maybe in Zip 3.1. If + cbytes OK cast to ush and store. win32/win32zip.c (Ed) + 8. Use zipmessage_nl() for bytes security message so new lines are + handled and message goes in log file. win32/win32zip.c (Ed) + 9. Add new option -RE to enable [list] (regex) matching in DOS and + WIN32 but disable [list] matching otherwise. Default behavior + is restored if ALLOW_REGEX is defined. globals.c, util.c, + zip.h, zip.c (Ed) +------------------------ January 20th 2008 version 3.0g13 -------------------- + 1. Update copyrights to 2008. zip.c, zipcloak.c, zipfile.c, zipnote.c, + zipsplit.c, zipup.c, README (Ed) + 2. Update Who. proginfo/infozip.who (Ed) +------------------------ January 30th 2008 version 3.0g14 -------------------- + 1. Update copyrights. fileio.c, globals.c, revision.h, util.c, zip.h, + win32/win32.c, win32/win32zip.c (Ed) + 2. Updates. README, proginfo/infozip.who (Ed) + 3. Update announcement and WhatsNew. zip30g.ann, WHATSNEW (Ed) + 4. Add ALLOW_REGEX to INSTALL define list. INSTALL (Ed) + 5. Change -sd message. zip.c (Ed) + 6. For bzip2 check for binary and set binary/text flag. Handle -l and -ll + line end conversions for bzip2. zipup.c (Ed) +------------------------ February 3rd 2008 version 3.0g -------------------- + 1. Change && to || to fix logic bug in show files. zip.c (Johnny) + 2. Add CLEAN and CLEAN_ALL VMS targets. vms/descrip_mkdeps.mms (SMS) +----------------------- February 22nd 2008 version 3.0h01 -------------------- + 1. Update some echo statements to use CFLAGS_OPT. Add GNUC check. + unix/configure (SMS) + 2. Only store UID and GID if 16 bit. unix/unix.c (Ed) +----------------------- March 21st 2008 version 3.0h02 -------------------- + 1. Change long Unicode escapes from 8 characters to 6 characters based on + change in UnZip 6.0. fileio.c (Ed) + 2. Put zuebcmp() declaration in #if 0 block as definition already is. This + function would be used to allow Unicode escapes on the command line + without using the -UN=escape option, but the utility of this is still + being determined. zipfile.c (SMS, Ed) + 3. Remove declaration for unused bz_deflate_init(). zipup.c (SMS, Ed) + 4. Add release announcement file, anticipating the long-awaited release. + zip30.ann (Ed) + 5. Update WhatsNew. WHATSNEW (Ed) +----------------------- March 24th 2008 version 3.0h03 -------------------- + 1. Update Unix configure script to better test for modern HP-UX compiler. + unix/configure (SMS) + 2. Updated Beta Readme. betas_readme.txt (Ed) + 3. Update Install. INSTALL (Ed) + 4. Update ReadMe. README (Ed) + 5. Small change to main help screen. zip.c (Ed) + 6. Small update to top of ToDo list. Actual updating of items still + needs to be done. TODO (Ed) +----------------------- April 2nd 2008 version 3.0h04 -------------------- + 1. Update copyright. crc32.h (Christian) + 2. Remove zip.h include. crc32.h (Christian) + 3. Add local prototypes for Unicode functions. Add cast for split size + check. Make many Unicode functions local. #if 0 out currently unused + utf8_chars(). Fix memory leak in wide_to_local_string() by adding + free() for buffer on error return. Fix memory leak in copy_args() on + error return by adding free-args(). Add ZCONST to arg in + insert_arg(). Shorten some lines to less than 80 characters. Add + free() to get_longopt() to fix memory leak. fileio.c (Christian) + 4. Create Win32 versions of wide_to_local_string() and + local_to_wide_string() so can use Win32 conversion functions. + fileio.c, win32/win32.c (Christian) + 5. Update comments for get_option(). fileio.c (Ed) + 6. Update encryption code readme. README.cr (Ed) + 7. Add prototype for recmatchw(). util.c (Christian) + 8. Change count_args() from static to local. util.c (Christian) + 9. Change ifdefs for includes for prototypes for version_info(), + zipstdout(), and check_zipfile() for WINDLL and MACOS and add + check_unzip_version(). zip.c (Christian) +10. Change ifndef NO_SYMLINKS to ifdef S_IFLNK for determining compiler + information. zip.c (Christian) +11. Change UTF-8 locale from en_GB.UTF-8 to .UTF-8. zip.c (Christian) +12. Change cast of -1 for dot_size from uzoff_t to zoff_t. + zip.c (Christian) +13. Change prototype for set_filetype to include parameter char *. + Change prototype of has_win32_wide to include parameter void. + zip.h (Christian) +14. Add prototypes for find_next_signature(), find_signature(), + and is_signature(). Change duplicate prototype scanzipf_regnew() + to missing prototype scanzipf_fixnew(). Change comment for Adler-16 + checksum to CRC-32 checksum as that is being used at that point in + the code. Move multiple uname assignments to common assignment. + Add inameLocal for WIN32_OEM and use define for inameLocal if not + to save memory allocation when not not using WIN32_OEM. Also + change _INTERN_OEM(str1) to INTERN_TO_OEM(src, dst) for OEM + conversion. Format comment for vem to fit in 80 character lines. + zipfile.c (Christian) +15. Change variable a from buffer to a pointer and add abf as the + buffer for zgetline() to handle NULL case. zipnote.c (Christian) +16. Change comments to zipentry comments and zipfile comment in + messages. zipnote.c (Ed) +17. Use uidgid_16bit as flag variable instead of uid_size. Modify + size check that prevents saving Unix UIDs and GIDs in the old + Unix extra field if they are not 16 bits. Change memory + allocation based on uidgid_16bit. Delete unused code for memory + copy for extra field. unix/unix.c (Christian, Ed) +18. Change compiler flag from -zp8 to -Zp8 for LCC Win32. + win32/makefile.lcc (Christian) +19. Add ifndef debug. Add bzip2 support. Add additional compiler + flags. win32/makenoas.w32 (Christian) +----------------------- April 10th 2008 version 3.0h05 -------------------- + 1. Fix bug found by forum poster where Zip stops recursing down a tree + when option -AS is set and a directory without the Windows archive + bit is reached. Now Zip continues down the tree to include files with + the bit set. win32/win32zip.c (forum poster, Ed) + 2. Update comments. win32/osdep.h (Ed) + 3. Update VMS notes to better organize and add information about file + name case. Additional small updates. vms/notes.txt (SMS) + 4. Fix bugs from previous changes to unix. unix/unix.c (SMS, Christian, + Ed) + 5. Add unix IBM support. unix/unix.c (SMS) + 6. Update INSTALL to account for new distribution structure and other + changes. INSTALL (SMS, Ed) + 7. Update bzip2 install readme. bzip2/install.txt (SMS, Ed) + 8. Fix bug noted in forum where -@ and -x generated a "nothing to + select from error" by also checking filelist variable populated by + -@ for entries. zip.c (forum poster, Ed) +----------------------- April 20th 2008 version 3.0h06 -------------------- + 1. Start announcement for Zip 3.0h public beta. zip30h.ann (Ed) + 2. Update beta readme. betas_readme.txt (Ed) + 3. Update case of README.CR. INSTALL (Ed) + 4. Change -W to -ws for option to stop wildcards from scanning directory + boundaries in path. This frees up -W for later use, maybe as extendted + option introducer. zip.c, man/zip.1 (Ed) + 5. Updated date in announcement to May 4th. zip30.ann (Ed) + 6. Added announcement for public beta Zip 3.0h. zip30h.ann (Ed) + 7. Fix large file support for MinGW by checking for compiler environments + before the check for (generic) gcc. zipup.c, win32/osdep.h + (Will, Christian) + 8. Fix large file support for bzip2. Additionally, the "dot printout" + code has also been adapted for LARGE_FILE support. zipup.c + (Will, Christian) + 9. Add comments to top of configure. unix/configure (Ed) +10. Move comment and comment out value size check for UID/GID extra field. + unix/unix.c (Ed) +11. Change case of file ToDo to TODO for consistency and to work with Unix + package. TODO (SMS, Ed) +----------------------- April 26th 2008 version 3.0h07 -------------------- + 1. For -AS, which for Windows only includes files with the archive bit + set, exclude directory entries (by setting -D) as some directories may + not have any files with the archive bit set and so the directory would + be empty. zip.c (Ed) + 2. Fix UID/GID size detection to use byte sizes and remove data fit test. + unix/unix.c (Ed) + 3. Update announcement. zip30h.ann (Ed) + 4. Add new unix extra field with tag 'ux' that stores UIDs/GIDs of 1 to 4 + bytes (8 to 32 bits). unix/unix.c (Ed) + 5. Update VB readme. windll/vbz64/readVB64.txt (Ed) + 6. For Unicode escaped output also show escape for ASCII 7-bit if + isprintable() is false. fileio.c (Ed) + 7. Use locale "en_US.UTF-8" for Unix. zip.c (Ed) + 8. Also show escaped Unicode for new files in found list. zip.c (Ed) + 9. Update manual. man/zip.1, zip.txt (Ed) +------------------------ May 4th 2008 version 3.0h08 ----------------------- + 1. Handle when a bad Unicode string in archive forces + utf8_to_wide_string() to return a NULL string. Give warning if UTF-8 + in existing archive is bad. Put WIN32 wide local header initializations + in UNICODE_SUPPORT block. fileio.c, zipfile.c (Ed) + 2. Leave out Unicode escape code if not Unicode enabled. zip.c (Ed) + 3. Enable oem_to_local_string() and local_to_oem_string() for WIN32 + even if no Unicode. zip.h, win32/win32.c (Christian, Ed) + 4. Update comment about encryption code. zipcloak.c (Ed) + 4. Update zipmessage_nl() and zipmessage() from zip.c. zipcloak.c, + zipnote.c, zipsplit.c (Ed) + 5. Add Mac OS X library check. unix/configure (SMS) + 6. Add 16-bit UID/GID check. unix/configure (Christian, Ed) + 7. Format echo and comment statements a bit. unix/configure (Ed) + 8. Only compile in old 16-bit UID/GID code if new define UIDGID_NOT_16BIT + from unix configure script is not defined. unix/unix.c (Christian) + 9. A couple changes to updated 16-bit UID/GID code. Add 64-bit + UID/GID support to new Unix extra field. unix/unix.c (Ed) +10. Remove redundant "license" from options table. zipcloak.c (Ed) +11. Remove old unix build files. unix/configure-orig, unix/Makefile-orig + (Christian) +12. Add -O (--output-file) option to ZipCloak. Fix bug by setting + out_path. zipcloak.c (Ed) +------------------------ May 8th 2008 version 3.0h09 ----------------------- + 1. Update copyright. Add check for NO_UNICODE_SUPPORT. tailor.h (Ed) + 2. Fix bug where Unicode General Purpose Bit Flag 11 should force keeping + the old name field but it was being overwritten by the escaped name + in the central directory header. Fixed some ZIPERR() calls in + putcentral() that referred to putlocal(). zipfile.c (Ed) + 3. Add comment about OCRCU8 and OCRCTB. unix/configure (Ed) + 4. Change line in instructions to note that manuals should be made after + Zip is made. Change OCRTB to OCRCTB. Add $(OCRCTB) to rule for + zipcloak$E so crc32_.o is linked in. Add comment for NO_UNICODE_SUPPORT + flag. unix/makefile (Ed) + 5. Update WhatsNew. Add additional items to the Zip 3.1 list. Add note + about Zip 2.4. WHATSNEW (Ed) + 6. Update Zip 3.0h announcement. zip30h.ann (Ed) + 7. Update manual pages. man/zip.1, man/zipsplit.1, man/zipnote.1, + man/zipcloak.1 (Ed) + 8. Add noted for UTF-8 locale. zip.c (Ed) + 9. Set UTF-8 locale for Unix in utilities if UNICODE_SUPPORT enabled + so can display and process paths in archives correctly. zipsplit.c, + zipcloak.c, zipnote.c (Ed) +------------------------ May 12th 2008 version 3.0h10 ---------------------- + 1. Add use of new Unix UID/GID extra field and of old Unix 16-bit UID/GID + extra field when system uses 16-bit UIDs/GIDs to version information. + zip.c (SMS, Ed) + 2. Add Unicode Path and Unicode Comment extra fields to extra fields list. + Update new Unix extra field revision date. proginfo/extrafld.txt (Ed) + 3. Add Mac hardware platform to version information. unix/unix.c (SMS) +------------------------ May 19th 2008 version 3.0h11 ---------------------- + 1. Initialize f->namew when streaming stdin to fix bug. fileio.c (Ed) + 2. Change force_zip64 to start as -1 as unset, then use 1 for forcing use + of Zip64 and 0 for disabling use of Zip64. Add negation of -fz to + prevent use of Zip64 during streaming from stdin to a non-seekable + output where data descriptors will be used, which allows creating + archives with the old stream format but will fail if a large file is + streamed. Default is still to force Zip64 data descriptors when + streaming, which covers all cases but requires a Zip64 compatible + unzip. zip.c, globals.c, zipfile.c (Ed) + 3. Handle case of bad Unicode in archive. zipfile.c (Ed) +------------------------ May 22nd 2008 version 3.0h12 ---------------------- + 1. Fix bug introduced last beta that prevented streaming large files. Use + separate error message depending on if -fz- was used. zipfile.c (Ed) + 2. Change non existent to nonexistent. unix/configure (SMS) + 3. Don't output blank line when zipmessage_nl() gets passed an empty + string. This removes blank lines for skipped entries when -FS used. + zip.c (Ed) +------------------------ May 27th 2008 version 3.0h13 ---------------------- + 1. Change UNICODE_ALLOW_FORCE to UNICODE_SUPPORT, -UN=force to -UN=UTF8, + and unicode_force to utf8_force. This option now standard with Unicode + support and forces Zip to save UTF-8 paths and comments, when not ASCII, + as if UTF-8 were the native character set. globals.c, zip.c, zip.h (Ed) + 2. Add note to Todo that it's out of date. TODO (Ed) + 3. Update WhatsNew. WHATSNEW (Ed) + 4. Update Unicode help in extended help. zip.c (Ed) + 5. Update announcements. zip30h.ann, zip30.ann (Ed) + 6. Fix bug with -UN=UTF8. zip.c, zipfile.c (Ed) + 7. Update Zip manual. man/zip.1, zip.txt (Ed) + 8. Attempt an update to zip limits document. proginfo/ziplimit.txt (Ed) + 9. Update README regarding forum postings. README (Ed) +10. Remove duplicate initialization lines for found and fnxt. zip.c (SMS) +------------------------ May 28th 2008 version 3.0h14 ---------------------- + 1. Remove >= 0 check from wide character check as value is unsigned. + fileio.c (SMS) + 2. In putlocal(), move nam and use_uname to UNICODE_SUPPORT block. If + no UNICODE_SUPPORT use z->nam instead of nam. zipfile.c (SMS, Ed) + 3. Update announcement date for beta. zip30h.ann (Ed) +------------------------ May 31st 2008 version 3.0h ------------------------ + 1. In putlocal() if using UTF-8 bit then also set UTF-8 bit in z->lflg so + is set in local header for streaming. zipfile.c (Ed) + 2. Update announcement date for beta. zip30h.ann (Ed) + 3. Rename lib and dll projects to zip32z64 and update project files so + project name is same as lib and dll libraries. Export make files. + windll/visualc/dll/zip32z64.dsp, windll/visualc/dll/zip32z64.dsw, + windll/visualc/dll/zip32z64.mak, windll/visualc/libzip32z64.dsp, + windll/visualc/libzip32z64.dsw, windll/visualc/libzip32z64.mak (Ed) +------------------------ June 7th 2008 version 3.0i01 ---------------------- + 1. Update Mac ReadMe to note Mac OS X uses Unix port. macos/readme.1st (Ed) + 2. Change UNIX to Unix in manual. Update dates in manual and add note + about Mac OS X. Change switch to switches. zip.1 (SMS, Ed) + 3. Add version information under Windows by adding a version resource. + win32/vc6/zip.dsp, win32/vc6bz2/zip.dsp, win32/zip.rc (Ed) +------------------------ June 15th 2008 version 3.0i02 ---------------------- + 1. Update Install instructions. INSTALL (Ed) + 2. Update ReadMe. README (Ed) + 3. Update ToDo list. TODO (Ed) + 4. Update WhatsNew. WHATSNEW (Ed) + 5. Add note to WHERE. WHERE (Ed) + 6. Update announcement. zip30.ann (Ed) + 7. Review man pages and update Zip man page. Compile text files from man + pages. man/zip.1, zip.txt, zipnote.txt, zipsplit.txt, zipcloak.txt (Ed) + 8. Update extended help. zip.c (Ed) +------------------------ June 17th 2008 version 3.0i03 ---------------------- + 1. Fix bug where UTF-8 flag was not being set when using_utf8 was set as + result of UTF-8 being current character set. zipfile.c (Ed) + 2. Update man page globbing description. man/zip.1, zip.txt (SMS, Ed) + 3. Update web address to bzip2 package for VMS. vms/install_vms.txt (SMS) +------------------------ June 21st 2008 version 3.0i04 ---------------------- + 1. Update comments. zbz2err.c (Christian) + 2. Put use_uname in UNICODE_SUPPORT block. zipfile.c (Christian) + 3. Increase st to 0x1400. msdos/makefile.msc (Christian) + 4. Update copyright and put @CodeSize and @DataSize into ifndef blocks for + Huge, Large, Compact, Medium, and Small. msdos/match.asm (Christian) + 5. Add check to disable symbolic links. msdos/osdep.h (Christian) + 6. Put Mac OS X compiler check into if Mac OS X block to avoid problems on + some other Unix ports with the check. unix/configure (SMS) + 7. Move set_extra_field() to fix compile problem. unix/unix.c (SMS) + 8. Update USEBZIP2 to USEBZ2 and -DUSE_BZIP2 to -DBZIP2_SUPPORT. Drop + -DMSDOS compile flag. win32/makefile.w32 (Christian) + 9. Change BZIP2_SUPPORT to USEBZ2. win32/makenoas.w32 (Christian) +------------------------ June 23rd 2008 version 3.0i05 ---------------------- + 1. Update and unify resources. Remove any MFC dependencies from the resource + files zip.rc and windll.rc. win32/zip.rc and windll/windll.rc now read + the version info from revision.h. windll.rc internal flags modified to + "32-bit dll". zip.rc internal flags liberated from "winnt 32-bit" + to "generic 32-bit windows". Win32 zip.exe also supported on Win9x + (32-bit). Update makefiles for Borland, MSC, GCC(mingw32), Watcom + to support inclusion of zip.rc version resources into zip.exe binary. + revision.h, msdos/osdep.h, win32/makefile.bor, win32/makefile.gcc, + win32/makefile.w10, win32/makefile.w32, win32/makefile.wat, + win32/makenoas.w32, win32/zip.rc, windll/windll.rc (Christian) + 2. Remove unused files. win32/resource.h, windll/resource.h, + windll/windll.aps, windll/zipver.h, windll/visualc/dll/zip32z64.mak, + windll/visualc/lib/zip32z64.mak (Christian) + 3. Update VMS. vms/descrip_deps.mms (SMS) +------------------------ June 26th 2008 version 3.0i06 ---------------------- + 1. Update Install and Readme in preparation for release. Update WhatsNew. + INSTALL, README, WHATSNEW (Ed) + 2. Update announcement. zip30.ann (Ed) + 3. Update original Visual Basic project comments and documentation. + windll/vb/readmevb.txt, windll/vb/vbzip.vbp, windll/vb/vbzip.vbw, + windll/vb/vbzipbas.bas, windll/vb/vbzipfrm.frm (Ed) + 4. Add bzip2 version of djgpp 2.x makefile thanks to Robert. Assumes a + standard djgpp installation. msdos/makebz2.dj2 (Robert Riebisch, Ed) +------------------------ June 27th 2008 version 3.0i07 ---------------------- + 1. Add DJGPP to bzip2 install instructions. bzip2/install.txt, + msdos/makebz2.dj2 (Robert, Ed) +------------------------- July 5th 2008 version 3.0 ------------------------- + 1. Add -sd to extended help. zip.c (Will, Ed) + 2. Fix memory bug when rebuilding Zip64 central directory extra field which + can crash MinGW and other ports when processing large files. zipfile.c + (Will) + 3. Fix -v bug preventing display of version information when options in + environment variables. zip.c (Ed) + 4. Update WhatsNew. WHATSNEW (Ed) + 5. Update announcement. zip30.ann (Ed) diff --git a/third_party/infozip/zip/INSTALL b/third_party/infozip/zip/INSTALL new file mode 100644 index 000000000..be3e0c5b0 --- /dev/null +++ b/third_party/infozip/zip/INSTALL @@ -0,0 +1,368 @@ +HOW TO INSTALL ZIP + + Zip is distributed as C source code that can be compiled on a + wide range of systems: Unix, VMS, MSDOS, OS/2, NT, Amiga, Atari, + BeOS, VM/CMS, ... You will need Unzip 5.0p1 or later (under any + system) or PKUNZIP 2.04g or later (under MSDOS) to unpack the + distribution file, in this case zip30.zip. But since you read this, + you have unpacked it already, or you cheated and got a tar.Z file... + + Note: Zip 3.0 distribution kits (unlike previously distributed + Zip 2.x kits) are created with a top-level directory ("zip30") in + the archive, making the creating of the zipsrc directory optional. + +Installation on Unix (see below for installation on other systems) + + Let's assume that you start from scratch and have not yet unpacked + the sources. First step, then, is to unpack Zip. The following + assumes that you have zip30.zip in the current directory. + + For example, to extract to a new zipsrc directory (assuming + zip30.zip is in the current directory): + + mkdir zipsrc + cd zipsrc + cp ../zip30.zip . + unzip zip30.zip + cd zip30 + + To extract in an existing directory, such as /usr/local/src/zip: + + cd /usr/local/src/zip + (copy zip30.zip here) + unzip zip30.zip + cd zip30 + + The first extracts all source files and documentation to the + directory "zipsrc/zip30". The second places the zip30 directory + in the "/usr/local/src/zip" directory. Both then cd in to the + zip30 directory where Zip will be built. + + Note: This release now includes the standard encryption code + previously in the separate package zcrypt29.zip, but you still + can decide whether to activate the crypt code or not. Crypt is + enabled by default, but you may disable it by specifying the + option -DNO_CRYPT in the LOCAL_ZIP environment variable (or by + adding this option to the compilation options in the appropiate + makefile). See README.CR for more on crypt. + + You then do: + + make -f unix/Makefile system + + where "system" is one of: generic, generic_gcc, + att6300, coherent, cray_v3, minix, sco_x286, xenix, zilog. + + For Unix systems where "cc" is the preferred C compiler command, + try + + make -f unix/Makefile generic + + first. If "gcc" is preferred, specify "generic_gcc" instead of + "generic". This should work on most systems and automatically + selects compilation options based on a set of tests (in + unix/configure), including detection of large file support + sufficient to enable Zip64 large archive features. If "generic" + (or "generic_gcc" if that is used) fail, then one of the special + targets given above may work. + + Among other special systems are Cray Unicos, Zilog Zeus and MINIX. + + The optimization settings for many systems should be close, but + if you see optimization for your system is not ideal, send in + the changes so we can improve it. + + By default, Zip uses the "deflate" compression method. To add + the additional optional "bzip2" compression method, see the file + bzip2/install.txt. Note that bzip2 support is provided by + compiling or linking in the bzip2 library. See the bzip2 site + (http://www.bzip.org/) for more on bzip2. + + If you get error messages such as "constant expected" in + deflate.c, add -DDYN_ALLOC to CFLAGS in your makefile entry. + + If you have lots of memory, try compiling with -DBIG_MEM. If your + system supports mmap(), try compiling with -DMMAP. This generally + gives faster compression but uses more memory. See the unix/Makefile + entry mmap_gcc for an example. + + If none of these compiles, links, and functions properly on + your Unix system, then your system apparently has specific + requirements we did not account for. See the file README for how + to get help. + + If the appropriate system was selected, then the executables zip, + zipnote, zipcloak, and zipsplit will be created. You can copy + them to an appropriate directory in the search path using: + + make -f unix/Makefile install + + The defaults are /usr/local/bin for the executables and + /usr/local/man/man1 for the manual pages. Change the macros + BINDIR and MANDIR in makefile to change these if needed. + + If necessary, add the directory with the Zip executables to your + shell's PATH (or "path") variable. (C-shell users may need to + use the "rehash" command so csh can find the new command in the + path.) You should now be ready to use Zip. + + You can get rid of the now unnecessary source and object files + with: + + cd .. + rm -r zip30 + + This will remove the directory zip30 and its contents created + by unzip. You should keep the zip30.zip file around though, + in case you need to build it again or want to give it to a + colleague. + + You can add the following lines to the file /etc/magic for + usage by the 'file' command: + +0 string PK Zip archive +>4 byte 011 (at least v0.9 to extract) +>4 byte 012 (at least v1.0 to extract) +>4 byte 013 (at least v1.1 to extract) +>4 byte 024 (at least v2.0 to extract) +>4 byte 025 (at least v2.1 to extract) + + +Installation on other systems + + The steps for installation under VMS, MSDOS, OS/2, NT, Amiga and + Atari are similar to the above: first unzip the distribution + files into their own directory. The system-dependent files are + stored in special subdirectories. + + For all the non-Unix ports which support the creation of "UT" extra + fields (these ports contain USE_EF_UT_TIME in the list of optional + features displayed with "zip -v"), the timezone environment variable TZ + should be set according to the local timezone in order for the -f, -u, + -o, and similar options to work correctly. This is not needed for the + WIN32 and WinDLL ports, since they get the timezone information from + the OS by other means. + + + MSDOS: + + Do one of: + + make msdos\makefile.msc (Microsoft C 5.1) + nmake -f msdos\makefile.msc (Microsoft C 6.0 and newer) + make -fmsdos\makefile.bor -DCC_REV=1 (Borland Turbo C++ 1.0) + make -fmsdos\makefile.bor (Borland C++ 2.0 and newer) + make -fmsdos\makefile.tc (Borland Turbo C 2.0x) + make -f msdos/makefile.dj1 (DJGPP v1.12m4) + make -f msdos/makefile.dj2 (DJGPP v2.01 and newer) + make -f msdos/makefile.emx (gcc/emx 0.9b and newer) + make -f os2/makefile.os2 gccdos (gcc/emx 0.9b and newer) + wmake -f msdos\makefile.wat (Watcom C 11.x 16-bit) + wmake -f msdos\makefile.wat PM=1 (Watcom C 11.x 32-bit, PMODE/W) + + for Microsoft, Borland C++ and Turbo C, Watcom C/C++ and the various + free GNU C implementations, respectively. More detailed instructions + can be found in the respective makefiles. + + + WIN32 (Windows NT/2K/XP/2K3 and Windows 95/98/ME): + + Supported compilers are Microsoft Visual C++, Borland C++, Watcom C/C++, + and miscellaneous free GNU C implementations (gcc/mingw, CygWin, ...). + The makefiles supplied in the win32/ subdirectory contain further + information. + + + Windows DLL (WIN32): + + Supported environments are Visual C++ (32-bit only, 5.x and newer). + For instructions how to build the DLLs and where find the makefiles, + look into windll/contents. + + + OS/2: + + Type + + {make} -f os2/makefile.os2 + + to get a list of supported targets/compiling environments. + (replace "{make}" with the name of your OS/2 make utility.) + + To initiate the actual compiling process, you have to specify + a system target: + + {make} -f os2/makefile.os2 {system} + + An example: type + + nmake -f os2/makefile.os2 msc + + for Microsoft C 6.00. + + + VMS (OpenVMS): + + The most complete information on building and installing Zip on VMS + is in [.vms]install_vms.txt. Optimists in a hurry may wish to try + commands like these: + + @ [.VMS]BUILD_ZIP.COM + or: + MMS /DESCRIP = [.VMS]DESCRIP.MMS CLEAN ! Or MMK ... + MMS /DESCRIP = [.VMS]DESCRIP.MMS ! Or MMK ... + + When the executables have been created (or located if already installed), + most users define foreign command symbols for the Zip executables, like + this: + + ZIP :== $ dev:[dir]ZIP.EXE ! UNIX-like command line. + or: + ZIP :== $ dev:[dir]ZIP_CLI.EXE ! VMS-like command line. + + Such symbol definitions are often added to a user's + SYS$LOGIN:LOGIN.COM procedure, or to a common, site-specific + procedure, like SYS$MANAGER:SYLOGIN.COM. + + Additional installation options are described in install_vms.txt. + + The builders create help text files, ZIP.HLP and ZIP_CLI.HLP. Also + see install_vms.txt for how to create the help libraries. + + +Mac OS: + + Mac OS X is part of the Unix port, so use the Unix installation above. + + Mac OS before Mac OS X use the Mac OS port, though little testing has + been done for that port recently. See macos/README.TXT for more on + this port. + + +Compiler Flags + + Zip should compile fine out of the box for your port. In particular, + for Unix the command + make -f unix/Makefile generic + should automatically detect the features available on your system and + set the flags appropriately. In some cases, however, you may need to + set one or more compiler flags yourself to get Zip to compile or to + add features you want or remove features that cause trouble for your + port. Below are the more common compiler macros you can set. + + LARGE_FILE_SUPPORT + Tell Zip that the OS supports large files (generally files larger + than 4 GB). Zip will try to compile in the large file calls + (typically 64-bit) for the OS instead of using the standard + (typically 32-bit) file calls. On Unix Zip tries to switch over to + the 64-bit file environment. If setting this flag causes errors + or Zip still can't handle large files on that port, then probably + either Zip doesn't have the code to support large files on your OS + (write a patch and send it in to us) or your OS doesn't support large + files. + + Note that the flag ZIP64_SUPPORT must also be set to create archives + with large files. + + This flag should be set automatically on Unix, Win32, and some + other ports. Setting NO_LARGE_FILE_SUPPORT turns this flag off. + + ZIP64_SUPPORT + Enable the Zip64 code in Zip that supports the Zip64 extensions noted + in the PKWare AppNote. These extensions allow storing files larger + than 4 GB in archives and the creating of archives larger than 4 GB. + They also allow storing more than 64K files in an archive. Currently + Zip does not handle archives of PKZip version 4.5 or later unless + this flag is set. + + To enable large file support in Zip, you generally need to set both + LARGE_FILE_SUPPORT (to read and write large files) and ZIP64_SUPPORT + (to store them in and read them from archives). Files larger than + 4 GB may be invisible to Zip (directory scans don't see them) if + LARGE_FILE_SUPPORT is not enabled. + + Keeping LARGE_FILE_SUPPORT and ZIP64_SUPPORT separate allows easier + debugging of these features. When testing large file support on an + OS, first set just LARGE_FILE_SUPPORT to test the file calls (all + should compile and work as before with small files), then turn on + ZIP64_SUPPORT to let Zip recognize and handle large files. + + This flag should be set automatically on most ports if + LARGE_FILE_SUPPORT is set. Setting NO_ZIP64_SUPPORT turns this flag + off. + + UNICODE_SUPPORT + Enable storing and using UTF-8 paths. These paths are stored in + a backward-compatible way so that archives with UTF-8 paths still + work on zips and unzips that don't support Unicode. This support + follows the recent additions to the PKWare AppNote for Unicode + support, except that Unicode comments on systems where UTF-8 is + not the current character set is not implemented in this release. + + On some ports UNICODE_SUPPORT is set automatically if wide characters + are supported. Setting NO_UNICODE_SUPPORT turns off this flag. + + USE_EF_UT_TIME + Enables storing UT time in an extra field. This becomes useful + for ports that normally store file times as local time, resulting + in problems when files are moved across time zones and when + there are daylight savings time changes. Zip and UnZip will + automatically correct for time zone changes when UT time is stored. + + This is usually set by default. Use NO_EF_UT_TIME to turn this off. + + NTSD_EAS (Win32 only) + Enable storing Windows NT file security descriptors. This allows + restoring the descriptors (file ACL's, etc.). + + This is on by default for Win32. Use NO_NTSD_EAS to turn this off. + + BZIP2_SUPPORT + Enable compressing zip entries using the bzip2 library. You must get + the bzip2 library from somewhere else as we only provide a way to + compile or link the library in and compress files using bzip2. Enables + a new compression method, bzip2, that can be used instead of the default + Zip compression method deflate. + + This flag is set on Unix, including Mac OS X, when compiling using + generic if the bzip2 library is found. Set on Win32 if the bzip2 + projects are used. See the VMS documentation for when VMS sets this + flag. Setting NO_BZIP2_SUPPORT turns this off. + + See bzip2/install.txt for more on installing bzip2 support. + + WIN32_OEM (Win32 only) + Enable saving paths on Win32 in the OEM character set. Zip has stored + paths using the standard ANSI local character set, but other zips have + used the OEM character set on MSDOS and Win32. This flag should make + Zip more compatible with other DOS and Win32 zips and unzips. It also + enables the translation of OEM paths in DOS archives to ANSI and should + eliminate some problems with funny characters showing up in path names. + + If Unicode is enabled and used, Unicode paths generally override + local paths using OEM character sets. + + This flag is on by default on most Win32 ports. Some ports apparently + have problems with OEM conversions. If your port or compiler does + funny things with file names, you may want to turn this off. Defining + NO_WIN32_OEM turns this flag off. + + NO_STREAMING_STORE + Because storing zip archives inside a zip entry adds "false" signatures + and this causes problems when using data descriptors if the archive + needs fixing, this option is provided to force deflating when streaming. + This version of Zip includes an advanced algorithm for correctly finding + these signatures, but if an archive is "broke", there is no telling + what's where. This is only a problem if an archive becomes broke for + some reason, but to be safe define this. + + ALLOW_REGEX + For MSDOS and Windows, now "[list]" wildcard matching (where any + character between [ and ] can be used to match the character at that + position) is turned off unless the new -RE option is used. Defining + this flag forces "[list]" matching to be always on as in previous + releases. + + +For command help on any of the zip* utilities, simply enter +the name with no arguments. diff --git a/third_party/infozip/zip/LICENSE b/third_party/infozip/zip/LICENSE new file mode 100644 index 000000000..bcfe47e97 --- /dev/null +++ b/third_party/infozip/zip/LICENSE @@ -0,0 +1,60 @@ +This is version 2007-Mar-4 of the Info-ZIP license. +The definitive version of this document should be available at +ftp://ftp.info-zip.org/pub/infozip/license.html indefinitely and +a copy at http://www.info-zip.org/pub/infozip/license.html. + + +Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + +For the purposes of this copyright and license, "Info-ZIP" is defined as +the following set of individuals: + + Mark Adler, John Bush, Karl Davis, Harald Denker, Jean-Michel Dubois, + Jean-loup Gailly, Hunter Goatley, Ed Gordon, Ian Gorman, Chris Herborth, + Dirk Haase, Greg Hartwig, Robert Heath, Jonathan Hudson, Paul Kienitz, + David Kirschbaum, Johnny Lee, Onno van der Linden, Igor Mandrichenko, + Steve P. Miller, Sergio Monesi, Keith Owens, George Petrov, Greg Roelofs, + Kai Uwe Rommel, Steve Salisbury, Dave Smith, Steven M. Schweda, + Christian Spieler, Cosmin Truta, Antoine Verheijen, Paul von Behren, + Rich Wales, Mike White. + +This software is provided "as is," without warranty of any kind, express +or implied. In no event shall Info-ZIP or its contributors be held liable +for any direct, indirect, incidental, special or consequential damages +arising out of the use of or inability to use this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the above disclaimer and the following restrictions: + + 1. Redistributions of source code (in whole or in part) must retain + the above copyright notice, definition, disclaimer, and this list + of conditions. + + 2. Redistributions in binary form (compiled executables and libraries) + must reproduce the above copyright notice, definition, disclaimer, + and this list of conditions in documentation and/or other materials + provided with the distribution. The sole exception to this condition + is redistribution of a standard UnZipSFX binary (including SFXWiz) as + part of a self-extracting archive; that is permitted without inclusion + of this license, as long as the normal SFX banner has not been removed + from the binary or disabled. + + 3. Altered versions--including, but not limited to, ports to new operating + systems, existing ports with new graphical interfaces, versions with + modified or added functionality, and dynamic, shared, or static library + versions not from Info-ZIP--must be plainly marked as such and must not + be misrepresented as being the original source or, if binaries, + compiled from the original source. Such altered versions also must not + be misrepresented as being Info-ZIP releases--including, but not + limited to, labeling of the altered versions with the names "Info-ZIP" + (or any variation thereof, including, but not limited to, different + capitalizations), "Pocket UnZip," "WiZ" or "MacZip" without the + explicit permission of Info-ZIP. Such altered versions are further + prohibited from misrepresentative use of the Zip-Bugs or Info-ZIP + e-mail addresses or the Info-ZIP URL(s), such as to imply Info-ZIP + will provide support for the altered versions. + + 4. Info-ZIP retains the right to use the names "Info-ZIP," "Zip," "UnZip," + "UnZipSFX," "WiZ," "Pocket UnZip," "Pocket Zip," and "MacZip" for its + own source and binary releases. diff --git a/third_party/infozip/zip/README b/third_party/infozip/zip/README new file mode 100644 index 000000000..a55942576 --- /dev/null +++ b/third_party/infozip/zip/README @@ -0,0 +1,234 @@ +Zip 3.0 is the first Zip update adding large file support. For now Zip 2.3x +remains available and supported, but users should switch to this new release. + +Testing for Zip 3.0 has focused mainly on Unix, VMS, Max OS X, and Win32, +and some other ports may not be fully supported yet. If you find your +favorite port is broke, send us the details or, better, send bug fixes. It's +possible that support for some older ports may be dropped in the future. + + + +Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + +See the accompanying file LICENSE (the contents of which are also included +in unzip.h, zip.h and wiz.h) for terms of use. If, for some reason, all +of these files are missing, the Info-ZIP license also may be found at: +ftp://ftp.info-zip.org/pub/infozip/license.html and +http://www.info-zip.org/pub/infozip/license.html. + + +Zip 3.0 is a compression and file packaging utility. It is compatible with +PKZIP 2.04g (Phil Katz ZIP) for MSDOS systems. There is a companion to zip +called unzip (of course) which you should be able to find in the same place +you got zip. See the file 'WHERE' for details on ftp sites and mail +servers. + +So far zip has been ported to a wide array of Unix and other mainframes, +minis, and micros including VMS, OS/2, Minix, MSDOS, Windows, Atari, Amiga, +BeOS and VM/CMS. Although highly compatible with PKware's PKZIP and PKUNZIP +utilities of MSDOS fame, our primary objective has been one of portability +and other-than-MSDOS functionality. Features not found in the PKWare version +include creation of zip files in a pipe or on a device; VMS, BeOS and OS/2 +extended file attributes; conversion from Unix to MSDOS text file format; and, +of course, the ability to run on most of your favorite operating systems. And +it's free. + +See the file zip30.ann for a summary of new features in Zip 3.0 and WhatsNew +for the detailed list of new features and changes since Zip 2.32. The file +CHANGES details all day-to-day changes during development. + +Notes: + +Multi-volume support. This version does not support multi-volume spanned +archives as in pkzip 2.04g, and there is no intention at this point to support +spanned archives, but Zip 3.0 supports split archives. A split archive is an +archive split into a set of files, each file a piece of the archive and each +file using an extension, such as .z02 as in the file name archive.z02, that +provides the order of the splits. In contrast, a spanned archive is the +original multi-floppy archive supported by pkzip 2.0g where the split order +is contained in the volume labels. The contents of split and spanned archives +are mostly identical and there is a simple procedure to convert between the +formats. Many current unzips now support split archives. + +Zip64 support. This version supports Zip64 archives as described in the +PKWare AppNote. These archives use additional fields to support archives +greater than 2 GB and files in archives over the 2 GB previous limit (4 GB +on some ports). The Zip64 format also allows more than 64k entries in an +archive. Support by the OS for files larger than 4 GB is needed for Zip to +create and read large files and archives. On Unix, Win32, and some other +ports, large file and Zip64 support is automatically checked for and +compiled in if available. Use of Zip64 by Zip is automatic and to maximize +backward compatibility the Zip64 fields will only be used if needed. A +Zip64 archive requires a pkzip 4.5 compatible unzip, such as UnZip 6.0. + +Unicode support. This version has initial Unicode support. This allows +paths and names of files in other character sets to be accurately recreated +on OS that have sufficient character set support. On Win32, if wide +character calls are supported (not Win 9x unless Unicode support has been +added) all files (including paths with illegal characters in the current +character set) should now be readable by zip. Unicode support is provided +using a new set of UTF-8 path and comment extra fields and a new UTF-8 bit +for flagging when the current character set is already UTF-8. Zip 3.0 +maintains backward compatibility with older archives and is mostly compliant +with the new Unicode additions in the latest PKWare AppNote. The exception +is UTF-8 comments, which are not supported if UTF-8 is not the native +character set, but should be fully implemented in Zip 3.1. + +16-bit OS support. Though Zip 3.0 is designed to support the latest zip +standards and modern OS, some effort has been made to maintain support +for older and smaller systems. If you find Zip 3.0 does not fit on or +otherwise does not work well on a particular OS, send in the details and +we might be able to help. + +Compression methods. In addition to the standard store and deflate methods, +Zip now can use the bzip2 compression format using the bzip2 library. Though +bzip2 compression generally takes longer, in many cases using bzip2 results +in much better compression. However, many unzips may not yet support +bzip2 compressed entries in archives, so test your unzip first before using +bzip2 compression. + +Installation. Please read the file INSTALL for information on how to compile +and install zip, zipsplit, zipcloak, and zipnote and please read the manual +pages ZIP.txt, ZIPSPLIT.txt, ZIPCLOAK.txt, and ZIPNOTE.txt for information on +how to use them. Also, if you are using MSDOS or Windows, note that text +files in the distribution are generally in Unix line end format (LF only) +and Windows and DOS users will need to either convert the files as needed to +DOS line ends (CR LF) or extract the distribution contents using unzip -a. + +Utilities. At this point zipsplit, zipcloak, and zipnote should work with +large files, but they currently do not handle split archives. A work around +is to use zip to convert a split archive to a single file archive and then use +the utilities on that archive. + +Encryption. This version supports standard zip encryption. Until recently +the encryption code was distributed separately because of the US export +regulations but now is part of the main distribution. See crypt.c for +details. Decryption can be made with unzip 5.0p1 or later, or with zipcloak. + +Bug reports. All bug reports or patches should go to zip-bugs via the web +site contact form at http://www.info-zip.org/zip-bug.html (we have discontinued +the old email address zip-bugs@lists.wku.edu because of too much spam lately) +and suggestions for new features can be submitted there also (although we don't +promise to use all of them). We also are on SourceForge at +http://sourceforge.net/projects/infozip/ and now automatically get Bug Reports +and Feature Requests submitted there. In addition, a new Info-ZIP discussion +forum is available as well. See below. Though bug reports can be posted there, +we don't have automatic monitoring of all postings set up yet so you may want +to use the web form or SoureForge for a quicker response. A good approach may +be to post the details on the forum so others can benefit from the posting, +then use the web reply form to let us know you did that if you don't get a +reply in a reasonable time. + +Ports. If you're considering a port, please check in with zip-bugs FIRST, +since the code is constantly being updated behind the scenes. We'll +arrange to give you access to the latest source. + +Discussion group. If you'd like to keep up to date with our Zip (and companion +UnZip utility) development, join the ranks of BETA testers, add your own +thoughts and contributions, etc., check out the new discussion forum. This is +the latest offering, after the various Info-ZIP mailing-lists on +mxserver@lists.wku.edu (courtesy of Hunter Goatley) were no longer available +and the temporary QuickTopic discussion group for Info-ZIP issues at +http://www.quicktopic.com/27/H/V6ZQZ54uKNL died a horrible death due to large +amounts of spam. The new discussion forum is now available at +http://www.info-zip.org/board/board.pl (thanks again to Hunter Goatley) and +can be used to discuss issues, request features, and is one place new betas +and releases are announced. It also is a place to post bug reports, and +patches can be submitted as attachments. However, we don't yet get +automatic notification of all postings there so try one of the other methods +if you don't get a response. You can also post Bug Reports and Feature +Requests at Source Forge. However, the web site contact form remains +available if you would rather not post on the public forums. + +Frequently asked questions on zip and unzip: + +Q. When unzipping I get an error message about "compression method 8". + +A. This is standard deflate, which has been around for awhile. Please + get a current version of unzip. See the file 'WHERE' for details. + + +Q. How about "compression method 12"? + +A. Compression method 12 is bzip2 and requires a relatively modern unzip. + Please get the latest version of unzip. + + +Q. I can't extract this zip file that I just downloaded. I get + "zipfile is part of multi-disk archive" or some other message. + +A. Please make sure that you made the transfer in binary mode. Check + in particular that your copy has exactly the same size as the original. + Note that the above message also may actually mean you have only part + of a multi-part archive. Also note that UnZip 5.x does not and UnZip 6.0 + probably won't have multi-disk (split) archive support. A work around + is to use Zip 3.0 to convert the split archive to a single-file archive + then use UnZip on that archive. As a last result, if there's something + readable in what you have, zip -FF should be able to recover it. + + +Q. When running unzip, I get a message about "End-of-central-directory + signature not found". + +A. This usually means that your zip archive is damaged, or that you + have an uncompressed file with the same name in the same directory. + In the first case, it makes more sense to contact the person you + obtained the zip file from rather than the Info-ZIP software + developers, and to make sure that your copy is strictly identical to + the original. In the second case, use "unzip zipfile.zip" instead + of "unzip zipfile", to let unzip know which file is the zip archive + you want to extract. + + +Q. Why doesn't zip do just like PKZIP does? + +A. Zip is not a PKZIP clone and is not intended to be one. In some + cases we feel PKZIP does not do the right thing (e.g., not + including pathnames by default); in some cases the operating system + itself is responsible (e.g., under Unix it is the shell which + expands wildcards, not zip). Info-ZIP's and PKWARE's zipfiles + are interchangeable, not the programs. + + For example, if you are used to the following PKZIP command: + pkzip -rP foo *.c + you must use instead on Unix: + zip -R foo "*.c" + (the quotes are needed to let the shell know that it should + not expand the *.c argument but instead pass it on to the program, + but are not needed on ports that do not expand file paths like + MSDOS) + + +Q. Can I distribute zip and unzip sources and/or executables? + +A. You may redistribute the latest official distributions without any + modification, without even asking us for permission. You can charge + for the cost of the media (CDROM, diskettes, etc...) and a small copying + fee. If you want to distribute modified versions please contact us at + www.Info-ZIP.org first. You must not distribute beta versions. + The latest official distributions are always on ftp.Info-ZIP.org in + directory /pub/infozip and subdirectories and at SourceForge. + + +Q. Can I use the executables of zip and unzip to distribute my software? + +A. Yes, so long as it is made clear in the product documentation that + zip or unzip are not being sold, that the source code is freely + available, and that there are no extra or hidden charges resulting + from its use by or inclusion with the commercial product. See the + Info-ZIP license for more. Here is an example of a suitable notice: + + NOTE: is packaged on this CD using Info-ZIP's compression + utility. The installation program uses UnZip to read zip files from + the CD. Info-ZIP's software (Zip, UnZip and related utilities) is + freely distributed under the Info-ZIP license and can be obtained as + source code or executables from various anonymous-ftp sites, + including ftp://ftp.info-zip.org/pub/infozip. + + +Q. Can I use the source code of zip and unzip in my commercial application? + +A. Yes, as long as the conditions in the Info-ZIP license are met. We + recommend you include in your product documentation an acknowledgment + and note that the original compression sources are available at + www.Info-ZIP.org. If you have special requirements contact us. diff --git a/third_party/infozip/zip/README.CR b/third_party/infozip/zip/README.CR new file mode 100644 index 000000000..c777d19f2 --- /dev/null +++ b/third_party/infozip/zip/README.CR @@ -0,0 +1,119 @@ +_____________________________________________________________________________ + + This is Info-ZIP's README.CR for zcrypt29.zip, last updated 27 March 2008. +_____________________________________________________________________________ + + +The files described below contain the encryption/decryption code for Zip 2.31, +UnZip 5.52, and WiZ 5.02 (and later). These files are included in the main +source distributions for all of these now, but the encryption patch is still +available for earlier versions of these. This file both describes the history +of the encryption package and notes the current conditions for use. Check +the comments at the top of crypt.c and crypt.h for additional information. + +As of version 2.9, this encryption source code is copyrighted by Info-ZIP; +see the enclosed LICENSE file for details. Older versions remain in the pub- +lic domain. Zcrypt was originally written in Europe and, as of April 2000, +can be freely distributed from the US as well as other countries. + +(The ability to export from the US is new and is due to a change in the Bureau +of Export Administration's regulations, as published in Volume 65, Number +10, of the Federal Register [14 January 2000]. Info-ZIP filed the required +notification via e-mail on 9 April 2000; see the USexport.msg file in this +archive. However, as of June 2002, it can now be freely distributed in both +source and object forms from any country, including the USA under License +Exception TSU of the U.S. Export Administration Regulations (section 740.13(e)) +of 6 June 2002.) + + LIKE ANYTHING ELSE THAT IS FREE, ZIP, UNZIP AND THEIR ASSOCIATED + UTILITIES ARE PROVIDED AS IS AND COME WITH NO WARRANTY OF ANY KIND, + EITHER EXPRESSED OR IMPLIED. IN NO EVENT WILL THE AUTHORS BE LIABLE + FOR ANY DAMAGES RESULTING FROM THE USE OF THIS SOFTWARE. + +The encryption code is a direct transcription of the algorithm from +Roger Schlafly, described by Phil Katz in the file appnote.txt. This +file is distributed with the PKZIP program (even in the version without +encryption capabilities). Note that the encryption will probably resist +attacks by amateurs if the password is well chosen and long enough (at +least 8 characters) but it will probably not resist attacks by experts. +Paul Kocher has made available information concerning a known-plaintext +attack for the PKWARE encryption scheme; see http://www.cryptography.com/ +for details.) Short passwords consisting of lowercase letters only can be +recovered in a few hours on any workstation. But for casual cryptography +designed to keep your mother from reading your mail, it's OK. + +For more serious encryption, check into PGP (Pretty Good Privacy), a +public-key-based encryption system available from various Internet sites. +PGP has Zip and UnZip built into it. The most recent version at the time +this was originally written was 6.5, although older versions were still +widespread. At the time of this writing there are now GPG, PGP Universal +2.0, and various others based on OpenPGP. + +We are looking at adding AES strong encryption to future versions of Zip and +UnZip. + +Zip 2.3x and UnZip 5.5x and later are compatible with PKZIP 2.04g. (Thanks +to Phil Katz for accepting our suggested minor changes to the zipfile format.) + +IMPORTANT NOTE: + + Zip archives produced by Zip 2.0 or later must not be *updated* by + Zip 1.1 or PKZIP 1.10 or PKZIP 1.93a, if they contain encrypted members + or if they have been produced in a pipe or on a non-seekable device. + The old versions of Zip or PKZIP would destroy the zip structure. The + old versions can list the contents of the zipfile but cannot extract + it anyway (because of the new compression algorithm). If you do not + use encryption and compress regular disk files, you need not worry about + this problem. + + +Contents that were distributed and now are part of the main source files: + + file what it is + ---- ---------- + README.CR this file + LICENSE Info-ZIP license (terms of reuse and redistribution) + USexport.msg export notice sent to US Bureau of Export Administration + WHERE where Zip/UnZip/WiZ and related utilities can be found + crypt.c code for encryption and decryption + crypt.h code for encryption and decryption + file_id.diz description file for some BBSes + +Most all of the files are in Unix (LF only) format. On MSDOS systems, you +can use the -a option of UnZip to convert the source files to CRLF +format. This is only necessary if you wish to edit the files -- they +will compile as is with Microsoft C and Turbo/Borland C++ 1.0 or +later. However, you will have to convert the files (using "unzip -a") +to the CRLF format to compile with the older Turbo C 1.0 or 2.0. You +should be able to find Zip and UnZip in the same place you found this +(see ftp://ftp.info-zip.org/pub/infozip/Info-ZIP.html or the file +"WHERE" for details). + +Current releases all have encryption built in. To update previous versions +using the zcrypt sources: + + (1) Get the main sources (e.g., Zip 2.3) and unpack into a working + directory, as usual. + + (2) Overwrite the dummy crypt.c and crypt.h from the main sources with + the versions from this package. If you want to overwrite directly + out of the zcrypt29 archive, do not use UnZip's freshen/updating + option; the dummy files may be newer than the real sources in + zcrypt29. ("unzip -o zcrypt29 -d /your/working/dir" will do the + Right Thing in most cases, although it may overwrite a newer WHERE + file under some circumstances.) + + (3) Read the main INSTALL document and compile normally! No makefile + changes are necessary on account of the zcrypt sources. You can + check that the version you just compiled has encryption or decryption + support enabled by typing "zip -v" or "unzip -v" and verifying that + the last "special compilation option" says encryption or decryption + is included. + +Encryption enables new "-e" and "-P password" options in Zip, and a new +"-P password" option in UnZip--see the normal Zip and UnZip documentation +for details. (Note that passing a plaintext password on the command line +is potentially much more insecure than being prompted for it interactively, +which is the default for UnZip and for Zip with "-e". Also note that the +interactive method allows UnZip to deal with archives that use different +passwords for different files.) diff --git a/third_party/infozip/zip/TODO b/third_party/infozip/zip/TODO new file mode 100644 index 000000000..8d517328a --- /dev/null +++ b/third_party/infozip/zip/TODO @@ -0,0 +1,142 @@ +Todo list (last updated 12 June 2008). + +Features for next official version: + +- Extended attributes for Windows, Linux, and Mac OS X. +- Win32 ACL rewrite to use backup api to create new and more useful extra + field (need unzip support) (Kai). +- Allow -d@ to read in a list of names to delete (11/17/2005). +- AES encryption (3/19/05). + +Features that may make the next release: + +- Allow reading in list of files using @filename. +- When -R, -x, or -i pattern ends in a directory add / to the end + (11/5/2004 Nehal). +- Decide if -R, -i and -x should use external rather than internal patterns. + Also, change pattern matching to not do ex2in() and then in2ex() if + appropriate. (12/26/2005 SMS) +- Though Unicode paths have been implemented and tested, Unicode comments + are not yet supported (except for comments on UTF-8 native systems which + are supported). +- Verbose mode -v may still need work. + +- Add C# example for Zip 3.0 (need to be converted to new DLLs) - original + C# example added with note. +- Path Prefix maybe, so entries added to an archive can have a directory + path string prepended to each path so can zip multiple drives and avoid + name conflicts (4/17/2006). +- UNC paths like \\server\path (4/26/2005). +- Support for other languages maybe. + +- Add About page option similar to -h2 and -v but lists Info-ZIP + information (could be -sa) (4/29/2006). +- Update utilities ZipSplit, ZipNote, and ZipCloak to handle split archives. +- Update ziperr and finish if needed. +- Review memory allocation and fill in memory leaks if any. +- Enhance -FF to fix common problems such as archives ftp in text mode + and fixing checksums so entries can be extracted if that makes + sense (6/17/2007). +- Add \ to / conversion in zipsplit to fix problem in + 1/29/2004 email. +- Encryption bug with small stored file (12/27/2005) (fixed?). + +- When updating large archives with few entries being + updated maybe display something in large periods of + quiet (1/23/2006). +- Windows OEM comments (5/17/2006). +- Example of using MVS zip and unzip (3/30/2004) (Need one). +- UTF-8 comments need to be implemented (6/17/2007) +- Maybe convert ../ in archive (5/20/2006). +- Per so many buffers dll callback (12/23/2005 Ale). +- Allow rename stdin "-" to something else (12/27/2005 gregor). +- Check for possible buffer overrun weaknesses while reading zip files. +- Do Active Template Library (ATL) (4/27/2005). +- Flush Win16 support - to be determined (Mike). +- Way to convert file names on input, converting foo.c to dir/foo_bar.c + for instance (4/8/2004, 3/12/2004). +- French WiZ (not a Zip thing actually but dependent on zip and unzip). +- Then there is that wierd ^D being converted to \000 error reported + in 6/21/2003 email when Zip is outputted into a pipe on Windows ports. + +Old list: + +Main features still missing for next official version (last updated 2/11/2001): + +- what about the binary/text detection ? (seems done) +- -b and -t options in help screen (covered in -h2) +- findfirst/findnext and after that LSSTAT (performance!!) +- use IS_EXEC from djgpp stat.h +- use install in unix/Makefile instead of mkdir -p, look at install sh script. +- #elif for those ports that can handle it. +- what about zopen vs. fopen ? +- Add zcreate or zfcreate for win32. +- Assembler stuff in match.S (subexpressions) +- zipping huge files (> 2G, unsigned 32bit) (done) +- Testsuite for zip and unzip (John D. Mitchell) +- make a version.c or version.h that includes all the compiler names +- run utils with dmalloc(). +- what to do with zip -F and zip -FF (readzipfile2()) ? (done?) +- profiling of the code +- multi disk zip files (could be done) +- zipfile modification tool (Greg) +- Implement -- option (Thomas Klauser, wiz@danbala.tuwien.ac.at) (could be done) +- don't add files with "Archive bit" or add files with "Archive bit" + (uwe.becher@metronet.de) (could be done with -AS and -AC) +- 32 bit file attributes +- generate output without having to seek at all (this seems to be stream output) +- remove contractions from zip error messages, make them clearer (Steve) +- display "[text]" for ascii files when not quiet (no -q) (Timo Salmi) +- does zipnote accept names with version number? +- for a WORM, zip should create temp file only when updating; new archives + should be created directly. +- APPNOTE.TXT specifies "4) The entries in the central directory may + not necessarily be in the same order that files appear in the zipfile" + but readzipfile() relies on same order. (new read does not, and now + the read for -FF searches for central directory matches rather than + rely on the order) +- on Mac, MPW C 3.3.1 requires #if (a || b) ["#if a || b" taken as "#if a"] +- on Unix, let -S be "include non-regular files without reading from them" + (as pkzip on Unix). This requires unzip support. +- zip -l should do ebcdic->ascii translation on CMS and MVS +- zip as subroutine (zdig/241) (some work done on this) +- accept k and M in zipsplit +- store / (part of file name) as ! in OS/2 (problem only with -E ?) +- in addition to -l (LF to CR LF) and -ll (CR LF to LF) add -lc + (LF to CR LF but CR LF remains unchanged) + +Known bugs: + +- On VMS, zip fails reading some files with "byte record too large for + user's buffer". You must use the "-V" option for such files. + (many changes to VMS so may be fixed) + +- on MSDOS, zip386.exe does not like "zip -bc: foo ..." + +- on MSDOS, zip386.exe is sometimes much slower than zip.exe. This is + probably a problem with DJGPP (to be investigated). + +- on NT with C shell, zip should not do file name expansion again. + +- zip zipfile ... ignores existing zipfile if name does not have an extension + (except for the -A option, generally used on self-extracting files). + (archives should probably have extensions. Things like archive.jar work) + +- For an sfx file without extension, "zip -A sfx" works but "zip sfx -A" + doesn't. (because options were required first, but now both OK) + +- When storing files in a zipfile (-0), zip marks all of them as binary. + +- On VMS, some indexed files are not restored correctly after zip -V and unzip. + (This is now known to be a problem of UnZip. The workaround for Zip 2.2 + and newer is to use PK-style VMS extra fields; this is now the default. + NOTE that UnZip 5.32 has been fixed [971019]!) (many VMS changes so + this may be fixed) + +- zip and unzip should use the same pattern matching rules, particularly + on MSDOS and OS/2. On OS/2, "zip foo *.*" should also match files + without extension. + Partially DONE (OS/2 "*.*" matches "*".) + +- there should be a way to avoid updating archive members (only addition + of new files allowed) diff --git a/third_party/infozip/zip/USexport.msg b/third_party/infozip/zip/USexport.msg new file mode 100644 index 000000000..068aa9f3f --- /dev/null +++ b/third_party/infozip/zip/USexport.msg @@ -0,0 +1,75 @@ +From roelofs (at) sonic.net Tue Jun 17 08:26:55 2003 +Date: Tue, 17 Jun 2003 08:26:50 -0700 +Message-Id: <200306171526.h5HFQoaw014091 (at) bolt.sonic.net> +From: Greg Roelofs +Reply-To: Greg Roelofs +To: crypt (at) bis.doc.gov, enc (at) ncsc.mil, web_site (at) bis.doc.gov +Subject: TSU NOTIFICATION - Encryption (Info-ZIP zcrypt.zip) +Cc: newt (at) pobox.com, zip-bugs (at) lists.wku.edu + + + SUBMISSION TYPE: TSU + SUBMITTED BY: Greg Roelofs + SUBMITTED FOR: the Info-ZIP group (an informal, Internet-based + collection of software developers with the contact + address given in next item) + POINT OF CONTACT: Zip-Bugs (at) lists.wku.edu + PHONE and/or FAX: n/a + MANUFACTURER: n/a + PRODUCT NAME/MODEL #: zcrypt + ECCN: 5D002 + + NOTIFICATION: + + ftp://ftp.info-zip.org/pub/infozip/src/zcrypt.zip + + +FURTHER COMMENTS: + +(1) This notice is being sent in order to ensure that we may legally + take advantage of the 6 June 2002 amendment to 740.13 regarding + "corresponding object code." The encryption code in question is + unchanged since our original notification of 9 April 2000, appended + below and also reproduced within the above zcrypt.zip archive. + (Indeed, there has been no change to the core encryption/decryption + code in well over five years.) + +(2) The (larger) source archives for Zip, UnZip, MacZip, WiZ, and + potentially other packages, currently available in the same ftp + directory given above, also contain (or may contain) copies of + the same zcrypt source code. + +(3) ftp.info-zip.org currently points to a site in Germany, so techni- + cally it is not involved in "US export" in any direct way. However, + we encourage other sites to "mirror" our software, and some of these + mirror sites may be US-based (and therefore involved in reexport of + the code in question). In addition, some Info-ZIP members reside in + the US, and www.info-zip.org currently points to a site in Kentucky. + + +ORIGINAL NOTIFICATION: + +From roelofs (at) sonic.net Sun Apr 9 15:11:45 2000 +Date: Sun, 9 Apr 2000 15:11:27 -0700 +Message-Id: <200004092211.PAA20023 (at) sonic.net> +From: Greg Roelofs +To: crypt (at) bxa.doc.gov +Subject: notice of export of unrestricted encryption source code +Cc: newt (at) pobox.com, zip-bugs (at) lists.wku.edu + +The Info-ZIP group, an informal, Internet-based collection of software +developers with contact address Zip-Bugs (at) lists.wku.edu, hereby notifies +the US Bureau of Export Administration (BXA) of the posting of freely +available encryption source code on the Internet under License Exception +TSU, to commence later today at this location: + + ftp://ftp.info-zip.org/pub/infozip/src/zcrypt.zip + +This notification is in accordance with section 740.13(e) of the amended +Export Administration Regulations, as published in the 14 January 2000 +issue of the Federal Register. + +-- +Greg Roelofs newt (at) pobox.com http://pobox.com/~newt/ +Newtware, PNG Group, Info-ZIP, Philips Research, ... + diff --git a/third_party/infozip/zip/WHATSNEW b/third_party/infozip/zip/WHATSNEW new file mode 100644 index 000000000..9e8d52bca --- /dev/null +++ b/third_party/infozip/zip/WHATSNEW @@ -0,0 +1,333 @@ +What's New + +Last updated 1 July 2008 + +This file is the full list of new features and major changes for Zip 3.0 +by beta release. See the announcement file zip30.ann for a quick summary +of all features and changes in Zip 3.0. Also see the file README for +release information, INSTALL for installation procedures, and the manual +pages zip.txt, zipsplit.txt, zipcloak.txt, and zipnote.txt for how to use +the new features. The file CHANGES has all the day-to-day changes made +during development. + + +Below are some of the more significant items on the list for Zip 3.1 +(see ToDo for a more complete list): + +- AES encryption. +- Extended attributes for Windows, Linux, and Mac OS X. +- Support -d@ for deleting list of files. +- Decide if -R, -i and -x should use external rather than internal patterns. +- Though Unicode paths have been implemented and tested, Unicode comments + are not yet supported (except for comments on UTF-8 native systems which + are supported). +- Verbose mode -v may still need work. +- When pattern is directory add end / automatically. +- Add C# example for Zip 3.0 (need to be converted to new DLLs) - original + C# example added with note. +- Path Prefix maybe, so entries added to an archive can have a directory + path string prepended to each path. +- UNC path support maybe. +- Support for other languages maybe. +- Send in your suggestions. +- ... + + +MAJOR CHANGES BY BETA VERSION +----------------------------- + +New things in Zip 3.0 since Zip 3.0h: + +- Unicode fixes. +- Test and fix various ports as needed. +- Update Win32 resource to support more Windows ports. +- Add djgpp 2.x makefile that includes bzip2. +- Add Win32 version resource to Win32 executable. +- Bug fixes. +- Documentation updates. +- Package for release. + + +New things in Zip 3.0h + +- Allow -@ and -x to work together. +- Unicode code cleanup. +- Allow forcing use of UTF-8 storage in standard path and comment. +- Update symbolic link checks. +- Add support for storing 32-bit UIDs/GIDs using new extra field. + Backward compatible support for the old 16-bit UID/GID extra field + remains if Zip is compiled on an OS that has 16-bit UID/GID + storage. +- Update VMS notes. +- Directory scan using -AS (include only files with Windows archive + bit set) now ignores archive bit on directories to include all files + with archive bit set in all directories. Also, to avoid empty + directories being created, -AS now does not store directory + entries. +- Add Unix IBM support. +- Change -W to -ws to free -W for later use. +- Fix large file support for MinGW. +- Fix large file support for bzip2. +- Fix compile error in ZipCloak when UNICODE_SUPPORT is not enabled. +- Fix Unicode bug in ZipCloak involving Unicode paths. +- Long Unicode escapes changed from #Lxxxxxxxx to #Lxxxxxx to shorten + paths with escaped Unicode. +- Bug fixes. + + +New things in Zip 3.0g + +- Add split support to VB project for Zip64. +- Disable reading of Unix FIFOs unless new -FI option used to avoid an + archiving operation stopping when it hits an active unfed FIFO. +- The "[list]" wildcard expression (regular expression matching of any + character or range of characters in list) is now disabled on DOS and + Windows as it has caused confusion when filenames have [ and ] in + them. The new -RE option reenables it. +- Add negation to many display options such as -dc and -db. +- Allow -FF to read and fix archives having local entries that appear + after central directory entries. +- Bug fixes. + + +New things in Zip 3.0f + +- bzip2 - The bzip2 compression method looks supported for at least + Windows, Unix, and VMS using the bzip2 library. A new option, -Z cm, + selects the compression method. + +- Split archives - Can now use -s to create a split archive. The + default is to update split files as the archive is being written, + which requires all splits to remain open until the archive is done. + This should be no problem when writing the archive to a hard drive, + for example, and this approach creates archives that should be + supported by all unzips that support splits. Adding the -sp option + enables split pause mode that instead writes splits that do not + need updating and pauses Zip after each split. This allows splits + to be written directly to removable media, however -sp archives + may not be as universally compatible. + +- Unicode support - Zip now stores Unicode paths that should be more + portable across character sets and languages. The unzip must have + Unicode support enabled or the Unicode paths are ignored. If + reading an archive with Unicode paths, unsupported characters are + replaced by #Uxxxx and #Lxxxxxxxx escapes in the file name. Option + -UN controls how Unicode is handled. Also, on systems where the + current character set is UTF-8, preliminary support for the new + General Purpose Bit Flag, bit 11, UTF-8 flag, that indicates UTF-8 + is stored in the path and comment fields is implemented for paths. +- Unicode on Win32 - On WIN32 systems that support the wide character + calls (mainly NT and later systems using NTFS), when UNICODE SUPPORT + is enabled Zip will now do directory scans using Unicode and convert + the Unicode paths to the local character set for storage in the standard + path field and store UTF-8 in the Unicode extra field. This allows + directory scans to complete successfully regardless of the character + set the path is in. On Win9x systems wide character scans are not + generally supported and Zip automatically uses a local character scan + instead. + +- Keep extra fields option - The default operation has been, and continues + to be, to read then strip old extra fields when reading entries from an + existing archive and then recreate the extra fields that Zip knows about. + Extra fields specific to each operating system get added by default also. + The new option -X- (negated -X) keeps any old extra fields, copying + them to the updated archive unchanged (unless Zip has updated them). + The unnegated -X still strips most all extra fields except Zip64, + Unicode, and UT time. + +- License - minor updates to the license. + +- Windows OEM - When compiled with WIN32_OEM (the default for WIN32), + Zip on WIN32 now stores OEM paths, which should be more compatible + with other zips and should fix some character set problems. +- Windows Archive Bit support - On Windows can now use new -AS + (include if archive bit set) option to select files with the DOS + archive bit set and use new -AC (clear archive bits) option to clear + the archive bits on files after the archive has been created. + But -DF is probably better. + +- Difference mode - A new option -DF (--dif) creates an output archive + that includes only files changed or new since the input archive was + created. Can use to create incremental backups. +- File Sync - The new option -FS enables File Sync, a new mode that + synchronizes the entries in an archive with the files on the file + system, adding updating, and deleting entries as needed. This + should create the same results as creating a new archive, but + since existing entries are copied, may be much faster. + +- Copy Mode - A new --out option allows creating a new archive with a + different name than the input archive, leaving the input archive + unchanged. This allows updating split archives. It also allows + for a new copy mode to select entries in one archive and copy them + directly to a new archive. +- Empty archives - Now an empty archive is created when -i or -i@ is used + and the file patterns given do not match anything. This has been + requested to support scripts. + +- Global dots - A new -dg option now displays progress dots as -dd does, + but instead of displaying them for each file, the dots track the total + bytes read for the archive. The -dg option also works when -q is used + to disable most output, which allows for something like zip -qdgds 100m + to be used to not display specific files but display a dot every 100 MB + as a global status. +- Date range - Can now use -t and -tt to set a date range +- Fix options - Option -F redone and can recover files from an archive + with a mostly complete central directory more reliably, but no longer + can handle truncated archives. Option -FF redone and now can salvage + files from slightly more damaged archives, including truncated archives. + In some ways -F is less powerful but more stable than it was and -FF will + be needed where -F in Zip 2.32 was enough. One big change is -F and -FF + both now support split archives. +- Console writing - Updates to how messages are written to the console have + been made including more consistent handling of line breaks. +- Show Files options - Option -sf lists the files that would be operated + on. This option can be used alone to list the files in an archive. + Also see options -su and -sU for showing Unicode paths. +- UnZip Check - Now check that UnZip 6.00 or later is being used for + unzip if testing a Zip64 archive. A new option -TT can be used to set + the unzip to use with the -T check. Currently UnZip does not support + split archives so split archives can't be tested by UnZip. +- Streaming - Directories are now handled better when streaming. +- Case matching - Normally all matching against archive entries is case + sensitive, so *.BAR will not match or find foo.bar in an archive + when deleting, copying, or freshening entries (deleting and copying + only on VMS). New option -ic (--ignore-case) enables case insensitive + matching. Currently -ic is only implemented on WIN32 and VMS. + +- Delete date bug fixed - Bug when using -d to delete files while + using -t or -tt to select the files based on date is fixed +- Large file encryption bug fixed - Fix for bug that very rarely + results in bad data being stored when deflating and encrypting + uncompressable data and resulting in CRC errors when extracting, + but the chance of error increases with file size (thanks to + WinZip for finding this bug). See CHANGES for details. + + +New things in Zip 3.0e + +- Bugs described in Debian patches 004 (unix configure script update) and + 005 (large path bug) fixed +- Various fixes +- Add optional running stats and also end stats if not all files could + be read +- Options -l and -ll now do quick binary check on first buffer and skip + formatting if first buffer has binary - still check at end to note + if formatting was done on file that was later determined to be binary, + but now potential file corruption is generally avoided +- Main binary check now uses new algorithm that should also treat UTF-8 and + other similar encodings as text, allowing proper line end translation + for UTF-8 files +- When output is not updatable by seeking back and Zip64 is enabled, output + is forced to Zip64 to avoid possible later need for Zip64 when not enabled +- More work on splits, but still not usable +- Fixes for djgpp +- Add log file capability to save all errors and optionally messages +- Add code to test for a Zip64 archive when compiled without Zip64 support +- New VC6 projects for Win32 and WinDLL +- Updates to extended help +- Changes to force-zip64 option +- ZE_BIG error now given also for files too big to read or write +- Fix file delete bug +- Update license +- Update export documentation +- Add VMS extended filename support +- Add directory traversal improvements, some for Win32 ports and some for + all ports, that can result in a 10 times increase in speed in some cases + + +New things in Zip 3.0d + +- Some large file crypt fixes +- Some updates to support WiZ +- On VMS, changed -V (/VMS) processing to truncate file at EOF, allowing + greater compatability with non-VMS systems. New -VV (/VMS=ALL) option + saves all allocated blocks in a file. (Previously, -V did neither.) +- On VMS, pushed 2GB file size limit with -V out to 4GB +- On VMS (recent, non-VAX), with SET PROCESS /PARSE = EXTEND, + command-line case is preserved. This obviates quoting upper-case + options, like -V, when enabled +- On VMS, fixed problems with mixed-case directory names. Also changed + to keep ODS5 extended file name escape characters ("^") out of the + archived names in simple cases +- Changes to the display dots +- Option -W should now force wildcard matching to not cross directory + separators. For example, a/b*r/d will match a/bar/d but not a/ba/r/d +- Option -nw should turn off all wildcard matching so foo[bar] is matched + literally and [bar] is not considered a regular expression +- Atheos port +- Debugging of Unix and VMS large file ports. Most features may work now + on these ports for large files. Still need to fix 2 GB to 4 GB when not + compiled with large file support +- On VMS, added an open callback function which (where supported) senses + the process RMS_DEFAULT values for file extend quantity (deq) + multi-block count (mbc), and multi-buffer count (mbf), and sets the + FAB/RAB parameters accordingly. The default deq is now much larger + than before (16384 blocks, was none), and the default mbc is now 127 + (up from 64), speeding creation of a large archive file. The "-v" + option shows some of the activity. On old VMS versions, RMS_DEFAULT + sensing (GETJPI) fails (silently, without "-v"), and no changes will + be made. Even there, (DCL) SET RMS /EXTEND = can help + performance. RMS_DEFAULT values override built-in default values. + + +New things in Zip 3.0c + +- Converted to using 64-bit file environment instead of transitional functions + like fseeko64 for ports that support it +- Added "--" argument to read all following arguments as paths +- Second help page added +- Binary detection adjusted from 20% binary is binary to 2% +- When -R and -i used together now -i has precedence over -R +- Archive names with spaces can now be tested on MSDOS and Win32 + + +New things in Zip 3.0b + +- Fixed ifdefs so can test base code by compiling with NO_LARGE_FILE_SUPPORT, then + compiling with NO_ZIP64_SUPPORT to test 64-bit file calls (if port enables) but + otherwise use base code, and compiling normally to enable Zip64 code +- Unix Zip64 fixes - should now be able to create and read large files +- WinDLL changes to support Zip64. Zip 3.0 dll named Zip32z64.dll +- New VB example to show use of Zip32z64.dll +- New options -sc (show final command line and exit) and -sd (show each + step zip is doing, a little different than verbose which is still there) added + to help debug but both or at least -sd might go away in the release +- Some minor posted bugs fixed (see Changes) + + +New things in Zip 3.0a + +- Initial Zip64 support allowing large files and large numbers of files +- New command line processor +- Other changes, see file Changes + + +Note: Zip 2.4 was never released. That code was the start of the Zip 3.0 +effort above. + + +New things in Zip 2.3 + +- IBM OS/390 port (Unix like, but EBCDIC) by Paul von Behren +- Apple Macintosh (MACOS) port by Dirk Haase +- Theos port by Jean-Michel Dubois +- Multibyte characterset support by Yoshioka Tsuneo +- Support for ISO 8601 date format with -t and -tt options +- Info-ZIP license + + +New things in Zip 2.2 + +- BEOS port by Chris Herborth +- QDOS port by Jonathan Hudson +- TANDEM port by Dave Smith +- WINDLL port (16-bit Win 3.x and 32-bit WinNT/Win95) by Mike White +- SYSV packages support by John Bush +- zip -P SeCrEt encrypts entries in the zip file with password SeCrEt + (WARNING: THIS IS INSECURE, use at your own risk) +- zip -R recurses into subdirectories of current dir like "PKZIP -rP" +- zip -x@exclude.lst excludes files specified in the file exclude.lst +- zip -i@include.lst includes files specified in the file include.lst +- zip -@ only handles one filename per line, but supports whitespace in names +- zip -t mmddyyyy, 4 digit year number for uniqueness of years beyond 2000 +- zip -tt mmddyyyy only includes files before a specified date diff --git a/third_party/infozip/zip/WHERE b/third_party/infozip/zip/WHERE new file mode 100644 index 000000000..94a0d55c6 --- /dev/null +++ b/third_party/infozip/zip/WHERE @@ -0,0 +1,261 @@ +__________________________________________________________________________ + + This is the Info-ZIP file ``WHERE,'' last updated on 1 March 2005. +__________________________________________________________________________ + + This file is out of date. We plan to update the structure of the ftp + site shortly and should be updating this file as soon as that's done. + + The latest version of this file can be found online at: + + ftp://ftp.info-zip.org/pub/infozip/doc/WHERE + + Note that some ftp sites may not yet have the latest versions of Zip + and UnZip when you read this. The latest versions always appear in + ftp://ftp.info-zip.org/pub/infozip/ (and subdirectories thereof) first, + except for encryption binaries, which always appear in + ftp://ftp.icce.rug.nl/infozip/ (and subdirectories) first. + + IF YOU FIND AN ERROR: please let us know! We don't have time to + check each and every site personally (or even collectively), so any + number of the sites listed below may have moved or disappeared en- + tirely. E-mail to Zip-Bugs@lists.wku.edu and we'll update this file. +__________________________________________________________________________ + + +Info-ZIP's home WWW site is listed on Yahoo and is at: + + ftp://ftp.info-zip.org/pub/infozip/Info-ZIP.html (master version) + http://ftp.info-zip.org/pub/infozip/ (master version) + http://www.info-zip.org/ + +Note that the old sites at http://www.cdrom.com/pub/infozip/ and +http://www.freesoftware.com/pub/infozip are PERMANENTLY BROKEN. They +cannot be updated or removed, apparently. + +The Zip and UnZip pages have links to most known mirror sites carrying our +source and/or binary distributions, and they generally are more up-to-date +and have better information than what you are reading: + + ftp://ftp.info-zip.org/pub/infozip/Zip.html + ftp://ftp.info-zip.org/pub/infozip/UnZip.html + +The related zlib package by Info-ZIP's Jean-loup Gailly and Mark Adler is at: + + http://www.zlib.net/ + +Source-code archives for Info-ZIP's portable Zip, UnZip, and related +utilities: + + zip231.zip Zip 2.31 (deflation; includes zipnote/zipsplit/zipcloak) + zip231.tar.Z ditto, compress'd tar format + + zip11.zip Zip 1.1 (shrinking, implosion; compatible w. PKUNZIP 1.1) + zip11.tar.Z ditto, compress'd tar format + + unzip552.zip UnZip 5.52 (all methods[*]; unzip/funzip/unzipsfx/zipgrep) + unzip552.tar.gz ditto, gzip'd tar format + unzip552.tar.Z ditto, compress'd tar format + + unred552.zip UnZip 5.52 add-on, contains copyrighted unreduce support + + zcrypt29.zip encryption support for Zip 2.3[**] + zcrypt10.zip encryption support for Zip 1.1 + + MacZip106src.zip contains all the GUI stuff and the project files to build + the MacZip main-app. To build MacZip successfully, both + the Zip 2.31 and UnZip 5.52 sources are required, too. + + wiz502.zip WiZ 5.02, Windows 9x/NT GUI front-end for Info-ZIP DLLs + wiz502+dlls.zip WiZ 5.02, Windows 9x/NT GUI front-end plus DLL sources + +[*] Unreducing is disabled by default, but is available as add-on. + As of July 2004, Unisys's LZW patent was expired worldwide, and + unshrinking is turned on by default since the release of UnZip 5.52. + See UnZip's INSTALL file for details. + +[**] As of January 2000, US export regulations were amended to allow export + of free encryption source code from the US. As of June 2002, these + regulations were further relaxed to allow export of encryption binaries + associated with free encryption source code. The Zip 2.31, UnZip 5.52 + and Wiz 5.02 archives now include full crypto source code. As of the + Zip 2.31 release, all official binaries include encryption support; the + former "zcr" archives ceased to exist. + (Note that restrictions may still exist in other countries, of course.) + +Executables archives (and related files) for Info-ZIP's software; not all +of these will be immediately available due to lack of access to appropriate +systems on the part of Info-ZIP members. + + zip231x.zip MSDOS executables and docs + zip231x1.zip OS/2 1.x (16-bit) executables and docs + zip231x2.zip OS/2 2/3/4.x (32-bit) executables and docs + zip231xA.zip Amiga executables and docs + zip231xB.zip BeOS executables and docs + zip231xC.zip VM/CMS executable and docs + zip231xK.zip Tandem NSK executables and docs + zip231xM.xmit MVS classic executable + zip231xM-docs.zip MVS classic port, docs only + zip231dN.zip WinNT/Win9x (Intel) DLL, header files, docs + zip231xN.zip WinNT/Win9x (Intel) executables and docs + zip231xN-axp.zip WinNT (Alpha AXP) executables and docs + zip231xN-mip.zip WinNT (MIPS R4000) executables and docs + zip231xN-ppc.zip WinNT (PowerPC) executables and docs + zip231xO.zip IBM OS/390 Open Edition binaries and docs + zip231xQ.zip SMS/QDOS executables and docs + zip231xR.zip Acorn RISC OS executables and docs + zip231xT.zip Atari TOS executables and docs + zip231-vms-axp-obj.zip + VMS (Alpha AXP) object libs, link procedure and docs + zip231-vms-axp-exe.zip + VMS (Alpha AXP) executables for VMS 6.1 or later and docs + zip231-vms-vax-decc-obj.zip + VMS (VAX) object libs (new DEC C), link procedure and docs + zip231-vms-vax-decc-exe.zip + VMS (VAX) executables (DEC C) for VMS 6.1 or later; docs + zip231-vms-vax-vaxc-obj.zip + VMS (VAX) object libs (old VAX C), link procedure and docs + zip231x.hqx Macintosh BinHex'd executables and docs + + unz552x.exe MSDOS self-extracting executable (16-bit unzip, ..., docs) + unz552x3.exe MSDOS self-extracting executable (16-, 32-bit unzip, docs) + unz552x1.exe OS/2 1.x (16-bit) self-extracting executables and docs + unz552x2.exe OS/2 2/3/4.x (32-bit) self-extracting executables and docs + unz552d2.zip OS/2 2/3/4.x (32-bit) DLL, header file, demo exe and docs + unz552xA.ami Amiga self-extracting executables and docs + unz552xA.lha Amiga executables and docs, LHa archive + unz552xB.sfx BeOS self-extracting executables and docs + unz552xB.tar.gz BeOS executables and docs, gzip'd tar archive + unz552xC.mod VM/CMS executable module in "packed" format + unz552xC-docs.zip VM/CMS docs, only + unz552xF.zip FlexOS executable and docs + unz552xK.zip Tandem NSK executable and docs + unz552xM.xmit MVS classic executable + unz552xM-docs.zip MVS classic port, docs only + unz552dN.zip NT4/W2K/XP/2K3/W9x (32-bit Intel) DLL, header files, docs + unz552xN.exe NT/2K/XP/2K3/W9x self-extracting i386 executables and docs + unz552xN-axp.exe WinNT (Alpha AXP) self-extracting executables and docs + unz552xN-mip.exe WinNT (MIPS R4000) self-extracting executables and docs + unz552xN-ppc.exe WinNT (PowerPC) self-extracting executables and docs + unz552xQ.sfx SMS/QDOS self-extracting executables and docs + unz552xO.tar.Z IBM OS/390 Open edition (Unix-like), exes and docs + unz552xR.exe Acorn RISC OS self-extracting executables and docs + unz552xR.spk Acorn RISC OS Spark'd executables and docs + unz552xT.tos Atari TOS self-extracting executables and docs + unz552x-vms-axp-obj.bck VMS backup saveset, + contains UnZip (Alpha) obj libs, link procedure, docs + unz552x-vms-axp-obj.exe VMS (Alpha AXP) SFX archive (statically linked), + contains UnZip (Alpha) obj libs, link procedure, docs + unz552x-vms-axp-exe.exe VMS (Alpha AXP) SFX archive (dynamically linked), + contains UnZip (Alpha AXP, DEC C) executables and docs, + smaller than object archive, but requires VMS 6.1 + unz552x-vms-vax-decc-obj.bck VMS backup saveset, + contains UnZip (new DEC C) obj libs, link procedure, docs + unz552x-vms-vax-decc-obj.exe VMS (VAX) SFX archive (statically linked), + contains UnZip (new DEC C) obj libs, link procedure, docs + unz552x-vms-vax-decc-exe.exe VMS (VAX) SFX archive (dynamically linked), + contains UnZip (new DEC C) executables and docs, + smaller than object archive, but requires VMS 6.1 + unz552x-vms-vax-vaxc-obj.bck VMS backup saveset, + contains UnZip (old VAX C) obj libs, link procedure, docs + unz552x-vms-vax-vaxc-obj.exe VMS (VAX) SFX archive (statically linked), + contains UnZip (old VAX C) obj libs, link procedure, docs + unz552x.hqx Macintosh BinHex'd executables and docs for unzip + (unz552x.tar.{Z,gz} Unix exes/docs for Solaris 2.x, SCO Unix, Linux, etc., + depending on directory/location; generally only provided + in cases where the OS does *not* ship with a bundled C + compiler) + + MacZip106nc.hqx Macintosh combined Zip&UnZip application with GUI, + executables and docs (no encryption) + MacZip106c.hqx Macintosh combined Zip&UnZip application with GUI, + executables and docs (with encryption) + + wiz502xN.exe WiZ 5.02 32-bit (Win9x/NT/2K/XP/2K3) app+docs (self-extr.) + + UnzpHist.zip complete changes-history of UnZip and its precursors + ZipHist.zip complete changes-history of Zip + +ftp/web sites for the US-exportable sources and executables: + + NOTE: Look for the Info-ZIP file names given above (not PKWARE or third- + party stuff) in the following locations. Some sites like to use slightly + different names, such as zip-2.31.tar.gz instead of zip231.tar.Z. + + ftp://ftp.info-zip.org/pub/infozip/ [THE INFO-ZIP HOME SITE] + ftp://sunsite.doc.ic.ac.uk/packages/zip/ [MIRRORS THE INFO-ZIP HOME SITE] + ftp://unix.hensa.ac.uk/mirrors/uunet/pub/archiving/zip/ + + ftp://ftp.cmdl.noaa.gov/aerosol/doc/archiver/{all,dos,os2,mac,vax_alpha}/ + ftp://garbo.uwasa.fi/pc/arcers/ [AND OTHER GARBO MIRRORS] + ftp://garbo.uwasa.fi/unix/arcers/ [AND OTHER GARBO MIRRORS] + ftp://ftp.elf.stuba.sk/pub/pc/pack/ [AND OTHER STUBA MIRRORS] + ftp://ftp-os2.cdrom.com/pub/os2/archiver/ + ftp://ftp-os2.nmsu.edu/os2/archiver/ + ftp://ftp.informatik.tu-muenchen.de/pub/comp/os/os2/archiver/ + ftp://sumex-aim.stanford.edu/info-mac/cmp/ + ftp://ftp.wustl.edu/pub/aminet/util/arc/ [AND OTHER AMINET MIRRORS] + ftp://atari.archive.umich.edu/pub/Archivers/ [AND OTHER UMICH MIRRORS] + http://www.umich.edu/~archive/atari/Archivers/ + ftp://jake.educom.com.au/pub/infozip/acorn/ [Acorn RISC OS] + http://www.sitec.net/maczip/ [MacZip port] + +ftp/web sites for the encryption and decryption sources and/or executables: + + Outside the US: + ftp://ftp.info-zip.org/pub/infozip/ [THE INFO-ZIP HOME SITE] + ftp://ftp.icce.rug.nl/infozip/ [THE INFO-ZIP ENCRYPTION HOME SITE] + ftp://ftp.elf.stuba.sk/pub/pc/pack/ + ftp://garbo.uwasa.fi/pc/arcers/ + ftp://ftp.inria.fr/system/arch-compr/ + ftp://ftp.leo.org/pub/comp/os/os2/leo/archiver/ + (mail server at ftp-mailer@ftp.leo.org) + + ftp://ftp.win.tue.nl/pub/compression/zip/ + ftp://ftp.uni-erlangen.de/pub/pc/msdos/arc-utils/zip/ + + +The primary distribution site for the MacZip port can be found at: + + http://www.sitec.net/maczip/ + +ftp sites for VMS-format Zip and UnZip packages (sources, object files and +executables, no encryption/decryption--see also "Mail servers" section below): + + ftp.spc.edu [192.107.46.27] and ftp.wku.edu: + + [.MACRO32]AAAREADME.TXT + [.MACRO32.SAVESETS]UNZIP.BCK or UNZIP.ZIP (if already have older version) + [.MACRO32.SAVESETS]ZIP.ZIP + +To find other ftp/web sites: + + The "archie" ftp database utility can be used to find an ftp site near + you (although the command-line versions always seem to find old ver- + sions...the `FTPsearch' server at http://ftpsearch.ntnu.no/ftpsearch + --formerly `Archie 95'--is quite up-to-date, however). Or check a stan- + dard WWW search engine like AltaVista (http://www.altavista.digital.com/) + or Yahoo (http://www.yahoo.com/). If you don't know how to use these, + DON'T ASK US--read the web sites' help pages or check the Usenet groups + news.announce.newusers or news.answers or some such, or ask your system + administrator. + +Mail servers: + + To get the encryption sources by e-mail, send the following commands + to ftp-mailer@informatik.tu-muenchen.de: + + get /pub/comp/os/os2/archiver/zcrypt29.zip + quit + + To get the VMS Zip/UnZip package by e-mail, send the following + commands in the body of a mail message to fileserv@wku.edu (the + "HELP" command is also accepted): + + SEND FILESERV_TOOLS + SEND UNZIP + SEND ZIP + + To get Atari executables by e-mail, send a message to + atari@atari.archive.umich.edu for information about the mail server. +__________________________________________________________________________ diff --git a/third_party/infozip/zip/api.c b/third_party/infozip/zip/api.c new file mode 100644 index 000000000..8a013e5bb --- /dev/null +++ b/third_party/infozip/zip/api.c @@ -0,0 +1,719 @@ +/* clang-format off */ +/* + api.c - Zip 3 + + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/*--------------------------------------------------------------------------- + + api.c + + This module supplies a Zip dll engine for use directly from C/C++ + programs. + + The entry points are: + + ZpVer *ZpVersion(void); + int EXPENTRY ZpInit(LPZIPUSERFUNCTIONS lpZipUserFunc); + int EXPENTRY ZpArchive(ZCL C, LPZPOPT Opts); + + This module is currently only used by the Windows dll, and is not used at + all by any of the other platforms, although it should be easy enough to + implement this on most platforms. + + ---------------------------------------------------------------------------*/ +#define __API_C + +#include +#ifdef WINDLL +# include +# include "windll/windll.h" +#endif + +#ifdef OS2 +# define INCL_DOSMEMMGR +# include +#endif + +#ifdef __BORLANDC__ +#include +#endif +#include +#include +#include "api.h" /* this includes zip.h */ +#include "crypt.h" +#include "revision.h" +#ifdef USE_ZLIB +# include "zlib.h" +#endif + + +DLLPRNT *lpZipPrint; +DLLPASSWORD *lpZipPassword; +extern DLLCOMMENT *lpComment; +ZIPUSERFUNCTIONS ZipUserFunctions, far * lpZipUserFunctions; + +int ZipRet; +char szOrigDir[PATH_MAX]; +BOOL fNo_int64 = FALSE; /* flag for DLLSERVICE_NO_INT64 */ + +/* Local forward declarations */ +extern int zipmain OF((int, char **)); +int AllocMemory(unsigned int, char *, char *, BOOL); +int ParseString(LPSTR, unsigned int); +void FreeArgVee(void); + +ZPOPT Options; +char **argVee; +unsigned int argCee; + +/*--------------------------------------------------------------------------- + Local functions + ---------------------------------------------------------------------------*/ + +char szRootDir[PATH_MAX], szExcludeList[PATH_MAX], szIncludeList[PATH_MAX], szTempDir[PATH_MAX]; + +int ParseString(LPSTR s, unsigned int ArgC) +{ +unsigned int i; +int root_flag, m, j; +char *str1, *str2, *str3; +size_t size; + +i = ArgC; +str1 = (char *) malloc(lstrlen(s)+4); +lstrcpy(str1, s); +lstrcat(str1, " @"); + +if ((szRootDir != NULL) && (szRootDir[0] != '\0')) + { + root_flag = TRUE; + if (szRootDir[lstrlen(szRootDir)-1] != '\\') + lstrcat(szRootDir, "\\"); + } +else + root_flag = FALSE; + +str2 = strchr(str1, '\"'); /* get first occurance of double quote */ + +while ((str3 = strchr(str1, '\t')) != NULL) + { + str3[0] = ' '; /* Change tabs into a single space */ + } + +/* Note that if a quoted string contains multiple adjacent spaces, they + will not be removed, because they could well point to a valid + folder/file name. +*/ +while ((str2 = strchr(str1, '\"')) != NULL) + /* Found a double quote if not NULL */ + { + str3 = strchr(str2+1, '\"'); /* Get the second quote */ + if (str3 == NULL) + { + free(str1); + return ZE_PARMS; /* Something is screwy with the + string, bail out */ + } + str3[0] = '\0'; /* terminate str2 with a NULL */ + + /* strip unwanted fully qualified path from entry */ + if (root_flag) + if ((_strnicmp(szRootDir, str2+1, lstrlen(szRootDir))) == 0) + { + m = 0; + str2++; + for (j = lstrlen(szRootDir); j < lstrlen(str2); j++) + str2[m++] = str2[j]; + str2[m] = '\0'; + str2--; + } + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + /* argCee is incremented in AllocMemory */ + if (AllocMemory(i, str2+1, "Creating file list from string", TRUE) != ZE_OK) + { + free(str1); + return ZE_MEM; + } + i++; + str3+=2; /* Point past the whitespace character */ + str2[0] = '\0'; /* Terminate str1 */ + lstrcat(str1, str3); + } /* end while */ + +/* points to first occurance of a space */ +str2 = strchr(str1, ' '); + +/* Go through the string character by character, looking for instances + of two spaces together. Terminate when you find the trailing @ +*/ +while ((str2[0] != '\0') && (str2[0] != '@')) + { + while ((str2[0] == ' ') && (str2[1] == ' ')) + { + str3 = &str2[1]; + str2[0] = '\0'; + lstrcat(str1, str3); + } + str2++; + } + +/* Do we still have a leading space? */ +if (str1[0] == ' ') + { + str3 = &str1[1]; + lstrcpy(str1, str3); /* Dump the leading space */ + } + + +/* Okay, now we have gotten rid of any tabs and replaced them with + spaces, and have replaced multiple spaces with a single space. We + couldn't do this before because the folder names could have actually + contained these characters. +*/ + +str2 = str3 = str1; + +while ((str2[0] != '\0') && (str3[0] != '@')) + { + str3 = strchr(str2+1, ' '); + str3[0] = '\0'; + /* strip unwanted fully qualified path from entry */ + if (root_flag) + if ((_strnicmp(szRootDir, str2, lstrlen(szRootDir))) == 0) + { + m = 0; + for (j = lstrlen(Options.szRootDir); j < lstrlen(str2); j++) + str2[m++] = str2[j]; + str2[m] = '\0'; + } + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + if (AllocMemory(i, str2, "Creating file list from string", TRUE) != ZE_OK) + { + free(str1); + return ZE_MEM; + } + i++; + str3++; + str2 = str3; + } +free(str1); +return ZE_OK; +} + +int AllocMemory(unsigned int i, char *cmd, char *str, BOOL IncrementArgCee) +{ +if ((argVee[i] = (char *) malloc( sizeof(char) * strlen(cmd)+1 )) == NULL) + { + if (IncrementArgCee) + argCee++; + FreeArgVee(); + fprintf(stdout, "Unable to allocate memory in zip library at %s\n", str); + return ZE_MEM; + } +strcpy( argVee[i], cmd ); +argCee++; +return ZE_OK; +} + +void FreeArgVee(void) +{ +unsigned i; + +/* Free the arguments in the array */ +for (i = 0; i < argCee; i++) + { + free (argVee[i]); + argVee[i] = NULL; + } +/* Then free the array itself */ +free(argVee); + +/* Restore the original working directory */ +chdir(szOrigDir); +#ifdef __BORLANDC__ +setdisk(toupper(szOrigDir[0]) - 'A'); +#endif + +} + + +/*--------------------------------------------------------------------------- + Documented API entry points + ---------------------------------------------------------------------------*/ + +int EXPENTRY ZpInit(LPZIPUSERFUNCTIONS lpZipUserFunc) +{ +ZipUserFunctions = *lpZipUserFunc; +lpZipUserFunctions = &ZipUserFunctions; + +if (!lpZipUserFunctions->print || + !lpZipUserFunctions->comment) + return FALSE; + +return TRUE; +} + +int EXPENTRY ZpArchive(ZCL C, LPZPOPT Opts) +/* Add, update, freshen, or delete zip entries in a zip file. See the + command help in help() zip.c */ +{ +int k, j, m; +size_t size; + +Options = *Opts; /* Save off options, and make them available locally */ +szRootDir[0] = '\0'; +szExcludeList[0] = '\0'; +szIncludeList[0] = '\0'; +szTempDir[0] = '\0'; +if (Options.szRootDir) lstrcpy(szRootDir, Options.szRootDir); +if (Options.szExcludeList) lstrcpy(szExcludeList, Options.szExcludeList); +if (Options.szIncludeList) lstrcpy(szIncludeList, Options.szIncludeList); +if (Options.szTempDir) lstrcpy(szTempDir, Options.szTempDir); + +getcwd(szOrigDir, PATH_MAX); /* Save current drive and directory */ + +if ((szRootDir != NULL) && (szRootDir[0] != '\0')) + { + /* Make sure there isn't a trailing slash */ + if (szRootDir[lstrlen(szRootDir)-1] == '\\') + szRootDir[lstrlen(szRootDir)-1] = '\0'; + + chdir(szRootDir); +#ifdef __BORLANDC__ + setdisk(toupper(szRootDir[0]) - 'A'); +#endif + } + +argCee = 0; + +/* malloc additional 40 to allow for additional command line arguments. Note + that we are also adding in the count for the include lists as well as the + exclude list. */ +if ((argVee = (char **)malloc((C.argc+40)*sizeof(char *))) == NULL) + { + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } +if ((argVee[argCee] = (char *) malloc( sizeof(char) * strlen("wiz.exe")+1 )) == NULL) + { + free(argVee); + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } +strcpy( argVee[argCee], "wiz.exe" ); +argCee++; + + +/* Set compression level efficacy -0...-9 */ +if (AllocMemory(argCee, "-0", "Compression", FALSE) != ZE_OK) + return ZE_MEM; + +/* Check to see if the compression level is set to a valid value. If + not, then set it to the default. +*/ +if ((Options.fLevel < '0') || (Options.fLevel > '9')) + { + Options.fLevel = '6'; + if (!Options.fDeleteEntries) + fprintf(stdout, "Compression level set to invalid value. Setting to default\n"); + } + +argVee[argCee-1][1] = Options.fLevel; + +if (Options.fOffsets) /* Update offsets for SFX prefix */ + { + if (AllocMemory(argCee, "-A", "Offsets", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fDeleteEntries) /* Delete files from zip file -d */ + { + if (AllocMemory(argCee, "-d", "Delete", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fNoDirEntries) /* Do not add directory entries -D */ + { + if (AllocMemory(argCee, "-D", "No Dir Entries", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fFreshen) /* Freshen zip file--overwrite only -f */ + { + if (AllocMemory(argCee, "-f", "Freshen", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fRepair) /* Fix archive -F or -FF */ + { + if (Options.fRepair == 1) + { + if (AllocMemory(argCee, "-F", "Repair", FALSE) != ZE_OK) + return ZE_MEM; + } + else + { + if (AllocMemory(argCee, "-FF", "Repair", FALSE) != ZE_OK) + return ZE_MEM; + } + } +if (Options.fGrow) /* Allow appending to a zip file -g */ + { + if (AllocMemory(argCee, "-g", "Appending", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fJunkDir) /* Junk directory names -j */ + { + if (AllocMemory(argCee, "-j", "Junk Dir Names", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fEncrypt) /* encrypt -e */ + { + if (AllocMemory(argCee, "-e", "Encrypt", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fJunkSFX) /* Junk sfx prefix */ + { + if (AllocMemory(argCee, "-J", "Junk SFX", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (Options.fForce) /* Make entries using DOS names (k for Katz) -k */ + { + if (AllocMemory(argCee, "-k", "Force DOS", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (Options.fLF_CRLF) /* Translate LF_CRLF -l */ + { + if (AllocMemory(argCee, "-l", "LF-CRLF", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fCRLF_LF) /* Translate CR/LF to LF -ll */ + { + if (AllocMemory(argCee, "-ll", "CRLF-LF", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fMove) /* Delete files added to or updated in zip file -m */ + { + if (AllocMemory(argCee, "-m", "Move", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (Options.fLatestTime) /* Set zip file time to time of latest file in it -o */ + { + if (AllocMemory(argCee, "-o", "Time", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (Options.fComment) /* Add archive comment "-z" */ + { + if (AllocMemory(argCee, "-z", "Comment", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (Options.fQuiet) /* quiet operation -q */ + { + if (AllocMemory(argCee, "-q", "Quiet", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fRecurse == 1) /* recurse into subdirectories -r */ + { + if (AllocMemory(argCee, "-r", "Recurse -r", FALSE) != ZE_OK) + return ZE_MEM; + } +else if (Options.fRecurse == 2) /* recurse into subdirectories -R */ + { + if (AllocMemory(argCee, "-R", "Recurse -R", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fSystem) /* include system and hidden files -S */ + { + if (AllocMemory(argCee, "-S", "System", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fExcludeDate) /* Exclude files newer than specified date -tt */ + { + if ((Options.Date != NULL) && (Options.Date[0] != '\0')) + { + if (AllocMemory(argCee, "-tt", "Date", FALSE) != ZE_OK) + return ZE_MEM; + if (AllocMemory(argCee, Options.Date, "Date", FALSE) != ZE_OK) + return ZE_MEM; + } + } + +if (Options.fIncludeDate) /* include files newer than specified date -t */ + { + if ((Options.Date != NULL) && (Options.Date[0] != '\0')) + { + if (AllocMemory(argCee, "-t", "Date", FALSE) != ZE_OK) + return ZE_MEM; + if (AllocMemory(argCee, Options.Date, "Date", FALSE) != ZE_OK) + return ZE_MEM; + } + } + +if (Options.fUpdate) /* Update zip file--overwrite only if newer -u */ + { + if (AllocMemory(argCee, "-u", "Update", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fVerbose) /* Mention oddities in zip file structure -v */ + { + if (AllocMemory(argCee, "-v", "Verbose", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.fVolume) /* Include volume label -$ */ + { + if (AllocMemory(argCee, "-$", "Volume", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.szSplitSize != NULL) /* Turn on archive splitting */ + { + if (AllocMemory(argCee, "-s", "Splitting", FALSE) != ZE_OK) + return ZE_MEM; + if (AllocMemory(argCee, Options.szSplitSize, "Split size", FALSE) != ZE_OK) + return ZE_MEM; + } +if (lpZipUserFunctions->split != NULL) /* Turn on archive split destinations select */ + { + if (AllocMemory(argCee, "-sp", "Split Pause Select Destination", FALSE) != ZE_OK) + return ZE_MEM; + } +#ifdef WIN32 +if (Options.fPrivilege) /* Use privileges -! */ + { + if (AllocMemory(argCee, "-!", "Privileges", FALSE) != ZE_OK) + return ZE_MEM; + } +#endif +if (Options.fExtra) /* Exclude extra attributes -X */ + { + if (AllocMemory(argCee, "-X", "Extra", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.IncludeList != NULL) /* Include file list -i */ + { + if (AllocMemory(argCee, "-i", "Include file list", FALSE) != ZE_OK) + return ZE_MEM; + k = 0; + if (Options.IncludeListCount > 0) + while ((Options.IncludeList[k] != NULL) && (Options.IncludeListCount != k+1)) + { + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + if (AllocMemory(argCee, Options.IncludeList[k], "Include file list array", TRUE) != ZE_OK) + { + return ZE_MEM; + } + k++; + } + else + while (Options.IncludeList[k] != NULL) + { + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + FreeArgVee(); + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + if (AllocMemory(argCee, Options.IncludeList[k], "Include file list array", TRUE) != ZE_OK) + return ZE_MEM; + k++; + } + + if (AllocMemory(argCee, "@", "End of Include List", FALSE) != ZE_OK) + return ZE_MEM; + } +if (Options.ExcludeList != NULL) /* Exclude file list -x */ + { + if (AllocMemory(argCee, "-x", "Exclude file list", FALSE) != ZE_OK) + return ZE_MEM; + k = 0; + if (Options.ExcludeListCount > 0) + while ((Options.ExcludeList[k] != NULL) && (Options.ExcludeListCount != k+1)) + { + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + if (AllocMemory(argCee, Options.ExcludeList[k], "Exclude file list array", TRUE) != ZE_OK) + return ZE_MEM; + k++; + } + else + while (Options.ExcludeList[k] != NULL) + { + size = _msize(argVee); + if ((argVee = (char **)realloc(argVee, size + sizeof(char *))) == NULL) + { + FreeArgVee(); + fprintf(stdout, "Unable to allocate memory in zip dll\n"); + return ZE_MEM; + } + if (AllocMemory(argCee, Options.ExcludeList[k], "Exclude file list array", TRUE) != ZE_OK) + return ZE_MEM; + k++; + } + if (AllocMemory(argCee, "@", "End of Exclude List", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (szIncludeList != NULL && szIncludeList[0] != '\0') /* Include file list -i */ + { + if (AllocMemory(argCee, "-i", "Include file list", FALSE) != ZE_OK) + return ZE_MEM; + if ((k = ParseString(szIncludeList, argCee)) != ZE_OK) + return k; /* Something was screwy with the parsed string + bail out */ + if (AllocMemory(argCee, "@", "End of Include List", FALSE) != ZE_OK) + return ZE_MEM; + } +if (szExcludeList != NULL && szExcludeList[0] != '\0') /* Exclude file list -x */ + { + if (AllocMemory(argCee, "-x", "Exclude file list", FALSE) != ZE_OK) + return ZE_MEM; + + if ((k = ParseString(szExcludeList, argCee)) != ZE_OK) + return k; /* Something was screwy with the parsed string + bail out */ + + if (AllocMemory(argCee, "@", "End of Exclude List", FALSE) != ZE_OK) + return ZE_MEM; + } + +if ((szTempDir != NULL) && (szTempDir[0] != '\0') + && Options.fTemp) /* Use temporary directory -b */ + { + if (AllocMemory(argCee, "-b", "Temp dir switch command", FALSE) != ZE_OK) + return ZE_MEM; + if (AllocMemory(argCee, szTempDir, "Temporary directory", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (AllocMemory(argCee, C.lpszZipFN, "Zip file name", FALSE) != ZE_OK) + return ZE_MEM; + +if ((szRootDir != NULL) && (szRootDir[0] != '\0')) + { + if (szRootDir[lstrlen(szRootDir)-1] != '\\') + lstrcat(szRootDir, "\\"); /* append trailing \\ */ + if (C.FNV != NULL) + { + for (k = 0; k < C.argc; k++) + { + if (AllocMemory(argCee, C.FNV[k], "Making argv", FALSE) != ZE_OK) + return ZE_MEM; + if ((_strnicmp(szRootDir, C.FNV[k], lstrlen(szRootDir))) == 0) + { + m = 0; + for (j = lstrlen(szRootDir); j < lstrlen(C.FNV[k]); j++) + argVee[argCee-1][m++] = C.FNV[k][j]; + argVee[argCee-1][m] = '\0'; + } + } + } + + } +else + if (C.FNV != NULL) + for (k = 0; k < C.argc; k++) + { + if (AllocMemory(argCee, C.FNV[k], "Making argv", FALSE) != ZE_OK) + return ZE_MEM; + } + +if (C.lpszAltFNL != NULL) + { + if ((k = ParseString(C.lpszAltFNL, argCee)) != ZE_OK) + return k; /* Something was screwy with the parsed string + bail out + */ + } + + + +argVee[argCee] = NULL; + +ZipRet = zipmain(argCee, argVee); + +/* Free the arguments in the array. Note this also restores the + current directory + */ +FreeArgVee(); + +return ZipRet; +} + +#if CRYPT +int encr_passwd(int modeflag, char *pwbuf, int size, const char *zfn) + { + return (*lpZipUserFunctions->password)(pwbuf, size, ((modeflag == ZP_PW_VERIFY) ? + "Verify password: " : "Enter password: "), + (char *)zfn); + } +#endif /* CRYPT */ + +void EXPENTRY ZpVersion(ZpVer far * p) /* should be pointer to const struct */ + { + p->structlen = ZPVER_LEN; + +#ifdef BETA + p->flag = 1; +#else + p->flag = 0; +#endif +#ifdef CRYPT + p->fEncryption = TRUE; +#else + p->fEncryption = FALSE; +#endif + lstrcpy(p->betalevel, Z_BETALEVEL); + lstrcpy(p->date, REVDATE); + +#ifdef ZLIB_VERSION + lstrcpy(p->zlib_version, ZLIB_VERSION); + p->flag |= 2; +#else + p->zlib_version[0] = '\0'; +#endif + +#ifdef ZIP64_SUPPORT + p->flag |= 4; /* Flag that ZIP64 was compiled in. */ +#endif + + p->zip.major = Z_MAJORVER; + p->zip.minor = Z_MINORVER; + p->zip.patchlevel = Z_PATCHLEVEL; + +#ifdef OS2 + p->os2dll.major = D2_MAJORVER; + p->os2dll.minor = D2_MINORVER; + p->os2dll.patchlevel = D2_PATCHLEVEL; +#endif +#ifdef WINDLL + p->windll.major = DW_MAJORVER; + p->windll.minor = DW_MINORVER; + p->windll.patchlevel = DW_PATCHLEVEL; +#endif + } diff --git a/third_party/infozip/zip/api.h b/third_party/infozip/zip/api.h new file mode 100644 index 000000000..fccd4b80f --- /dev/null +++ b/third_party/infozip/zip/api.h @@ -0,0 +1,185 @@ +/* clang-format off */ +/* + api.h - Zip 3 + + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* Only the Windows DLL is currently supported */ +#ifndef _ZIPAPI_H +#define _ZIPAPI_H + +#include "third_party/infozip/zip/zip.h" + +#ifdef WIN32 +# ifndef PATH_MAX +# define PATH_MAX 260 +# endif +#else +# ifndef PATH_MAX +# define PATH_MAX 128 +# endif +#endif + +#if defined(WINDLL) || defined(API) +#include +/* Porting definations between Win 3.1x and Win32 */ +#ifdef WIN32 +# define far +# define _far +# define __far +# define near +# define _near +# define __near +#endif + +/*--------------------------------------------------------------------------- + Prototypes for public Zip API (DLL) functions. + ---------------------------------------------------------------------------*/ + +#define ZPVER_LEN sizeof(ZpVer) +/* These defines are set to zero for now, until OS/2 comes out + with a dll. + */ +#define D2_MAJORVER 0 +#define D2_MINORVER 0 +#define D2_PATCHLEVEL 0 + +/* intended to be a private struct: */ +typedef struct _zip_ver { + uch major; /* e.g., integer 5 */ + uch minor; /* e.g., 2 */ + uch patchlevel; /* e.g., 0 */ + uch not_used; +} _zip_version_type; + +typedef struct _ZpVer { + ulg structlen; /* length of the struct being passed */ + ulg flag; /* bit 0: is_beta bit 1: uses_zlib */ + char betalevel[10]; /* e.g., "g BETA" or "" */ + char date[20]; /* e.g., "4 Sep 95" (beta) or "4 September 1995" */ + char zlib_version[10]; /* e.g., "0.95" or NULL */ + BOOL fEncryption; /* TRUE if encryption enabled, FALSE otherwise */ + _zip_version_type zip; + _zip_version_type os2dll; + _zip_version_type windll; +} ZpVer; + +# ifndef EXPENTRY +# define EXPENTRY WINAPI +# endif + +#ifndef DEFINED_ONCE +#define DEFINED_ONCE +typedef int (WINAPI DLLPRNT) (LPSTR, unsigned long); +typedef int (WINAPI DLLPASSWORD) (LPSTR, int, LPCSTR, LPCSTR); +#endif +#ifdef ZIP64_SUPPORT +typedef int (WINAPI DLLSERVICE) (LPCSTR, unsigned __int64); +typedef int (WINAPI DLLSERVICE_NO_INT64) (LPCSTR, unsigned long, unsigned long); +#else +typedef int (WINAPI DLLSERVICE) (LPCSTR, unsigned long); +#endif +typedef int (WINAPI DLLSPLIT) (LPSTR); +typedef int (WINAPI DLLCOMMENT)(LPSTR); + +/* Structures */ + +typedef struct { /* zip options */ +LPSTR Date; /* Date to include after */ +LPSTR szRootDir; /* Directory to use as base for zipping */ +LPSTR szTempDir; /* Temporary directory used during zipping */ +BOOL fTemp; /* Use temporary directory '-b' during zipping */ +BOOL fSuffix; /* include suffixes (not implemented) */ +BOOL fEncrypt; /* encrypt files */ +BOOL fSystem; /* include system and hidden files */ +BOOL fVolume; /* Include volume label */ +BOOL fExtra; /* Exclude extra attributes */ +BOOL fNoDirEntries; /* Do not add directory entries */ +BOOL fExcludeDate; /* Exclude files newer than specified date */ +BOOL fIncludeDate; /* Include only files newer than specified date */ +BOOL fVerbose; /* Mention oddities in zip file structure */ +BOOL fQuiet; /* Quiet operation */ +BOOL fCRLF_LF; /* Translate CR/LF to LF */ +BOOL fLF_CRLF; /* Translate LF to CR/LF */ +BOOL fJunkDir; /* Junk directory names */ +BOOL fGrow; /* Allow appending to a zip file */ +BOOL fForce; /* Make entries using DOS names (k for Katz) */ +BOOL fMove; /* Delete files added or updated in zip file */ +BOOL fDeleteEntries; /* Delete files from zip file */ +BOOL fUpdate; /* Update zip file--overwrite only if newer */ +BOOL fFreshen; /* Freshen zip file--overwrite only */ +BOOL fJunkSFX; /* Junk SFX prefix */ +BOOL fLatestTime; /* Set zip file time to time of latest file in it */ +BOOL fComment; /* Put comment in zip file */ +BOOL fOffsets; /* Update archive offsets for SFX files */ +BOOL fPrivilege; /* Use privileges (WIN32 only) */ +BOOL fEncryption; /* TRUE if encryption supported, else FALSE. + this is a read only flag */ +LPSTR szSplitSize; /* This string contains the size that you want to + split the archive into. i.e. 100 for 100 bytes, + 2K for 2 k bytes, where K is 1024, m for meg + and g for gig. If this string is not NULL it + will automatically be assumed that you wish to + split an archive. */ +LPSTR szIncludeList; /* Pointer to include file list string (for VB) */ +long IncludeListCount; /* Count of file names in the include list array */ +char **IncludeList; /* Pointer to include file list array. Note that the last + entry in the array must be NULL */ +LPSTR szExcludeList; /* Pointer to exclude file list (for VB) */ +long ExcludeListCount; /* Count of file names in the include list array */ +char **ExcludeList; /* Pointer to exclude file list array. Note that the last + entry in the array must be NULL */ +int fRecurse; /* Recurse into subdirectories. 1 => -r, 2 => -R */ +int fRepair; /* Repair archive. 1 => -F, 2 => -FF */ +char fLevel; /* Compression level (0 - 9) */ +} ZPOPT, _far *LPZPOPT; + +typedef struct { + int argc; /* Count of files to zip */ + LPSTR lpszZipFN; /* name of archive to create/update */ + char **FNV; /* array of file names to zip up */ + LPSTR lpszAltFNL; /* pointer to a string containing a list of file + names to zip up, separated by whitespace. Intended + for use only by VB users, all others should set this + to NULL. */ +} ZCL, _far *LPZCL; + +typedef struct { + DLLPRNT *print; + DLLCOMMENT *comment; + DLLPASSWORD *password; + DLLSPLIT *split; /* This MUST be set to NULL unless you want to be queried + for a destination for each split archive. */ +#ifdef ZIP64_SUPPORT + DLLSERVICE *ServiceApplication64; + DLLSERVICE_NO_INT64 *ServiceApplication64_No_Int64; +#else + DLLSERVICE *ServiceApplication; +#endif +} ZIPUSERFUNCTIONS, far * LPZIPUSERFUNCTIONS; + +extern LPZIPUSERFUNCTIONS lpZipUserFunctions; + +void EXPENTRY ZpVersion(ZpVer far *); +int EXPENTRY ZpInit(LPZIPUSERFUNCTIONS lpZipUserFunc); +int EXPENTRY ZpArchive(ZCL C, LPZPOPT Opts); + +#if defined(ZIPLIB) || defined(COM_OBJECT) +# define ydays zp_ydays +#endif + + + +/* Functions not yet supported */ +#if 0 +int EXPENTRY ZpMain (int argc, char **argv); +int EXPENTRY ZpAltMain (int argc, char **argv, ZpInit *init); +#endif +#endif /* WINDLL? || API? */ + +#endif /* _ZIPAPI_H */ diff --git a/third_party/infozip/zip/crc32.c b/third_party/infozip/zip/crc32.c new file mode 100644 index 000000000..ac1459f82 --- /dev/null +++ b/third_party/infozip/zip/crc32.c @@ -0,0 +1,733 @@ +/* clang-format off */ +/* + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* crc32.c -- compute the CRC-32 of a data stream + * Copyright (C) 1995 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + * + * Thanks to Rodney Brown for his contribution of faster + * CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing + * tables for updating the shift register in one step with three exclusive-ors + * instead of four steps with four exclusive-ors. This results about a factor + * of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3. + */ + +/* $Id: crc32.c,v 2.0 2007/01/07 05:20:36 spc Exp $ */ + +#define __CRC32_C /* identifies this source module */ + +#include "third_party/infozip/zip/zip.h" + +#if (!defined(USE_ZLIB) || defined(USE_OWN_CRCTAB)) + +#ifndef ZCONST +# define ZCONST const +#endif + +#include "third_party/infozip/zip/crc32.h" + +/* When only the table of precomputed CRC values is needed, only the basic + system-independent table containing 256 entries is created; any support + for "unfolding" optimization is disabled. + */ +#if (defined(USE_ZLIB) || defined(CRC_TABLE_ONLY)) +# ifdef IZ_CRCOPTIM_UNFOLDTBL +# undef IZ_CRCOPTIM_UNFOLDTBL +# endif +#endif /* (USE_ZLIB || CRC_TABLE_ONLY) */ + +#if defined(IZ_CRCOPTIM_UNFOLDTBL) +# define CRC_TBLS 4 +#else +# define CRC_TBLS 1 +#endif + + +/* + Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: + x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + + Polynomials over GF(2) are represented in binary, one bit per coefficient, + with the lowest powers in the most significant bit. Then adding polynomials + is just exclusive-or, and multiplying a polynomial by x is a right shift by + one. If we call the above polynomial p, and represent a byte as the + polynomial q, also with the lowest power in the most significant bit (so the + byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p, + where a mod b means the remainder after dividing a by b. + + This calculation is done using the shift-register method of multiplying and + taking the remainder. The register is initialized to zero, and for each + incoming bit, x^32 is added mod p to the register if the bit is a one (where + x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by + x (which is shifting right by one and adding x^32 mod p if the bit shifted + out is a one). We start with the highest power (least significant bit) of + q and repeat for all eight bits of q. + + The first (or only) table is simply the CRC of all possible eight bit values. + This is all the information needed to generate CRC's on data a byte-at-a-time + for all combinations of CRC register values and incoming bytes. + The remaining 3 tables (if IZ_CRCOPTIM_UNFOLDTBL is enabled) allow for + word-at-a-time CRC calculation, where a word is four bytes. +*/ + +#ifdef DYNAMIC_CRC_TABLE + +/* ========================================================================= + * Make the crc table. This function is needed only if you want to compute + * the table dynamically. + */ + +local void make_crc_table OF((void)); + +#if (defined(DYNALLOC_CRCTAB) && defined(REENTRANT)) + error: Dynamic allocation of CRC table not safe with reentrant code. +#endif /* DYNALLOC_CRCTAB && REENTRANT */ + +#ifdef DYNALLOC_CRCTAB + local ulg near *crc_table = NULL; +# if 0 /* not used, since sizeof("near *") <= sizeof(int) */ + /* Use this section when access to a "local int" is faster than access to + a "local pointer" (e.g.: i86 16bit code with far pointers). */ + local int crc_table_empty = 1; +# define CRC_TABLE_IS_EMPTY (crc_table_empty != 0) +# define MARK_CRCTAB_FILLED crc_table_empty = 0 +# define MARK_CRCTAB_EMPTY crc_table_empty = 1 +# else + /* Use this section on systems where the size of pointers and ints is + equal (e.g.: all 32bit systems). */ +# define CRC_TABLE_IS_EMPTY (crc_table == NULL) +# define MARK_CRCTAB_FILLED crc_table = crctab_p +# define MARK_CRCTAB_EMPTY crc_table = NULL +# endif +#else /* !DYNALLOC_CRCTAB */ + local ulg near crc_table[CRC_TBLS*256]; + local int crc_table_empty = 1; +# define CRC_TABLE_IS_EMPTY (crc_table_empty != 0) +# define MARK_CRCTAB_FILLED crc_table_empty = 0 +#endif /* ?DYNALLOC_CRCTAB */ + + +local void make_crc_table() +{ + ulg c; /* crc shift register */ + int n; /* counter for all possible eight bit values */ + int k; /* byte being shifted into crc apparatus */ +#ifdef DYNALLOC_CRCTAB + ulg near *crctab_p; /* temporary pointer to allocated crc_table area */ +#else /* !DYNALLOC_CRCTAB */ +# define crctab_p crc_table +#endif /* DYNALLOC_CRCTAB */ + +#ifdef COMPUTE_XOR_PATTERN + /* This piece of code has been left here to explain how the XOR pattern + * used in the creation of the crc_table values can be recomputed. + * For production versions of this function, it is more efficient to + * supply the resultant pattern at compile time. + */ + ulg xor; /* polynomial exclusive-or pattern */ + /* terms of polynomial defining this crc (except x^32): */ + static ZCONST uch p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; + + /* make exclusive-or pattern from polynomial (0xedb88320L) */ + xor = 0L; + for (n = 0; n < sizeof(p)/sizeof(uch); n++) + xor |= 1L << (31 - p[n]); +#else +# define xor 0xedb88320L +#endif + +#ifdef DYNALLOC_CRCTAB + crctab_p = (ulg near *) nearmalloc (CRC_TBLS*256*sizeof(ulg)); + if (crctab_p == NULL) { + ziperr(ZE_MEM, "crc_table allocation"); + } +#endif /* DYNALLOC_CRCTAB */ + + /* generate a crc for every 8-bit value */ + for (n = 0; n < 256; n++) { + c = (ulg)n; + for (k = 8; k; k--) + c = c & 1 ? xor ^ (c >> 1) : c >> 1; + crctab_p[n] = REV_BE(c); + } + +#ifdef IZ_CRCOPTIM_UNFOLDTBL + /* generate crc for each value followed by one, two, and three zeros */ + for (n = 0; n < 256; n++) { + c = crctab_p[n]; + for (k = 1; k < 4; k++) { + c = CRC32(c, 0, crctab_p); + crctab_p[k*256+n] = c; + } + } +#endif /* IZ_CRCOPTIM_UNFOLDTBL */ + + MARK_CRCTAB_FILLED; +} + +#else /* !DYNAMIC_CRC_TABLE */ + +#ifdef DYNALLOC_CRCTAB + error: Inconsistent flags, DYNALLOC_CRCTAB without DYNAMIC_CRC_TABLE. +#endif + +/* ======================================================================== + * Table of CRC-32's of all single-byte values (made by make_crc_table) + */ +local ZCONST ulg near crc_table[CRC_TBLS*256] = { +# ifdef IZ_CRC_BE_OPTIMIZ + 0x00000000L, 0x96300777L, 0x2c610eeeL, 0xba510999L, 0x19c46d07L, + 0x8ff46a70L, 0x35a563e9L, 0xa395649eL, 0x3288db0eL, 0xa4b8dc79L, + 0x1ee9d5e0L, 0x88d9d297L, 0x2b4cb609L, 0xbd7cb17eL, 0x072db8e7L, + 0x911dbf90L, 0x6410b71dL, 0xf220b06aL, 0x4871b9f3L, 0xde41be84L, + 0x7dd4da1aL, 0xebe4dd6dL, 0x51b5d4f4L, 0xc785d383L, 0x56986c13L, + 0xc0a86b64L, 0x7af962fdL, 0xecc9658aL, 0x4f5c0114L, 0xd96c0663L, + 0x633d0ffaL, 0xf50d088dL, 0xc8206e3bL, 0x5e10694cL, 0xe44160d5L, + 0x727167a2L, 0xd1e4033cL, 0x47d4044bL, 0xfd850dd2L, 0x6bb50aa5L, + 0xfaa8b535L, 0x6c98b242L, 0xd6c9bbdbL, 0x40f9bcacL, 0xe36cd832L, + 0x755cdf45L, 0xcf0dd6dcL, 0x593dd1abL, 0xac30d926L, 0x3a00de51L, + 0x8051d7c8L, 0x1661d0bfL, 0xb5f4b421L, 0x23c4b356L, 0x9995bacfL, + 0x0fa5bdb8L, 0x9eb80228L, 0x0888055fL, 0xb2d90cc6L, 0x24e90bb1L, + 0x877c6f2fL, 0x114c6858L, 0xab1d61c1L, 0x3d2d66b6L, 0x9041dc76L, + 0x0671db01L, 0xbc20d298L, 0x2a10d5efL, 0x8985b171L, 0x1fb5b606L, + 0xa5e4bf9fL, 0x33d4b8e8L, 0xa2c90778L, 0x34f9000fL, 0x8ea80996L, + 0x18980ee1L, 0xbb0d6a7fL, 0x2d3d6d08L, 0x976c6491L, 0x015c63e6L, + 0xf4516b6bL, 0x62616c1cL, 0xd8306585L, 0x4e0062f2L, 0xed95066cL, + 0x7ba5011bL, 0xc1f40882L, 0x57c40ff5L, 0xc6d9b065L, 0x50e9b712L, + 0xeab8be8bL, 0x7c88b9fcL, 0xdf1ddd62L, 0x492dda15L, 0xf37cd38cL, + 0x654cd4fbL, 0x5861b24dL, 0xce51b53aL, 0x7400bca3L, 0xe230bbd4L, + 0x41a5df4aL, 0xd795d83dL, 0x6dc4d1a4L, 0xfbf4d6d3L, 0x6ae96943L, + 0xfcd96e34L, 0x468867adL, 0xd0b860daL, 0x732d0444L, 0xe51d0333L, + 0x5f4c0aaaL, 0xc97c0dddL, 0x3c710550L, 0xaa410227L, 0x10100bbeL, + 0x86200cc9L, 0x25b56857L, 0xb3856f20L, 0x09d466b9L, 0x9fe461ceL, + 0x0ef9de5eL, 0x98c9d929L, 0x2298d0b0L, 0xb4a8d7c7L, 0x173db359L, + 0x810db42eL, 0x3b5cbdb7L, 0xad6cbac0L, 0x2083b8edL, 0xb6b3bf9aL, + 0x0ce2b603L, 0x9ad2b174L, 0x3947d5eaL, 0xaf77d29dL, 0x1526db04L, + 0x8316dc73L, 0x120b63e3L, 0x843b6494L, 0x3e6a6d0dL, 0xa85a6a7aL, + 0x0bcf0ee4L, 0x9dff0993L, 0x27ae000aL, 0xb19e077dL, 0x44930ff0L, + 0xd2a30887L, 0x68f2011eL, 0xfec20669L, 0x5d5762f7L, 0xcb676580L, + 0x71366c19L, 0xe7066b6eL, 0x761bd4feL, 0xe02bd389L, 0x5a7ada10L, + 0xcc4add67L, 0x6fdfb9f9L, 0xf9efbe8eL, 0x43beb717L, 0xd58eb060L, + 0xe8a3d6d6L, 0x7e93d1a1L, 0xc4c2d838L, 0x52f2df4fL, 0xf167bbd1L, + 0x6757bca6L, 0xdd06b53fL, 0x4b36b248L, 0xda2b0dd8L, 0x4c1b0aafL, + 0xf64a0336L, 0x607a0441L, 0xc3ef60dfL, 0x55df67a8L, 0xef8e6e31L, + 0x79be6946L, 0x8cb361cbL, 0x1a8366bcL, 0xa0d26f25L, 0x36e26852L, + 0x95770cccL, 0x03470bbbL, 0xb9160222L, 0x2f260555L, 0xbe3bbac5L, + 0x280bbdb2L, 0x925ab42bL, 0x046ab35cL, 0xa7ffd7c2L, 0x31cfd0b5L, + 0x8b9ed92cL, 0x1daede5bL, 0xb0c2649bL, 0x26f263ecL, 0x9ca36a75L, + 0x0a936d02L, 0xa906099cL, 0x3f360eebL, 0x85670772L, 0x13570005L, + 0x824abf95L, 0x147ab8e2L, 0xae2bb17bL, 0x381bb60cL, 0x9b8ed292L, + 0x0dbed5e5L, 0xb7efdc7cL, 0x21dfdb0bL, 0xd4d2d386L, 0x42e2d4f1L, + 0xf8b3dd68L, 0x6e83da1fL, 0xcd16be81L, 0x5b26b9f6L, 0xe177b06fL, + 0x7747b718L, 0xe65a0888L, 0x706a0fffL, 0xca3b0666L, 0x5c0b0111L, + 0xff9e658fL, 0x69ae62f8L, 0xd3ff6b61L, 0x45cf6c16L, 0x78e20aa0L, + 0xeed20dd7L, 0x5483044eL, 0xc2b30339L, 0x612667a7L, 0xf71660d0L, + 0x4d476949L, 0xdb776e3eL, 0x4a6ad1aeL, 0xdc5ad6d9L, 0x660bdf40L, + 0xf03bd837L, 0x53aebca9L, 0xc59ebbdeL, 0x7fcfb247L, 0xe9ffb530L, + 0x1cf2bdbdL, 0x8ac2bacaL, 0x3093b353L, 0xa6a3b424L, 0x0536d0baL, + 0x9306d7cdL, 0x2957de54L, 0xbf67d923L, 0x2e7a66b3L, 0xb84a61c4L, + 0x021b685dL, 0x942b6f2aL, 0x37be0bb4L, 0xa18e0cc3L, 0x1bdf055aL, + 0x8def022dL +# ifdef IZ_CRCOPTIM_UNFOLDTBL + , + 0x00000000L, 0x41311b19L, 0x82623632L, 0xc3532d2bL, 0x04c56c64L, + 0x45f4777dL, 0x86a75a56L, 0xc796414fL, 0x088ad9c8L, 0x49bbc2d1L, + 0x8ae8effaL, 0xcbd9f4e3L, 0x0c4fb5acL, 0x4d7eaeb5L, 0x8e2d839eL, + 0xcf1c9887L, 0x5112c24aL, 0x1023d953L, 0xd370f478L, 0x9241ef61L, + 0x55d7ae2eL, 0x14e6b537L, 0xd7b5981cL, 0x96848305L, 0x59981b82L, + 0x18a9009bL, 0xdbfa2db0L, 0x9acb36a9L, 0x5d5d77e6L, 0x1c6c6cffL, + 0xdf3f41d4L, 0x9e0e5acdL, 0xa2248495L, 0xe3159f8cL, 0x2046b2a7L, + 0x6177a9beL, 0xa6e1e8f1L, 0xe7d0f3e8L, 0x2483dec3L, 0x65b2c5daL, + 0xaaae5d5dL, 0xeb9f4644L, 0x28cc6b6fL, 0x69fd7076L, 0xae6b3139L, + 0xef5a2a20L, 0x2c09070bL, 0x6d381c12L, 0xf33646dfL, 0xb2075dc6L, + 0x715470edL, 0x30656bf4L, 0xf7f32abbL, 0xb6c231a2L, 0x75911c89L, + 0x34a00790L, 0xfbbc9f17L, 0xba8d840eL, 0x79dea925L, 0x38efb23cL, + 0xff79f373L, 0xbe48e86aL, 0x7d1bc541L, 0x3c2ade58L, 0x054f79f0L, + 0x447e62e9L, 0x872d4fc2L, 0xc61c54dbL, 0x018a1594L, 0x40bb0e8dL, + 0x83e823a6L, 0xc2d938bfL, 0x0dc5a038L, 0x4cf4bb21L, 0x8fa7960aL, + 0xce968d13L, 0x0900cc5cL, 0x4831d745L, 0x8b62fa6eL, 0xca53e177L, + 0x545dbbbaL, 0x156ca0a3L, 0xd63f8d88L, 0x970e9691L, 0x5098d7deL, + 0x11a9ccc7L, 0xd2fae1ecL, 0x93cbfaf5L, 0x5cd76272L, 0x1de6796bL, + 0xdeb55440L, 0x9f844f59L, 0x58120e16L, 0x1923150fL, 0xda703824L, + 0x9b41233dL, 0xa76bfd65L, 0xe65ae67cL, 0x2509cb57L, 0x6438d04eL, + 0xa3ae9101L, 0xe29f8a18L, 0x21cca733L, 0x60fdbc2aL, 0xafe124adL, + 0xeed03fb4L, 0x2d83129fL, 0x6cb20986L, 0xab2448c9L, 0xea1553d0L, + 0x29467efbL, 0x687765e2L, 0xf6793f2fL, 0xb7482436L, 0x741b091dL, + 0x352a1204L, 0xf2bc534bL, 0xb38d4852L, 0x70de6579L, 0x31ef7e60L, + 0xfef3e6e7L, 0xbfc2fdfeL, 0x7c91d0d5L, 0x3da0cbccL, 0xfa368a83L, + 0xbb07919aL, 0x7854bcb1L, 0x3965a7a8L, 0x4b98833bL, 0x0aa99822L, + 0xc9fab509L, 0x88cbae10L, 0x4f5def5fL, 0x0e6cf446L, 0xcd3fd96dL, + 0x8c0ec274L, 0x43125af3L, 0x022341eaL, 0xc1706cc1L, 0x804177d8L, + 0x47d73697L, 0x06e62d8eL, 0xc5b500a5L, 0x84841bbcL, 0x1a8a4171L, + 0x5bbb5a68L, 0x98e87743L, 0xd9d96c5aL, 0x1e4f2d15L, 0x5f7e360cL, + 0x9c2d1b27L, 0xdd1c003eL, 0x120098b9L, 0x533183a0L, 0x9062ae8bL, + 0xd153b592L, 0x16c5f4ddL, 0x57f4efc4L, 0x94a7c2efL, 0xd596d9f6L, + 0xe9bc07aeL, 0xa88d1cb7L, 0x6bde319cL, 0x2aef2a85L, 0xed796bcaL, + 0xac4870d3L, 0x6f1b5df8L, 0x2e2a46e1L, 0xe136de66L, 0xa007c57fL, + 0x6354e854L, 0x2265f34dL, 0xe5f3b202L, 0xa4c2a91bL, 0x67918430L, + 0x26a09f29L, 0xb8aec5e4L, 0xf99fdefdL, 0x3accf3d6L, 0x7bfde8cfL, + 0xbc6ba980L, 0xfd5ab299L, 0x3e099fb2L, 0x7f3884abL, 0xb0241c2cL, + 0xf1150735L, 0x32462a1eL, 0x73773107L, 0xb4e17048L, 0xf5d06b51L, + 0x3683467aL, 0x77b25d63L, 0x4ed7facbL, 0x0fe6e1d2L, 0xccb5ccf9L, + 0x8d84d7e0L, 0x4a1296afL, 0x0b238db6L, 0xc870a09dL, 0x8941bb84L, + 0x465d2303L, 0x076c381aL, 0xc43f1531L, 0x850e0e28L, 0x42984f67L, + 0x03a9547eL, 0xc0fa7955L, 0x81cb624cL, 0x1fc53881L, 0x5ef42398L, + 0x9da70eb3L, 0xdc9615aaL, 0x1b0054e5L, 0x5a314ffcL, 0x996262d7L, + 0xd85379ceL, 0x174fe149L, 0x567efa50L, 0x952dd77bL, 0xd41ccc62L, + 0x138a8d2dL, 0x52bb9634L, 0x91e8bb1fL, 0xd0d9a006L, 0xecf37e5eL, + 0xadc26547L, 0x6e91486cL, 0x2fa05375L, 0xe836123aL, 0xa9070923L, + 0x6a542408L, 0x2b653f11L, 0xe479a796L, 0xa548bc8fL, 0x661b91a4L, + 0x272a8abdL, 0xe0bccbf2L, 0xa18dd0ebL, 0x62defdc0L, 0x23efe6d9L, + 0xbde1bc14L, 0xfcd0a70dL, 0x3f838a26L, 0x7eb2913fL, 0xb924d070L, + 0xf815cb69L, 0x3b46e642L, 0x7a77fd5bL, 0xb56b65dcL, 0xf45a7ec5L, + 0x370953eeL, 0x763848f7L, 0xb1ae09b8L, 0xf09f12a1L, 0x33cc3f8aL, + 0x72fd2493L + , + 0x00000000L, 0x376ac201L, 0x6ed48403L, 0x59be4602L, 0xdca80907L, + 0xebc2cb06L, 0xb27c8d04L, 0x85164f05L, 0xb851130eL, 0x8f3bd10fL, + 0xd685970dL, 0xe1ef550cL, 0x64f91a09L, 0x5393d808L, 0x0a2d9e0aL, + 0x3d475c0bL, 0x70a3261cL, 0x47c9e41dL, 0x1e77a21fL, 0x291d601eL, + 0xac0b2f1bL, 0x9b61ed1aL, 0xc2dfab18L, 0xf5b56919L, 0xc8f23512L, + 0xff98f713L, 0xa626b111L, 0x914c7310L, 0x145a3c15L, 0x2330fe14L, + 0x7a8eb816L, 0x4de47a17L, 0xe0464d38L, 0xd72c8f39L, 0x8e92c93bL, + 0xb9f80b3aL, 0x3cee443fL, 0x0b84863eL, 0x523ac03cL, 0x6550023dL, + 0x58175e36L, 0x6f7d9c37L, 0x36c3da35L, 0x01a91834L, 0x84bf5731L, + 0xb3d59530L, 0xea6bd332L, 0xdd011133L, 0x90e56b24L, 0xa78fa925L, + 0xfe31ef27L, 0xc95b2d26L, 0x4c4d6223L, 0x7b27a022L, 0x2299e620L, + 0x15f32421L, 0x28b4782aL, 0x1fdeba2bL, 0x4660fc29L, 0x710a3e28L, + 0xf41c712dL, 0xc376b32cL, 0x9ac8f52eL, 0xada2372fL, 0xc08d9a70L, + 0xf7e75871L, 0xae591e73L, 0x9933dc72L, 0x1c259377L, 0x2b4f5176L, + 0x72f11774L, 0x459bd575L, 0x78dc897eL, 0x4fb64b7fL, 0x16080d7dL, + 0x2162cf7cL, 0xa4748079L, 0x931e4278L, 0xcaa0047aL, 0xfdcac67bL, + 0xb02ebc6cL, 0x87447e6dL, 0xdefa386fL, 0xe990fa6eL, 0x6c86b56bL, + 0x5bec776aL, 0x02523168L, 0x3538f369L, 0x087faf62L, 0x3f156d63L, + 0x66ab2b61L, 0x51c1e960L, 0xd4d7a665L, 0xe3bd6464L, 0xba032266L, + 0x8d69e067L, 0x20cbd748L, 0x17a11549L, 0x4e1f534bL, 0x7975914aL, + 0xfc63de4fL, 0xcb091c4eL, 0x92b75a4cL, 0xa5dd984dL, 0x989ac446L, + 0xaff00647L, 0xf64e4045L, 0xc1248244L, 0x4432cd41L, 0x73580f40L, + 0x2ae64942L, 0x1d8c8b43L, 0x5068f154L, 0x67023355L, 0x3ebc7557L, + 0x09d6b756L, 0x8cc0f853L, 0xbbaa3a52L, 0xe2147c50L, 0xd57ebe51L, + 0xe839e25aL, 0xdf53205bL, 0x86ed6659L, 0xb187a458L, 0x3491eb5dL, + 0x03fb295cL, 0x5a456f5eL, 0x6d2fad5fL, 0x801b35e1L, 0xb771f7e0L, + 0xeecfb1e2L, 0xd9a573e3L, 0x5cb33ce6L, 0x6bd9fee7L, 0x3267b8e5L, + 0x050d7ae4L, 0x384a26efL, 0x0f20e4eeL, 0x569ea2ecL, 0x61f460edL, + 0xe4e22fe8L, 0xd388ede9L, 0x8a36abebL, 0xbd5c69eaL, 0xf0b813fdL, + 0xc7d2d1fcL, 0x9e6c97feL, 0xa90655ffL, 0x2c101afaL, 0x1b7ad8fbL, + 0x42c49ef9L, 0x75ae5cf8L, 0x48e900f3L, 0x7f83c2f2L, 0x263d84f0L, + 0x115746f1L, 0x944109f4L, 0xa32bcbf5L, 0xfa958df7L, 0xcdff4ff6L, + 0x605d78d9L, 0x5737bad8L, 0x0e89fcdaL, 0x39e33edbL, 0xbcf571deL, + 0x8b9fb3dfL, 0xd221f5ddL, 0xe54b37dcL, 0xd80c6bd7L, 0xef66a9d6L, + 0xb6d8efd4L, 0x81b22dd5L, 0x04a462d0L, 0x33cea0d1L, 0x6a70e6d3L, + 0x5d1a24d2L, 0x10fe5ec5L, 0x27949cc4L, 0x7e2adac6L, 0x494018c7L, + 0xcc5657c2L, 0xfb3c95c3L, 0xa282d3c1L, 0x95e811c0L, 0xa8af4dcbL, + 0x9fc58fcaL, 0xc67bc9c8L, 0xf1110bc9L, 0x740744ccL, 0x436d86cdL, + 0x1ad3c0cfL, 0x2db902ceL, 0x4096af91L, 0x77fc6d90L, 0x2e422b92L, + 0x1928e993L, 0x9c3ea696L, 0xab546497L, 0xf2ea2295L, 0xc580e094L, + 0xf8c7bc9fL, 0xcfad7e9eL, 0x9613389cL, 0xa179fa9dL, 0x246fb598L, + 0x13057799L, 0x4abb319bL, 0x7dd1f39aL, 0x3035898dL, 0x075f4b8cL, + 0x5ee10d8eL, 0x698bcf8fL, 0xec9d808aL, 0xdbf7428bL, 0x82490489L, + 0xb523c688L, 0x88649a83L, 0xbf0e5882L, 0xe6b01e80L, 0xd1dadc81L, + 0x54cc9384L, 0x63a65185L, 0x3a181787L, 0x0d72d586L, 0xa0d0e2a9L, + 0x97ba20a8L, 0xce0466aaL, 0xf96ea4abL, 0x7c78ebaeL, 0x4b1229afL, + 0x12ac6fadL, 0x25c6adacL, 0x1881f1a7L, 0x2feb33a6L, 0x765575a4L, + 0x413fb7a5L, 0xc429f8a0L, 0xf3433aa1L, 0xaafd7ca3L, 0x9d97bea2L, + 0xd073c4b5L, 0xe71906b4L, 0xbea740b6L, 0x89cd82b7L, 0x0cdbcdb2L, + 0x3bb10fb3L, 0x620f49b1L, 0x55658bb0L, 0x6822d7bbL, 0x5f4815baL, + 0x06f653b8L, 0x319c91b9L, 0xb48adebcL, 0x83e01cbdL, 0xda5e5abfL, + 0xed3498beL + , + 0x00000000L, 0x6567bcb8L, 0x8bc809aaL, 0xeeafb512L, 0x5797628fL, + 0x32f0de37L, 0xdc5f6b25L, 0xb938d79dL, 0xef28b4c5L, 0x8a4f087dL, + 0x64e0bd6fL, 0x018701d7L, 0xb8bfd64aL, 0xddd86af2L, 0x3377dfe0L, + 0x56106358L, 0x9f571950L, 0xfa30a5e8L, 0x149f10faL, 0x71f8ac42L, + 0xc8c07bdfL, 0xada7c767L, 0x43087275L, 0x266fcecdL, 0x707fad95L, + 0x1518112dL, 0xfbb7a43fL, 0x9ed01887L, 0x27e8cf1aL, 0x428f73a2L, + 0xac20c6b0L, 0xc9477a08L, 0x3eaf32a0L, 0x5bc88e18L, 0xb5673b0aL, + 0xd00087b2L, 0x6938502fL, 0x0c5fec97L, 0xe2f05985L, 0x8797e53dL, + 0xd1878665L, 0xb4e03addL, 0x5a4f8fcfL, 0x3f283377L, 0x8610e4eaL, + 0xe3775852L, 0x0dd8ed40L, 0x68bf51f8L, 0xa1f82bf0L, 0xc49f9748L, + 0x2a30225aL, 0x4f579ee2L, 0xf66f497fL, 0x9308f5c7L, 0x7da740d5L, + 0x18c0fc6dL, 0x4ed09f35L, 0x2bb7238dL, 0xc518969fL, 0xa07f2a27L, + 0x1947fdbaL, 0x7c204102L, 0x928ff410L, 0xf7e848a8L, 0x3d58149bL, + 0x583fa823L, 0xb6901d31L, 0xd3f7a189L, 0x6acf7614L, 0x0fa8caacL, + 0xe1077fbeL, 0x8460c306L, 0xd270a05eL, 0xb7171ce6L, 0x59b8a9f4L, + 0x3cdf154cL, 0x85e7c2d1L, 0xe0807e69L, 0x0e2fcb7bL, 0x6b4877c3L, + 0xa20f0dcbL, 0xc768b173L, 0x29c70461L, 0x4ca0b8d9L, 0xf5986f44L, + 0x90ffd3fcL, 0x7e5066eeL, 0x1b37da56L, 0x4d27b90eL, 0x284005b6L, + 0xc6efb0a4L, 0xa3880c1cL, 0x1ab0db81L, 0x7fd76739L, 0x9178d22bL, + 0xf41f6e93L, 0x03f7263bL, 0x66909a83L, 0x883f2f91L, 0xed589329L, + 0x546044b4L, 0x3107f80cL, 0xdfa84d1eL, 0xbacff1a6L, 0xecdf92feL, + 0x89b82e46L, 0x67179b54L, 0x027027ecL, 0xbb48f071L, 0xde2f4cc9L, + 0x3080f9dbL, 0x55e74563L, 0x9ca03f6bL, 0xf9c783d3L, 0x176836c1L, + 0x720f8a79L, 0xcb375de4L, 0xae50e15cL, 0x40ff544eL, 0x2598e8f6L, + 0x73888baeL, 0x16ef3716L, 0xf8408204L, 0x9d273ebcL, 0x241fe921L, + 0x41785599L, 0xafd7e08bL, 0xcab05c33L, 0x3bb659edL, 0x5ed1e555L, + 0xb07e5047L, 0xd519ecffL, 0x6c213b62L, 0x094687daL, 0xe7e932c8L, + 0x828e8e70L, 0xd49eed28L, 0xb1f95190L, 0x5f56e482L, 0x3a31583aL, + 0x83098fa7L, 0xe66e331fL, 0x08c1860dL, 0x6da63ab5L, 0xa4e140bdL, + 0xc186fc05L, 0x2f294917L, 0x4a4ef5afL, 0xf3762232L, 0x96119e8aL, + 0x78be2b98L, 0x1dd99720L, 0x4bc9f478L, 0x2eae48c0L, 0xc001fdd2L, + 0xa566416aL, 0x1c5e96f7L, 0x79392a4fL, 0x97969f5dL, 0xf2f123e5L, + 0x05196b4dL, 0x607ed7f5L, 0x8ed162e7L, 0xebb6de5fL, 0x528e09c2L, + 0x37e9b57aL, 0xd9460068L, 0xbc21bcd0L, 0xea31df88L, 0x8f566330L, + 0x61f9d622L, 0x049e6a9aL, 0xbda6bd07L, 0xd8c101bfL, 0x366eb4adL, + 0x53090815L, 0x9a4e721dL, 0xff29cea5L, 0x11867bb7L, 0x74e1c70fL, + 0xcdd91092L, 0xa8beac2aL, 0x46111938L, 0x2376a580L, 0x7566c6d8L, + 0x10017a60L, 0xfeaecf72L, 0x9bc973caL, 0x22f1a457L, 0x479618efL, + 0xa939adfdL, 0xcc5e1145L, 0x06ee4d76L, 0x6389f1ceL, 0x8d2644dcL, + 0xe841f864L, 0x51792ff9L, 0x341e9341L, 0xdab12653L, 0xbfd69aebL, + 0xe9c6f9b3L, 0x8ca1450bL, 0x620ef019L, 0x07694ca1L, 0xbe519b3cL, + 0xdb362784L, 0x35999296L, 0x50fe2e2eL, 0x99b95426L, 0xfcdee89eL, + 0x12715d8cL, 0x7716e134L, 0xce2e36a9L, 0xab498a11L, 0x45e63f03L, + 0x208183bbL, 0x7691e0e3L, 0x13f65c5bL, 0xfd59e949L, 0x983e55f1L, + 0x2106826cL, 0x44613ed4L, 0xaace8bc6L, 0xcfa9377eL, 0x38417fd6L, + 0x5d26c36eL, 0xb389767cL, 0xd6eecac4L, 0x6fd61d59L, 0x0ab1a1e1L, + 0xe41e14f3L, 0x8179a84bL, 0xd769cb13L, 0xb20e77abL, 0x5ca1c2b9L, + 0x39c67e01L, 0x80fea99cL, 0xe5991524L, 0x0b36a036L, 0x6e511c8eL, + 0xa7166686L, 0xc271da3eL, 0x2cde6f2cL, 0x49b9d394L, 0xf0810409L, + 0x95e6b8b1L, 0x7b490da3L, 0x1e2eb11bL, 0x483ed243L, 0x2d596efbL, + 0xc3f6dbe9L, 0xa6916751L, 0x1fa9b0ccL, 0x7ace0c74L, 0x9461b966L, + 0xf10605deL +# endif /* IZ_CRCOPTIM_UNFOLDTBL */ +# else /* !IZ_CRC_BE_OPTIMIZ */ + 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, + 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, + 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, + 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, + 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, + 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, + 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, + 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, + 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, + 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, + 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, + 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, + 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, + 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, + 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, + 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, + 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, + 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, + 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, + 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, + 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, + 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, + 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, + 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, + 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, + 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, + 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, + 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, + 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, + 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, + 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, + 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, + 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, + 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, + 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, + 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, + 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, + 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, + 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, + 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, + 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, + 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, + 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, + 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, + 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, + 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, + 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, + 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, + 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, + 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, + 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, + 0x2d02ef8dL +# ifdef IZ_CRCOPTIM_UNFOLDTBL + , + 0x00000000L, 0x191b3141L, 0x32366282L, 0x2b2d53c3L, 0x646cc504L, + 0x7d77f445L, 0x565aa786L, 0x4f4196c7L, 0xc8d98a08L, 0xd1c2bb49L, + 0xfaefe88aL, 0xe3f4d9cbL, 0xacb54f0cL, 0xb5ae7e4dL, 0x9e832d8eL, + 0x87981ccfL, 0x4ac21251L, 0x53d92310L, 0x78f470d3L, 0x61ef4192L, + 0x2eaed755L, 0x37b5e614L, 0x1c98b5d7L, 0x05838496L, 0x821b9859L, + 0x9b00a918L, 0xb02dfadbL, 0xa936cb9aL, 0xe6775d5dL, 0xff6c6c1cL, + 0xd4413fdfL, 0xcd5a0e9eL, 0x958424a2L, 0x8c9f15e3L, 0xa7b24620L, + 0xbea97761L, 0xf1e8e1a6L, 0xe8f3d0e7L, 0xc3de8324L, 0xdac5b265L, + 0x5d5daeaaL, 0x44469febL, 0x6f6bcc28L, 0x7670fd69L, 0x39316baeL, + 0x202a5aefL, 0x0b07092cL, 0x121c386dL, 0xdf4636f3L, 0xc65d07b2L, + 0xed705471L, 0xf46b6530L, 0xbb2af3f7L, 0xa231c2b6L, 0x891c9175L, + 0x9007a034L, 0x179fbcfbL, 0x0e848dbaL, 0x25a9de79L, 0x3cb2ef38L, + 0x73f379ffL, 0x6ae848beL, 0x41c51b7dL, 0x58de2a3cL, 0xf0794f05L, + 0xe9627e44L, 0xc24f2d87L, 0xdb541cc6L, 0x94158a01L, 0x8d0ebb40L, + 0xa623e883L, 0xbf38d9c2L, 0x38a0c50dL, 0x21bbf44cL, 0x0a96a78fL, + 0x138d96ceL, 0x5ccc0009L, 0x45d73148L, 0x6efa628bL, 0x77e153caL, + 0xbabb5d54L, 0xa3a06c15L, 0x888d3fd6L, 0x91960e97L, 0xded79850L, + 0xc7cca911L, 0xece1fad2L, 0xf5facb93L, 0x7262d75cL, 0x6b79e61dL, + 0x4054b5deL, 0x594f849fL, 0x160e1258L, 0x0f152319L, 0x243870daL, + 0x3d23419bL, 0x65fd6ba7L, 0x7ce65ae6L, 0x57cb0925L, 0x4ed03864L, + 0x0191aea3L, 0x188a9fe2L, 0x33a7cc21L, 0x2abcfd60L, 0xad24e1afL, + 0xb43fd0eeL, 0x9f12832dL, 0x8609b26cL, 0xc94824abL, 0xd05315eaL, + 0xfb7e4629L, 0xe2657768L, 0x2f3f79f6L, 0x362448b7L, 0x1d091b74L, + 0x04122a35L, 0x4b53bcf2L, 0x52488db3L, 0x7965de70L, 0x607eef31L, + 0xe7e6f3feL, 0xfefdc2bfL, 0xd5d0917cL, 0xcccba03dL, 0x838a36faL, + 0x9a9107bbL, 0xb1bc5478L, 0xa8a76539L, 0x3b83984bL, 0x2298a90aL, + 0x09b5fac9L, 0x10aecb88L, 0x5fef5d4fL, 0x46f46c0eL, 0x6dd93fcdL, + 0x74c20e8cL, 0xf35a1243L, 0xea412302L, 0xc16c70c1L, 0xd8774180L, + 0x9736d747L, 0x8e2de606L, 0xa500b5c5L, 0xbc1b8484L, 0x71418a1aL, + 0x685abb5bL, 0x4377e898L, 0x5a6cd9d9L, 0x152d4f1eL, 0x0c367e5fL, + 0x271b2d9cL, 0x3e001cddL, 0xb9980012L, 0xa0833153L, 0x8bae6290L, + 0x92b553d1L, 0xddf4c516L, 0xc4eff457L, 0xefc2a794L, 0xf6d996d5L, + 0xae07bce9L, 0xb71c8da8L, 0x9c31de6bL, 0x852aef2aL, 0xca6b79edL, + 0xd37048acL, 0xf85d1b6fL, 0xe1462a2eL, 0x66de36e1L, 0x7fc507a0L, + 0x54e85463L, 0x4df36522L, 0x02b2f3e5L, 0x1ba9c2a4L, 0x30849167L, + 0x299fa026L, 0xe4c5aeb8L, 0xfdde9ff9L, 0xd6f3cc3aL, 0xcfe8fd7bL, + 0x80a96bbcL, 0x99b25afdL, 0xb29f093eL, 0xab84387fL, 0x2c1c24b0L, + 0x350715f1L, 0x1e2a4632L, 0x07317773L, 0x4870e1b4L, 0x516bd0f5L, + 0x7a468336L, 0x635db277L, 0xcbfad74eL, 0xd2e1e60fL, 0xf9ccb5ccL, + 0xe0d7848dL, 0xaf96124aL, 0xb68d230bL, 0x9da070c8L, 0x84bb4189L, + 0x03235d46L, 0x1a386c07L, 0x31153fc4L, 0x280e0e85L, 0x674f9842L, + 0x7e54a903L, 0x5579fac0L, 0x4c62cb81L, 0x8138c51fL, 0x9823f45eL, + 0xb30ea79dL, 0xaa1596dcL, 0xe554001bL, 0xfc4f315aL, 0xd7626299L, + 0xce7953d8L, 0x49e14f17L, 0x50fa7e56L, 0x7bd72d95L, 0x62cc1cd4L, + 0x2d8d8a13L, 0x3496bb52L, 0x1fbbe891L, 0x06a0d9d0L, 0x5e7ef3ecL, + 0x4765c2adL, 0x6c48916eL, 0x7553a02fL, 0x3a1236e8L, 0x230907a9L, + 0x0824546aL, 0x113f652bL, 0x96a779e4L, 0x8fbc48a5L, 0xa4911b66L, + 0xbd8a2a27L, 0xf2cbbce0L, 0xebd08da1L, 0xc0fdde62L, 0xd9e6ef23L, + 0x14bce1bdL, 0x0da7d0fcL, 0x268a833fL, 0x3f91b27eL, 0x70d024b9L, + 0x69cb15f8L, 0x42e6463bL, 0x5bfd777aL, 0xdc656bb5L, 0xc57e5af4L, + 0xee530937L, 0xf7483876L, 0xb809aeb1L, 0xa1129ff0L, 0x8a3fcc33L, + 0x9324fd72L + , + 0x00000000L, 0x01c26a37L, 0x0384d46eL, 0x0246be59L, 0x0709a8dcL, + 0x06cbc2ebL, 0x048d7cb2L, 0x054f1685L, 0x0e1351b8L, 0x0fd13b8fL, + 0x0d9785d6L, 0x0c55efe1L, 0x091af964L, 0x08d89353L, 0x0a9e2d0aL, + 0x0b5c473dL, 0x1c26a370L, 0x1de4c947L, 0x1fa2771eL, 0x1e601d29L, + 0x1b2f0bacL, 0x1aed619bL, 0x18abdfc2L, 0x1969b5f5L, 0x1235f2c8L, + 0x13f798ffL, 0x11b126a6L, 0x10734c91L, 0x153c5a14L, 0x14fe3023L, + 0x16b88e7aL, 0x177ae44dL, 0x384d46e0L, 0x398f2cd7L, 0x3bc9928eL, + 0x3a0bf8b9L, 0x3f44ee3cL, 0x3e86840bL, 0x3cc03a52L, 0x3d025065L, + 0x365e1758L, 0x379c7d6fL, 0x35dac336L, 0x3418a901L, 0x3157bf84L, + 0x3095d5b3L, 0x32d36beaL, 0x331101ddL, 0x246be590L, 0x25a98fa7L, + 0x27ef31feL, 0x262d5bc9L, 0x23624d4cL, 0x22a0277bL, 0x20e69922L, + 0x2124f315L, 0x2a78b428L, 0x2bbade1fL, 0x29fc6046L, 0x283e0a71L, + 0x2d711cf4L, 0x2cb376c3L, 0x2ef5c89aL, 0x2f37a2adL, 0x709a8dc0L, + 0x7158e7f7L, 0x731e59aeL, 0x72dc3399L, 0x7793251cL, 0x76514f2bL, + 0x7417f172L, 0x75d59b45L, 0x7e89dc78L, 0x7f4bb64fL, 0x7d0d0816L, + 0x7ccf6221L, 0x798074a4L, 0x78421e93L, 0x7a04a0caL, 0x7bc6cafdL, + 0x6cbc2eb0L, 0x6d7e4487L, 0x6f38fadeL, 0x6efa90e9L, 0x6bb5866cL, + 0x6a77ec5bL, 0x68315202L, 0x69f33835L, 0x62af7f08L, 0x636d153fL, + 0x612bab66L, 0x60e9c151L, 0x65a6d7d4L, 0x6464bde3L, 0x662203baL, + 0x67e0698dL, 0x48d7cb20L, 0x4915a117L, 0x4b531f4eL, 0x4a917579L, + 0x4fde63fcL, 0x4e1c09cbL, 0x4c5ab792L, 0x4d98dda5L, 0x46c49a98L, + 0x4706f0afL, 0x45404ef6L, 0x448224c1L, 0x41cd3244L, 0x400f5873L, + 0x4249e62aL, 0x438b8c1dL, 0x54f16850L, 0x55330267L, 0x5775bc3eL, + 0x56b7d609L, 0x53f8c08cL, 0x523aaabbL, 0x507c14e2L, 0x51be7ed5L, + 0x5ae239e8L, 0x5b2053dfL, 0x5966ed86L, 0x58a487b1L, 0x5deb9134L, + 0x5c29fb03L, 0x5e6f455aL, 0x5fad2f6dL, 0xe1351b80L, 0xe0f771b7L, + 0xe2b1cfeeL, 0xe373a5d9L, 0xe63cb35cL, 0xe7fed96bL, 0xe5b86732L, + 0xe47a0d05L, 0xef264a38L, 0xeee4200fL, 0xeca29e56L, 0xed60f461L, + 0xe82fe2e4L, 0xe9ed88d3L, 0xebab368aL, 0xea695cbdL, 0xfd13b8f0L, + 0xfcd1d2c7L, 0xfe976c9eL, 0xff5506a9L, 0xfa1a102cL, 0xfbd87a1bL, + 0xf99ec442L, 0xf85cae75L, 0xf300e948L, 0xf2c2837fL, 0xf0843d26L, + 0xf1465711L, 0xf4094194L, 0xf5cb2ba3L, 0xf78d95faL, 0xf64fffcdL, + 0xd9785d60L, 0xd8ba3757L, 0xdafc890eL, 0xdb3ee339L, 0xde71f5bcL, + 0xdfb39f8bL, 0xddf521d2L, 0xdc374be5L, 0xd76b0cd8L, 0xd6a966efL, + 0xd4efd8b6L, 0xd52db281L, 0xd062a404L, 0xd1a0ce33L, 0xd3e6706aL, + 0xd2241a5dL, 0xc55efe10L, 0xc49c9427L, 0xc6da2a7eL, 0xc7184049L, + 0xc25756ccL, 0xc3953cfbL, 0xc1d382a2L, 0xc011e895L, 0xcb4dafa8L, + 0xca8fc59fL, 0xc8c97bc6L, 0xc90b11f1L, 0xcc440774L, 0xcd866d43L, + 0xcfc0d31aL, 0xce02b92dL, 0x91af9640L, 0x906dfc77L, 0x922b422eL, + 0x93e92819L, 0x96a63e9cL, 0x976454abL, 0x9522eaf2L, 0x94e080c5L, + 0x9fbcc7f8L, 0x9e7eadcfL, 0x9c381396L, 0x9dfa79a1L, 0x98b56f24L, + 0x99770513L, 0x9b31bb4aL, 0x9af3d17dL, 0x8d893530L, 0x8c4b5f07L, + 0x8e0de15eL, 0x8fcf8b69L, 0x8a809decL, 0x8b42f7dbL, 0x89044982L, + 0x88c623b5L, 0x839a6488L, 0x82580ebfL, 0x801eb0e6L, 0x81dcdad1L, + 0x8493cc54L, 0x8551a663L, 0x8717183aL, 0x86d5720dL, 0xa9e2d0a0L, + 0xa820ba97L, 0xaa6604ceL, 0xaba46ef9L, 0xaeeb787cL, 0xaf29124bL, + 0xad6fac12L, 0xacadc625L, 0xa7f18118L, 0xa633eb2fL, 0xa4755576L, + 0xa5b73f41L, 0xa0f829c4L, 0xa13a43f3L, 0xa37cfdaaL, 0xa2be979dL, + 0xb5c473d0L, 0xb40619e7L, 0xb640a7beL, 0xb782cd89L, 0xb2cddb0cL, + 0xb30fb13bL, 0xb1490f62L, 0xb08b6555L, 0xbbd72268L, 0xba15485fL, + 0xb853f606L, 0xb9919c31L, 0xbcde8ab4L, 0xbd1ce083L, 0xbf5a5edaL, + 0xbe9834edL + , + 0x00000000L, 0xb8bc6765L, 0xaa09c88bL, 0x12b5afeeL, 0x8f629757L, + 0x37def032L, 0x256b5fdcL, 0x9dd738b9L, 0xc5b428efL, 0x7d084f8aL, + 0x6fbde064L, 0xd7018701L, 0x4ad6bfb8L, 0xf26ad8ddL, 0xe0df7733L, + 0x58631056L, 0x5019579fL, 0xe8a530faL, 0xfa109f14L, 0x42acf871L, + 0xdf7bc0c8L, 0x67c7a7adL, 0x75720843L, 0xcdce6f26L, 0x95ad7f70L, + 0x2d111815L, 0x3fa4b7fbL, 0x8718d09eL, 0x1acfe827L, 0xa2738f42L, + 0xb0c620acL, 0x087a47c9L, 0xa032af3eL, 0x188ec85bL, 0x0a3b67b5L, + 0xb28700d0L, 0x2f503869L, 0x97ec5f0cL, 0x8559f0e2L, 0x3de59787L, + 0x658687d1L, 0xdd3ae0b4L, 0xcf8f4f5aL, 0x7733283fL, 0xeae41086L, + 0x525877e3L, 0x40edd80dL, 0xf851bf68L, 0xf02bf8a1L, 0x48979fc4L, + 0x5a22302aL, 0xe29e574fL, 0x7f496ff6L, 0xc7f50893L, 0xd540a77dL, + 0x6dfcc018L, 0x359fd04eL, 0x8d23b72bL, 0x9f9618c5L, 0x272a7fa0L, + 0xbafd4719L, 0x0241207cL, 0x10f48f92L, 0xa848e8f7L, 0x9b14583dL, + 0x23a83f58L, 0x311d90b6L, 0x89a1f7d3L, 0x1476cf6aL, 0xaccaa80fL, + 0xbe7f07e1L, 0x06c36084L, 0x5ea070d2L, 0xe61c17b7L, 0xf4a9b859L, + 0x4c15df3cL, 0xd1c2e785L, 0x697e80e0L, 0x7bcb2f0eL, 0xc377486bL, + 0xcb0d0fa2L, 0x73b168c7L, 0x6104c729L, 0xd9b8a04cL, 0x446f98f5L, + 0xfcd3ff90L, 0xee66507eL, 0x56da371bL, 0x0eb9274dL, 0xb6054028L, + 0xa4b0efc6L, 0x1c0c88a3L, 0x81dbb01aL, 0x3967d77fL, 0x2bd27891L, + 0x936e1ff4L, 0x3b26f703L, 0x839a9066L, 0x912f3f88L, 0x299358edL, + 0xb4446054L, 0x0cf80731L, 0x1e4da8dfL, 0xa6f1cfbaL, 0xfe92dfecL, + 0x462eb889L, 0x549b1767L, 0xec277002L, 0x71f048bbL, 0xc94c2fdeL, + 0xdbf98030L, 0x6345e755L, 0x6b3fa09cL, 0xd383c7f9L, 0xc1366817L, + 0x798a0f72L, 0xe45d37cbL, 0x5ce150aeL, 0x4e54ff40L, 0xf6e89825L, + 0xae8b8873L, 0x1637ef16L, 0x048240f8L, 0xbc3e279dL, 0x21e91f24L, + 0x99557841L, 0x8be0d7afL, 0x335cb0caL, 0xed59b63bL, 0x55e5d15eL, + 0x47507eb0L, 0xffec19d5L, 0x623b216cL, 0xda874609L, 0xc832e9e7L, + 0x708e8e82L, 0x28ed9ed4L, 0x9051f9b1L, 0x82e4565fL, 0x3a58313aL, + 0xa78f0983L, 0x1f336ee6L, 0x0d86c108L, 0xb53aa66dL, 0xbd40e1a4L, + 0x05fc86c1L, 0x1749292fL, 0xaff54e4aL, 0x322276f3L, 0x8a9e1196L, + 0x982bbe78L, 0x2097d91dL, 0x78f4c94bL, 0xc048ae2eL, 0xd2fd01c0L, + 0x6a4166a5L, 0xf7965e1cL, 0x4f2a3979L, 0x5d9f9697L, 0xe523f1f2L, + 0x4d6b1905L, 0xf5d77e60L, 0xe762d18eL, 0x5fdeb6ebL, 0xc2098e52L, + 0x7ab5e937L, 0x680046d9L, 0xd0bc21bcL, 0x88df31eaL, 0x3063568fL, + 0x22d6f961L, 0x9a6a9e04L, 0x07bda6bdL, 0xbf01c1d8L, 0xadb46e36L, + 0x15080953L, 0x1d724e9aL, 0xa5ce29ffL, 0xb77b8611L, 0x0fc7e174L, + 0x9210d9cdL, 0x2aacbea8L, 0x38191146L, 0x80a57623L, 0xd8c66675L, + 0x607a0110L, 0x72cfaefeL, 0xca73c99bL, 0x57a4f122L, 0xef189647L, + 0xfdad39a9L, 0x45115eccL, 0x764dee06L, 0xcef18963L, 0xdc44268dL, + 0x64f841e8L, 0xf92f7951L, 0x41931e34L, 0x5326b1daL, 0xeb9ad6bfL, + 0xb3f9c6e9L, 0x0b45a18cL, 0x19f00e62L, 0xa14c6907L, 0x3c9b51beL, + 0x842736dbL, 0x96929935L, 0x2e2efe50L, 0x2654b999L, 0x9ee8defcL, + 0x8c5d7112L, 0x34e11677L, 0xa9362eceL, 0x118a49abL, 0x033fe645L, + 0xbb838120L, 0xe3e09176L, 0x5b5cf613L, 0x49e959fdL, 0xf1553e98L, + 0x6c820621L, 0xd43e6144L, 0xc68bceaaL, 0x7e37a9cfL, 0xd67f4138L, + 0x6ec3265dL, 0x7c7689b3L, 0xc4caeed6L, 0x591dd66fL, 0xe1a1b10aL, + 0xf3141ee4L, 0x4ba87981L, 0x13cb69d7L, 0xab770eb2L, 0xb9c2a15cL, + 0x017ec639L, 0x9ca9fe80L, 0x241599e5L, 0x36a0360bL, 0x8e1c516eL, + 0x866616a7L, 0x3eda71c2L, 0x2c6fde2cL, 0x94d3b949L, 0x090481f0L, + 0xb1b8e695L, 0xa30d497bL, 0x1bb12e1eL, 0x43d23e48L, 0xfb6e592dL, + 0xe9dbf6c3L, 0x516791a6L, 0xccb0a91fL, 0x740cce7aL, 0x66b96194L, + 0xde0506f1L +# endif /* IZ_CRCOPTIM_UNFOLDTBL */ +# endif /* ? IZ_CRC_BE_OPTIMIZ */ +}; +#endif /* ?DYNAMIC_CRC_TABLE */ + +/* use "OF((void))" here to work around a Borland TC++ 1.0 problem */ +#ifdef USE_ZLIB +ZCONST uLongf *get_crc_table OF((void)) +#else +ZCONST ulg near *get_crc_table OF((void)) +#endif +{ +#ifdef DYNAMIC_CRC_TABLE + if (CRC_TABLE_IS_EMPTY) + make_crc_table(); +#endif +#ifdef USE_ZLIB + return (ZCONST uLongf *)crc_table; +#else + return crc_table; +#endif +} + +#ifdef DYNALLOC_CRCTAB +void free_crc_table() +{ + if (!CRC_TABLE_IS_EMPTY) + { + nearfree((ulg near *)crc_table); + MARK_CRCTAB_EMPTY; + } +} +#endif + +#ifndef USE_ZLIB +#ifndef CRC_TABLE_ONLY +#ifndef ASM_CRC + +#define DO1(crc, buf) crc = CRC32(crc, *buf++, crc_32_tab) +#define DO2(crc, buf) DO1(crc, buf); DO1(crc, buf) +#define DO4(crc, buf) DO2(crc, buf); DO2(crc, buf) +#define DO8(crc, buf) DO4(crc, buf); DO4(crc, buf) + +#if (defined(IZ_CRC_BE_OPTIMIZ) || defined(IZ_CRC_LE_OPTIMIZ)) + +# ifdef IZ_CRCOPTIM_UNFOLDTBL +# ifdef IZ_CRC_BE_OPTIMIZ +# define DO_OPT4(c, buf4) c ^= *(buf4)++; \ + c = crc_32_tab[c & 0xff] ^ crc_32_tab[256+((c>>8) & 0xff)] ^ \ + crc_32_tab[2*256+((c>>16) & 0xff)] ^ crc_32_tab[3*256+(c>>24)] +# else /* !IZ_CRC_BE_OPTIMIZ */ +# define DO_OPT4(c, buf4) c ^= *(buf4)++; \ + c = crc_32_tab[3*256+(c & 0xff)] ^ crc_32_tab[2*256+((c>>8) & 0xff)] \ + ^ crc_32_tab[256+((c>>16) & 0xff)] ^ crc_32_tab[c>>24] +# endif /* ?IZ_CRC_BE_OPTIMIZ */ +# else /* !IZ_CRCOPTIM_UNFOLDTBL */ +# define DO_OPT4(c, buf4) c ^= *(buf4)++; \ + c = CRC32UPD(c, crc_32_tab); \ + c = CRC32UPD(c, crc_32_tab); \ + c = CRC32UPD(c, crc_32_tab); \ + c = CRC32UPD(c, crc_32_tab) +# endif /* ?IZ_CRCOPTIM_UNFOLDTBL */ + +# define DO_OPT16(crc, buf4) DO_OPT4(crc, buf4); DO_OPT4(crc, buf4); \ + DO_OPT4(crc, buf4); DO_OPT4(crc, buf4); + +#endif /* (IZ_CRC_BE_OPTIMIZ || IZ_CRC_LE_OPTIMIZ) */ + + +/* ========================================================================= */ +ulg crc32(crc, buf, len) + ulg crc; /* crc shift register */ + register ZCONST uch *buf; /* pointer to bytes to pump through */ + extent len; /* number of bytes in buf[] */ +/* Run a set of bytes through the crc shift register. If buf is a NULL + pointer, then initialize the crc shift register contents instead. + Return the current crc in either case. */ +{ + register z_uint4 c; + register ZCONST ulg near *crc_32_tab; + + if (buf == NULL) return 0L; + + crc_32_tab = get_crc_table(); + + c = (REV_BE((z_uint4)crc) ^ 0xffffffffL); + +#if (defined(IZ_CRC_BE_OPTIMIZ) || defined(IZ_CRC_LE_OPTIMIZ)) + /* Align buf pointer to next DWORD boundary. */ + while (len && ((ptrdiff_t)buf & 3)) { + DO1(c, buf); + len--; + } + { + ZCONST z_uint4 *buf4 = (ZCONST z_uint4 *)buf; + while (len >= 16) { + DO_OPT16(c, buf4); + len -= 16; + } + while (len >= 4) { + DO_OPT4(c, buf4); + len -= 4; + } + buf = (ZCONST uch *)buf4; + } +#else /* !(IZ_CRC_BE_OPTIMIZ || IZ_CRC_LE_OPTIMIZ) */ +#ifndef NO_UNROLLED_LOOPS + while (len >= 8) { + DO8(c, buf); + len -= 8; + } +#endif /* !NO_UNROLLED_LOOPS */ +#endif /* ?(IZ_CRC_BE_OPTIMIZ || IZ_CRC_LE_OPTIMIZ) */ + if (len) do { + DO1(c, buf); + } while (--len); + + return REV_BE(c) ^ 0xffffffffL; /* (instead of ~c for 64-bit machines) */ +} +#endif /* !ASM_CRC */ +#endif /* !CRC_TABLE_ONLY */ +#endif /* !USE_ZLIB */ +#endif /* !USE_ZLIB || USE_OWN_CRCTAB */ diff --git a/third_party/infozip/zip/crc32.h b/third_party/infozip/zip/crc32.h new file mode 100644 index 000000000..05e9de1be --- /dev/null +++ b/third_party/infozip/zip/crc32.h @@ -0,0 +1,61 @@ +/* clang-format off */ +/* + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* crc32.h -- compute the CRC-32 of a data stream + * Copyright (C) 1995 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifndef __crc32_h +#define __crc32_h /* identifies this source module */ + +/* This header should be read AFTER zip.h resp. unzip.h + * (the latter with UNZIP_INTERNAL defined...). + */ + +#ifndef OF +# define OF(a) a +#endif +#ifndef ZCONST +# define ZCONST const +#endif + +#ifdef DYNALLOC_CRCTAB + void free_crc_table OF((void)); +#endif +#ifndef USE_ZLIB + ZCONST ulg near *get_crc_table OF((void)); +#endif +#if (defined(USE_ZLIB) || defined(CRC_TABLE_ONLY)) +# ifdef IZ_CRC_BE_OPTIMIZ +# undef IZ_CRC_BE_OPTIMIZ +# endif +#else /* !(USE_ZLIB || CRC_TABLE_ONLY) */ + ulg crc32 OF((ulg crc, ZCONST uch *buf, extent len)); +#endif /* ?(USE_ZLIB || CRC_TABLE_ONLY) */ + +#ifndef CRC_32_TAB +# define CRC_32_TAB crc_32_tab +#endif + +#ifdef CRC32 +# undef CRC32 +#endif +#ifdef IZ_CRC_BE_OPTIMIZ +# define CRC32UPD(c, crctab) (crctab[((c) >> 24)] ^ ((c) << 8)) +# define CRC32(c, b, crctab) (crctab[(((int)(c) >> 24) ^ (b))] ^ ((c) << 8)) +# define REV_BE(w) (((w)>>24)+(((w)>>8)&0xff00)+ \ + (((w)&0xff00)<<8)+(((w)&0xff)<<24)) +#else +# define CRC32UPD(c, crctab) (crctab[((int)(c)) & 0xff] ^ ((c) >> 8)) +# define CRC32(c, b, crctab) (crctab[((int)(c) ^ (b)) & 0xff] ^ ((c) >> 8)) +# define REV_BE(w) w +#endif + +#endif /* !__crc32_h */ diff --git a/third_party/infozip/zip/crc_i386.S b/third_party/infozip/zip/crc_i386.S new file mode 100644 index 000000000..38dbc8684 --- /dev/null +++ b/third_party/infozip/zip/crc_i386.S @@ -0,0 +1,304 @@ +/* + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * crc_i386.S, optimized CRC calculation function for Zip and UnZip, + * created by Paul Kienitz and Christian Spieler. Last revised 07 Jan 2007. + * + * GRR 961110: incorporated Scott Field optimizations from win32/crc_i386.asm + * => overall 6% speedup in "unzip -tq" on 9MB zipfile (486-66) + * + * SPC 970402: revised for Rodney Brown's optimizations (32-bit-wide + * aligned reads for most of the data from buffer), can be + * disabled by defining the macro NO_32_BIT_LOADS + * + * SPC 971012: added Rodney Brown's additional tweaks for 32-bit-optimized + * CPUs (like the Pentium Pro, Pentium II, and probably some + * Pentium clones). This optimization is controlled by the + * preprocessor switch "__686" and is disabled by default. + * (This default is based on the assumption that most users + * do not yet work on a Pentium Pro or Pentium II machine ...) + * + * COS 050116: Enabled the 686 build by default, because there are hardly any + * pre-686 CPUs in serious use nowadays. (See SPC 970402 above.) + * + * SPC 060103: Updated code to incorporate newer optimizations found in zlib. + * + * SPC 070107: Added conditional switch to deactivate crc32() compilation. + * + * FLAT memory model assumed. Calling interface: + * - args are pushed onto the stack from right to left, + * - return value is given in the EAX register, + * - all other registers (with exception of EFLAGS) are preserved. (With + * GNU C 2.7.x, %edx and %ecx are `scratch' registers, but preserving + * them nevertheless adds only 4 single byte instructions.) + * + * This source generates the function + * ulg crc32(ulg crc, ZCONST uch *buf, extent len). + * + * Loop unrolling can be disabled by defining the macro NO_UNROLLED_LOOPS. + * This results in shorter code at the expense of reduced performance. + */ + +/* This file is NOT used in conjunction with zlib, or when only creation of + * the basic CRC_32_Table (for other purpose) is requested. + */ +#if !defined(USE_ZLIB) && !defined(CRC_TABLE_ONLY) + +/* Preprocess with -DNO_UNDERLINE if your C compiler does not prefix + * external symbols with an underline character '_'. + */ +#if defined(NO_UNDERLINE) || defined(__ELF__) +# define _crc32 crc32 +# define _get_crc_table get_crc_table +#endif +/* Use 16-byte alignment if your assembler supports it. Warning: gas + * uses a log(x) parameter (.align 4 means 16-byte alignment). On SVR4 + * the parameter is a number of bytes. + */ +#ifndef ALIGNMENT +# define ALIGNMENT .align 4,0x90 +#endif + +#if defined(i386) || defined(_i386) || defined(_I386) || defined(__i386) + +/* This version is for 386 Unix, OS/2, MSDOS in 32 bit mode (gcc & gas). + * Warning: it uses the AT&T syntax: mov source,dest + * This file is only optional. If you want to use the C version, + * remove -DASM_CRC from CFLAGS in Makefile and set OBJA to an empty string. + */ + + .file "crc_i386.S" + +#if !defined(PRE_686) && !defined(__686) + /* Optimize for Pentium Pro and compatible CPUs by default. */ +# define __686 +#endif + +#if defined(NO_STD_STACKFRAME) && defined(USE_STD_STACKFRAME) +# undef USE_STACKFRAME +#else + /* The default is to use standard stack frame entry, because it + * results in smaller code! + */ +# ifndef USE_STD_STACKFRAME +# define USE_STD_STACKFRAME +# endif +#endif + +#ifdef USE_STD_STACKFRAME +# define _STD_ENTRY pushl %ebp ; movl %esp,%ebp +# define arg1 8(%ebp) +# define arg2 12(%ebp) +# define arg3 16(%ebp) +# define _STD_LEAVE popl %ebp +#else /* !USE_STD_STACKFRAME */ +# define _STD_ENTRY +# define arg1 24(%esp) +# define arg2 28(%esp) +# define arg3 32(%esp) +# define _STD_LEAVE +#endif /* ?USE_STD_STACKFRAME */ + +/* + * These two (three) macros make up the loop body of the CRC32 cruncher. + * registers modified: + * eax : crc value "c" + * esi : pointer to next data byte (or lword) "buf++" + * registers read: + * edi : pointer to base of crc_table array + * scratch registers: + * ebx : index into crc_table array + * (requires upper three bytes = 0 when __686 is undefined) + */ +#ifndef __686 /* optimize for 386, 486, Pentium */ +#define Do_CRC /* c = (c >> 8) ^ table[c & 0xFF] */\ + movb %al, %bl ;/* tmp = c & 0xFF */\ + shrl $8, %eax ;/* c = (c >> 8) */\ + xorl (%edi, %ebx, 4), %eax ;/* c ^= table[tmp] */ +#else /* __686 : optimize for Pentium Pro and compatible CPUs */ +#define Do_CRC /* c = (c >> 8) ^ table[c & 0xFF] */\ + movzbl %al, %ebx ;/* tmp = c & 0xFF */\ + shrl $8, %eax ;/* c = (c >> 8) */\ + xorl (%edi, %ebx, 4), %eax ;/* c ^=table[tmp] */ +#endif /* ?__686 */ + +#define Do_CRC_byte /* c = (c >> 8) ^ table[(c^*buf++)&0xFF] */\ + xorb (%esi), %al ;/* c ^= *buf */\ + incl %esi ;/* buf++ */\ + Do_CRC + +#define Do_CRC_byteof(ofs) /* c = (c >> 8) ^ table[(c^*buf++)&0xFF] */\ + xorb ofs(%esi), %al ;/* c ^= *buf */\ + incl %esi ;/* buf++ */\ + Do_CRC + +#ifndef NO_32_BIT_LOADS +# ifdef IZ_CRCOPTIM_UNFOLDTBL + /* the edx register is needed in crc calculation */ +# define SavLen arg3 +# define UpdCRC_lword \ + movzbl %al, %ebx ; \ + movl 3072(%edi,%ebx,4), %edx ; \ + movzbl %ah, %ebx ; \ + shrl $16, %eax ; \ + xor 2048(%edi,%ebx,4), %edx ; \ + movzbl %al, %ebx ; \ + shrl $8,%eax ; \ + xorl 1024(%edi,%ebx,4), %edx ; \ + movl (%edi,%eax,4), %eax ; \ + xorl %edx,%eax ; +# define UpdCRC_lword_sh(dwPtrIncr) \ + movzbl %al, %ebx ; \ + movl 3072(%edi,%ebx,4), %edx ; \ + movzbl %ah, %ebx ; \ + shrl $16, %eax ; \ + xor 2048(%edi,%ebx,4), %edx ; \ + movzbl %al, %ebx ; \ + addl $4*(dwPtrIncr), %esi ;/* ((ulg *)buf)+=dwPtrIncr */\ + shrl $8,%eax ; \ + xorl 1024(%edi,%ebx,4), %edx ; \ + movl (%edi,%eax,4),%eax ; \ + xorl %edx,%eax ; +# else /* !IZ_CRCOPTIM_UNFOLDTBL */ + /* the edx register is not needed anywhere else */ +# define SavLen %edx +# define UpdCRC_lword \ + Do_CRC \ + Do_CRC \ + Do_CRC \ + Do_CRC +# define UpdCRC_lword_sh(dwPtrIncr) \ + Do_CRC \ + Do_CRC \ + addl $4*(dwPtrIncr), %esi ;/* ((ulg *)buf)++ */\ + Do_CRC \ + Do_CRC +# endif /* ?IZ_CRCOPTIM_UNFOLDTBL */ +#define Do_CRC_lword \ + xorl (%esi), %eax ;/* c ^= *(ulg *)buf */\ + UpdCRC_lword_sh(1) /* ... ((ulg *)buf)++ */ +#define Do_CRC_4lword \ + xorl (%esi), %eax ;/* c ^= *(ulg *)buf */\ + UpdCRC_lword \ + xorl 4(%esi), %eax ;/* c ^= *((ulg *)buf+1) */\ + UpdCRC_lword \ + xorl 8(%esi), %eax ;/* c ^= *((ulg *)buf+2) */\ + UpdCRC_lword \ + xorl 12(%esi), %eax ;/* c ^= *((ulg *)buf]+3 */\ + UpdCRC_lword_sh(4) /* ... ((ulg *)buf)+=4 */ +#endif /* !NO_32_BIT_LOADS */ + + + .text + + .globl _crc32 + +_crc32: /* ulg crc32(ulg crc, uch *buf, extent len) */ + _STD_ENTRY + pushl %edi + pushl %esi + pushl %ebx + pushl %edx + pushl %ecx + + movl arg2, %esi /* 2nd arg: uch *buf */ + subl %eax, %eax /* > if (!buf) */ + testl %esi, %esi /* > return 0; */ + jz .L_fine /* > else { */ + call _get_crc_table + movl %eax, %edi + movl arg1, %eax /* 1st arg: ulg crc */ +#ifndef __686 + subl %ebx, %ebx /* ebx=0; bl usable as dword */ +#endif + movl arg3, %ecx /* 3rd arg: extent len */ + notl %eax /* > c = ~crc; */ + + testl %ecx, %ecx +#ifndef NO_UNROLLED_LOOPS + jz .L_bail +# ifndef NO_32_BIT_LOADS + /* Assert now have positive length */ +.L_align_loop: + testl $3, %esi /* Align buf on lword boundary */ + jz .L_aligned_now + Do_CRC_byte + decl %ecx + jnz .L_align_loop +.L_aligned_now: +# endif /* !NO_32_BIT_LOADS */ + movl %ecx, SavLen /* save current value of len */ + shrl $4, %ecx /* ecx = len / 16 */ + jz .L_No_Sixteens +/* align loop head at start of 486 internal cache line !! */ + ALIGNMENT +.L_Next_Sixteen: +# ifndef NO_32_BIT_LOADS + Do_CRC_4lword +# else /* NO_32_BIT_LOADS */ + Do_CRC_byteof(0) + Do_CRC_byteof(1) + Do_CRC_byteof(2) + Do_CRC_byteof(3) + Do_CRC_byteof(4) + Do_CRC_byteof(5) + Do_CRC_byteof(6) + Do_CRC_byteof(7) + Do_CRC_byteof(8) + Do_CRC_byteof(9) + Do_CRC_byteof(10) + Do_CRC_byteof(11) + Do_CRC_byteof(12) + Do_CRC_byteof(13) + Do_CRC_byteof(14) + Do_CRC_byteof(15) + addl $16,%esi ;/* buf += 16 */ +# endif /* ?NO_32_BIT_LOADS */ + decl %ecx + jnz .L_Next_Sixteen + +.L_No_Sixteens: + movl SavLen, %ecx + andl $15, %ecx /* ecx = len % 16 */ +# ifndef NO_32_BIT_LOADS + shrl $2,%ecx /* ecx = len / 4 */ + jz .L_No_Fours +.L_Next_Four: + Do_CRC_lword + decl %ecx + jnz .L_Next_Four +.L_No_Fours: + movl SavLen,%ecx + andl $3,%ecx /* ecx = len % 4 */ +# endif /* !NO_32_BIT_LOADS */ +#endif /* !NO_UNROLLED_LOOPS */ + jz .L_bail /* > if (len) */ +/* align loop head at start of 486 internal cache line !! */ + ALIGNMENT +.L_loupe: /* > do { */ + Do_CRC_byte /* c = CRC32(c,*buf++,crctab);*/ + decl %ecx /* > } while (--len); */ + jnz .L_loupe + +.L_bail: /* > } */ + notl %eax /* > return ~c; */ +.L_fine: + popl %ecx + popl %edx + popl %ebx + popl %esi + popl %edi + _STD_LEAVE + ret + +#else + error: this asm version is for 386 only +#endif /* i386 || _i386 || _I386 || __i386 */ + +#endif /* !USE_ZLIB && !CRC_TABLE_ONLY */ diff --git a/third_party/infozip/zip/crypt.c b/third_party/infozip/zip/crypt.c new file mode 100644 index 000000000..f0eb037bd --- /dev/null +++ b/third_party/infozip/zip/crypt.c @@ -0,0 +1,692 @@ +/* clang-format off */ +/* + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The main encryption/decryption source code for Info-Zip software was + originally written in Europe. To the best of our knowledge, it can + be freely distributed in both source and object forms from any country, + including the USA under License Exception TSU of the U.S. Export + Administration Regulations (section 740.13(e)) of 6 June 2002. + + NOTE on copyright history: + Previous versions of this source package (up to version 2.8) were + not copyrighted and put in the public domain. If you cannot comply + with the Info-Zip LICENSE, you may want to look for one of those + public domain versions. + */ + +/* + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + */ + +#define ZCRYPT_INTERNAL +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/crypt.h" +#include "third_party/infozip/zip/ttyio.h" +#include "libc/rand/rand.h" + +#if CRYPT + +#ifndef FALSE +# define FALSE 0 +#endif + +#ifdef ZIP + /* For the encoding task used in Zip (and ZipCloak), we want to initialize + the crypt algorithm with some reasonably unpredictable bytes, see + the crypthead() function. The standard rand() library function is + used to supply these `random' bytes, which in turn is initialized by + a srand() call. The srand() function takes an "unsigned" (at least 16bit) + seed value as argument to determine the starting point of the rand() + pseudo-random number generator. + This seed number is constructed as "Seed = Seed1 .XOR. Seed2" with + Seed1 supplied by the current time (= "(unsigned)time()") and Seed2 + as some (hopefully) nondeterministic bitmask. On many (most) systems, + we use some "process specific" number, as the PID or something similar, + but when nothing unpredictable is available, a fixed number may be + sufficient. + NOTE: + 1.) This implementation requires the availability of the following + standard UNIX C runtime library functions: time(), rand(), srand(). + On systems where some of them are missing, the environment that + incorporates the crypt routines must supply suitable replacement + functions. + 2.) It is a very bad idea to use a second call to time() to set the + "Seed2" number! In this case, both "Seed1" and "Seed2" would be + (almost) identical, resulting in a (mostly) "zero" constant seed + number passed to srand(). + + The implementation environment defined in the "zip.h" header should + supply a reasonable definition for ZCR_SEED2 (an unsigned number; for + most implementations of rand() and srand(), only the lower 16 bits are + significant!). An example that works on many systems would be + "#define ZCR_SEED2 (unsigned)getpid()". + The default definition for ZCR_SEED2 supplied below should be regarded + as a fallback to allow successful compilation in "beta state" + environments. + */ +# include "libc/time/time.h" /* time() function supplies first part of crypt seed */ + /* "last resort" source for second part of crypt seed pattern */ +# ifndef ZCR_SEED2 +# define ZCR_SEED2 (unsigned)3141592654L /* use PI as default pattern */ +# endif +# ifdef GLOBAL /* used in Amiga system headers, maybe others too */ +# undef GLOBAL +# endif +# define GLOBAL(g) g +#else /* !ZIP */ +# define GLOBAL(g) G.g +#endif /* ?ZIP */ + + +#ifdef UNZIP + /* char *key = (char *)NULL; moved to globals.h */ +# ifndef FUNZIP + local int testp OF((__GPRO__ ZCONST uch *h)); + local int testkey OF((__GPRO__ ZCONST uch *h, ZCONST char *key)); +# endif +#else /* def UNZIP */ /* moved to globals.h for UnZip */ + local z_uint4 keys[3]; /* keys defining the pseudo-random sequence */ +#endif /* def UNZIP [else] */ + +#ifndef Trace +# ifdef CRYPT_DEBUG +# define Trace(x) fprintf x +# else +# define Trace(x) +# endif +#endif + +#include "third_party/infozip/zip/crc32.h" + +#ifdef IZ_CRC_BE_OPTIMIZ + local z_uint4 near crycrctab[256]; + local z_uint4 near *cry_crctb_p = NULL; + local z_uint4 near *crytab_init OF((__GPRO)); +# define CRY_CRC_TAB cry_crctb_p +# undef CRC32 +# define CRC32(c, b, crctab) (crctab[((int)(c) ^ (b)) & 0xff] ^ ((c) >> 8)) +#else +# define CRY_CRC_TAB CRC_32_TAB +#endif /* ?IZ_CRC_BE_OPTIMIZ */ + +/*********************************************************************** + * Return the next byte in the pseudo-random sequence + */ +int decrypt_byte(__G) + __GDEF +{ + unsigned temp; /* POTENTIAL BUG: temp*(temp^1) may overflow in an + * unpredictable manner on 16-bit systems; not a problem + * with any known compiler so far, though */ + + temp = ((unsigned)GLOBAL(keys[2]) & 0xffff) | 2; + return (int)(((temp * (temp ^ 1)) >> 8) & 0xff); +} + +/*********************************************************************** + * Update the encryption keys with the next byte of plain text + */ +int update_keys(__G__ c) + __GDEF + int c; /* byte of plain text */ +{ + GLOBAL(keys[0]) = CRC32(GLOBAL(keys[0]), c, CRY_CRC_TAB); + GLOBAL(keys[1]) = (GLOBAL(keys[1]) + + (GLOBAL(keys[0]) & 0xff)) + * 134775813L + 1; + { + register int keyshift = (int)(GLOBAL(keys[1]) >> 24); + GLOBAL(keys[2]) = CRC32(GLOBAL(keys[2]), keyshift, CRY_CRC_TAB); + } + return c; +} + + +/*********************************************************************** + * Initialize the encryption keys and the random header according to + * the given password. + */ +void init_keys(__G__ passwd) + __GDEF + ZCONST char *passwd; /* password string with which to modify keys */ +{ +#ifdef IZ_CRC_BE_OPTIMIZ + if (cry_crctb_p == NULL) { + cry_crctb_p = crytab_init(__G); + } +#endif + GLOBAL(keys[0]) = 305419896L; + GLOBAL(keys[1]) = 591751049L; + GLOBAL(keys[2]) = 878082192L; + while (*passwd != '\0') { + update_keys(__G__ (int)*passwd); + passwd++; + } +} + + +/*********************************************************************** + * Initialize the local copy of the table of precomputed crc32 values. + * Whereas the public crc32-table is optimized for crc32 calculations + * on arrays of bytes, the crypt code needs the crc32 values in an + * byte-order-independent form as 32-bit unsigned numbers. On systems + * with Big-Endian byte order using the optimized crc32 code, this + * requires inverting the byte-order of the values in the + * crypt-crc32-table. + */ +#ifdef IZ_CRC_BE_OPTIMIZ +local z_uint4 near *crytab_init(__G) + __GDEF +{ + int i; + + for (i = 0; i < 256; i++) { + crycrctab[i] = REV_BE(CRC_32_TAB[i]); + } + return crycrctab; +} +#endif + + +#ifdef ZIP + +/*********************************************************************** + * Write encryption header to file zfile using the password passwd + * and the cyclic redundancy check crc. + */ +void crypthead(passwd, crc) + ZCONST char *passwd; /* password string */ + ulg crc; /* crc of file being encrypted */ +{ + int n; /* index in random header */ + int t; /* temporary */ + int c; /* random byte */ + uch header[RAND_HEAD_LEN]; /* random header */ + static unsigned calls = 0; /* ensure different random header each time */ + + /* First generate RAND_HEAD_LEN-2 random bytes. We encrypt the + * output of rand() to get less predictability, since rand() is + * often poorly implemented. + */ + if (++calls == 1) { + srand((unsigned)time(NULL) ^ ZCR_SEED2); + } + init_keys(passwd); + for (n = 0; n < RAND_HEAD_LEN-2; n++) { + c = (rand() >> 7) & 0xff; + header[n] = (uch)zencode(c, t); + } + /* Encrypt random header (last two bytes is high word of crc) */ + init_keys(passwd); + for (n = 0; n < RAND_HEAD_LEN-2; n++) { + header[n] = (uch)zencode(header[n], t); + } + header[RAND_HEAD_LEN-2] = (uch)zencode((int)(crc >> 16) & 0xff, t); + header[RAND_HEAD_LEN-1] = (uch)zencode((int)(crc >> 24) & 0xff, t); + bfwrite(header, 1, RAND_HEAD_LEN, BFWRITE_DATA); +} + + +#ifdef UTIL + +/*********************************************************************** + * Encrypt the zip entry described by z from file in_file to file y + * using the password passwd. Return an error code in the ZE_ class. + */ +int zipcloak(z, passwd) + struct zlist far *z; /* zip entry to encrypt */ + ZCONST char *passwd; /* password string */ +{ + int c; /* input byte */ + int res; /* result code */ + zoff_t n; /* holds offset and counts size */ + int t; /* temporary */ + struct zlist far *localz; /* local header */ + uch buf[1024]; /* write buffer */ + int b; /* bytes in buffer */ + + /* Set encrypted bit, clear extended local header bit and write local + header to output file */ + if ((n = (zoff_t)zftello(y)) == (zoff_t)-1L) return ZE_TEMP; + + /* assume this archive is one disk and the file is open */ + + /* read the local header */ + res = readlocal(&localz, z); + + /* update disk and offset */ + z->dsk = 0; + z->off = n; + + /* Set encryption and unset any extended local header */ + z->flg |= 1, z->flg &= ~8; + localz->lflg |= 1, localz->lflg &= ~8; + + /* Add size of encryption header */ + localz->siz += RAND_HEAD_LEN; + z->siz = localz->siz; + + /* Put the local header */ + if ((res = putlocal(localz, PUTLOCAL_WRITE)) != ZE_OK) return res; + + /* Initialize keys with password and write random header */ + crypthead(passwd, localz->crc); + + /* Encrypt data */ + b = 0; + for (n = z->siz - RAND_HEAD_LEN; n; n--) { + if ((c = getc(in_file)) == EOF) { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + buf[b] = (uch)zencode(c, t); + b++; + if (b >= 1024) { + /* write the buffer */ + bfwrite(buf, 1, b, BFWRITE_DATA); + b = 0; + } + } + if (b) { + /* write the buffer */ + bfwrite(buf, 1, b, BFWRITE_DATA); + b = 0; + } + + /* Since we seek to the start of each local header can skip + reading any extended local header */ + /* + if ((flag & 8) != 0 && zfseeko(in_file, 16L, SEEK_CUR)) { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + if (fflush(y) == EOF) return ZE_TEMP; + */ + + /* Update number of bytes written to output file */ + tempzn += (4 + LOCHEAD) + localz->nam + localz->ext + localz->siz; + + /* Free local header */ + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + + return ZE_OK; +} + +/*********************************************************************** + * Decrypt the zip entry described by z from file in_file to file y + * using the password passwd. Return an error code in the ZE_ class. + */ +int zipbare(z, passwd) + struct zlist far *z; /* zip entry to encrypt */ + ZCONST char *passwd; /* password string */ +{ +#ifdef ZIP10 + int c0 /* byte preceding the last input byte */ +#endif + int c1; /* last input byte */ + /* all file offset and size now zoff_t - 8/28/04 EG */ + zoff_t size; /* size of input data */ + struct zlist far *localz; /* local header */ + uch buf[1024]; /* write buffer */ + int b; /* bytes in buffer */ + zoff_t n; + int r; /* size of encryption header */ + int res; /* return code */ + + /* Save position */ + if ((n = (zoff_t)zftello(y)) == (zoff_t)-1L) return ZE_TEMP; + + /* Read local header */ + res = readlocal(&localz, z); + + /* Update disk and offset */ + z->dsk = 0; + z->off = n; + + /* Initialize keys with password */ + init_keys(passwd); + + /* Decrypt encryption header, save last two bytes */ + c1 = 0; + for (r = RAND_HEAD_LEN; r; r--) { +#ifdef ZIP10 + c0 = c1; +#endif + if ((c1 = getc(in_file)) == EOF) { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + Trace((stdout, " (%02x)", c1)); + zdecode(c1); + Trace((stdout, " %02x", c1)); + } + Trace((stdout, "\n")); + + /* If last two bytes of header don't match crc (or file time in the + * case of an extended local header), back up and just copy. For + * pkzip 2.0, the check has been reduced to one byte only. + */ +#ifdef ZIP10 + if ((ush)(c0 | (c1<<8)) != + (z->flg & 8 ? (ush) z->tim & 0xffff : (ush)(z->crc >> 16))) { +#else + if ((ush)c1 != (z->flg & 8 ? (ush) z->tim >> 8 : (ush)(z->crc >> 24))) { +#endif + if (zfseeko(in_file, n, SEEK_SET)) { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + if ((res = zipcopy(z)) != ZE_OK) { + ziperr(res, "was copying an entry"); + } + return ZE_MISS; + } + + z->siz -= RAND_HEAD_LEN; + localz->siz = z->siz; + + localz->flg = z->flg &= ~9; + z->lflg = localz->lflg &= ~9; + + if ((res = putlocal(localz, PUTLOCAL_WRITE)) != ZE_OK) return res; + + /* Decrypt data */ + b = 0; + for (size = z->siz; size; size--) { + if ((c1 = getc(in_file)) == EOF) { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + zdecode(c1); + buf[b] = c1; + b++; + if (b >= 1024) { + /* write the buffer */ + bfwrite(buf, 1, b, BFWRITE_DATA); + b = 0; + } + } + if (b) { + /* write the buffer */ + bfwrite(buf, 1, b, BFWRITE_DATA); + b = 0; + } + /* Since we seek to the start of each local header can skip + reading any extended local header */ + + /* Update number of bytes written to output file */ + tempzn += (4 + LOCHEAD) + localz->nam + localz->ext + localz->siz; + + /* Free local header */ + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + + return ZE_OK; +} + + +#else /* !UTIL */ + +/*********************************************************************** + * If requested, encrypt the data in buf, and in any case call fwrite() + * with the arguments to zfwrite(). Return what fwrite() returns. + * + * now write to global y + * + * A bug has been found when encrypting large files that don't + * compress. See trees.c for the details and the fix. + */ +unsigned zfwrite(buf, item_size, nb) + zvoid *buf; /* data buffer */ + extent item_size; /* size of each item in bytes */ + extent nb; /* number of items */ +#if 0 + FILE *f; /* file to write to */ +#endif +{ + int t; /* temporary */ + + if (key != (char *)NULL) { /* key is the global password pointer */ + ulg size; /* buffer size */ + char *p = (char *)buf; /* steps through buffer */ + + /* Encrypt data in buffer */ + for (size = item_size*(ulg)nb; size != 0; p++, size--) { + *p = (char)zencode(*p, t); + } + } + /* Write the buffer out */ + return bfwrite(buf, item_size, nb, BFWRITE_DATA); +} + +#endif /* ?UTIL */ +#endif /* ZIP */ + + +#if (defined(UNZIP) && !defined(FUNZIP)) + +/*********************************************************************** + * Get the password and set up keys for current zipfile member. + * Return PK_ class error. + */ +int decrypt(__G__ passwrd) + __GDEF + ZCONST char *passwrd; +{ + ush b; + int n, r; + uch h[RAND_HEAD_LEN]; + + Trace((stdout, "\n[incnt = %d]: ", GLOBAL(incnt))); + + /* get header once (turn off "encrypted" flag temporarily so we don't + * try to decrypt the same data twice) */ + GLOBAL(pInfo->encrypted) = FALSE; + defer_leftover_input(__G); + for (n = 0; n < RAND_HEAD_LEN; n++) { + b = NEXTBYTE; + h[n] = (uch)b; + Trace((stdout, " (%02x)", h[n])); + } + undefer_input(__G); + GLOBAL(pInfo->encrypted) = TRUE; + + if (GLOBAL(newzip)) { /* this is first encrypted member in this zipfile */ + GLOBAL(newzip) = FALSE; + if (passwrd != (char *)NULL) { /* user gave password on command line */ + if (!GLOBAL(key)) { + if ((GLOBAL(key) = (char *)malloc(strlen(passwrd)+1)) == + (char *)NULL) + return PK_MEM2; + strcpy(GLOBAL(key), passwrd); + GLOBAL(nopwd) = TRUE; /* inhibit password prompting! */ + } + } else if (GLOBAL(key)) { /* get rid of previous zipfile's key */ + free(GLOBAL(key)); + GLOBAL(key) = (char *)NULL; + } + } + + /* if have key already, test it; else allocate memory for it */ + if (GLOBAL(key)) { + if (!testp(__G__ h)) + return PK_COOL; /* existing password OK (else prompt for new) */ + else if (GLOBAL(nopwd)) + return PK_WARN; /* user indicated no more prompting */ + } else if ((GLOBAL(key) = (char *)malloc(IZ_PWLEN+1)) == (char *)NULL) + return PK_MEM2; + + /* try a few keys */ + n = 0; + do { + r = (*G.decr_passwd)((zvoid *)&G, &n, GLOBAL(key), IZ_PWLEN+1, + GLOBAL(zipfn), GLOBAL(filename)); + if (r == IZ_PW_ERROR) { /* internal error in fetch of PW */ + free (GLOBAL(key)); + GLOBAL(key) = NULL; + return PK_MEM2; + } + if (r != IZ_PW_ENTERED) { /* user replied "skip" or "skip all" */ + *GLOBAL(key) = '\0'; /* We try the NIL password, ... */ + n = 0; /* and cancel fetch for this item. */ + } + if (!testp(__G__ h)) + return PK_COOL; + if (r == IZ_PW_CANCELALL) /* User replied "Skip all" */ + GLOBAL(nopwd) = TRUE; /* inhibit any further PW prompt! */ + } while (n > 0); + + return PK_WARN; + +} /* end function decrypt() */ + + + +/*********************************************************************** + * Test the password. Return -1 if bad, 0 if OK. + */ +local int testp(__G__ h) + __GDEF + ZCONST uch *h; +{ + int r; + char *key_translated; + + /* On systems with "obscure" native character coding (e.g., EBCDIC), + * the first test translates the password to the "main standard" + * character coding. */ + +#ifdef STR_TO_CP1 + /* allocate buffer for translated password */ + if ((key_translated = malloc(strlen(GLOBAL(key)) + 1)) == (char *)NULL) + return -1; + /* first try, test password translated "standard" charset */ + r = testkey(__G__ h, STR_TO_CP1(key_translated, GLOBAL(key))); +#else /* !STR_TO_CP1 */ + /* first try, test password as supplied on the extractor's host */ + r = testkey(__G__ h, GLOBAL(key)); +#endif /* ?STR_TO_CP1 */ + +#ifdef STR_TO_CP2 + if (r != 0) { +#ifndef STR_TO_CP1 + /* now prepare for second (and maybe third) test with translated pwd */ + if ((key_translated = malloc(strlen(GLOBAL(key)) + 1)) == (char *)NULL) + return -1; +#endif + /* second try, password translated to alternate ("standard") charset */ + r = testkey(__G__ h, STR_TO_CP2(key_translated, GLOBAL(key))); +#ifdef STR_TO_CP3 + if (r != 0) + /* third try, password translated to another "standard" charset */ + r = testkey(__G__ h, STR_TO_CP3(key_translated, GLOBAL(key))); +#endif +#ifndef STR_TO_CP1 + free(key_translated); +#endif + } +#endif /* STR_TO_CP2 */ + +#ifdef STR_TO_CP1 + free(key_translated); + if (r != 0) { + /* last resort, test password as supplied on the extractor's host */ + r = testkey(__G__ h, GLOBAL(key)); + } +#endif /* STR_TO_CP1 */ + + return r; + +} /* end function testp() */ + + +local int testkey(__G__ h, key) + __GDEF + ZCONST uch *h; /* decrypted header */ + ZCONST char *key; /* decryption password to test */ +{ + ush b; +#ifdef ZIP10 + ush c; +#endif + int n; + uch *p; + uch hh[RAND_HEAD_LEN]; /* decrypted header */ + + /* set keys and save the encrypted header */ + init_keys(__G__ key); + memcpy(hh, h, RAND_HEAD_LEN); + + /* check password */ + for (n = 0; n < RAND_HEAD_LEN; n++) { + zdecode(hh[n]); + Trace((stdout, " %02x", hh[n])); + } + + /* use fzofft to format zoff_t as strings - 10/19/04 from SMS */ + Trace((stdout, + "\n lrec.crc= %08lx crec.crc= %08lx pInfo->ExtLocHdr= %s\n", + GLOBAL(lrec.crc32), GLOBAL(pInfo->crc), + GLOBAL(pInfo->ExtLocHdr) ? "true":"false")); + Trace((stdout, " incnt = %d unzip offset into zipfile = %s\n", + GLOBAL(incnt), + fzofft(GLOBAL(cur_zipfile_bufstart)+(GLOBAL(inptr)-GLOBAL(inbuf)), + NULL, NULL))); + + /* same test as in zipbare(): */ + +#ifdef ZIP10 /* check two bytes */ + c = hh[RAND_HEAD_LEN-2], b = hh[RAND_HEAD_LEN-1]; + Trace((stdout, + " (c | (b<<8)) = %04x (crc >> 16) = %04x lrec.time = %04x\n", + (ush)(c | (b<<8)), (ush)(GLOBAL(lrec.crc32) >> 16), + ((ush)GLOBAL(lrec.last_mod_dos_datetime) & 0xffff)))); + if ((ush)(c | (b<<8)) != (GLOBAL(pInfo->ExtLocHdr) ? + ((ush)GLOBAL(lrec.last_mod_dos_datetime) & 0xffff) : + (ush)(GLOBAL(lrec.crc32) >> 16))) + return -1; /* bad */ +#else + b = hh[RAND_HEAD_LEN-1]; + Trace((stdout, " b = %02x (crc >> 24) = %02x (lrec.time >> 8) = %02x\n", + b, (ush)(GLOBAL(lrec.crc32) >> 24), + ((ush)GLOBAL(lrec.last_mod_dos_datetime) >> 8) & 0xff)); + if (b != (GLOBAL(pInfo->ExtLocHdr) ? + ((ush)GLOBAL(lrec.last_mod_dos_datetime) >> 8) & 0xff : + (ush)(GLOBAL(lrec.crc32) >> 24))) + return -1; /* bad */ +#endif + /* password OK: decrypt current buffer contents before leaving */ + for (n = (zoff_t)GLOBAL(incnt) > GLOBAL(csize) ? + (int)GLOBAL(csize) : GLOBAL(incnt), + p = GLOBAL(inptr); n--; p++) + zdecode(*p); + return 0; /* OK */ + +} /* end function testkey() */ + +#endif /* UNZIP && !FUNZIP */ + +#else /* !CRYPT */ + +/* something "externally visible" to shut up compiler/linker warnings */ +int zcr_dummy; + +#endif /* ?CRYPT */ diff --git a/third_party/infozip/zip/crypt.h b/third_party/infozip/zip/crypt.h new file mode 100644 index 000000000..0d8c54abd --- /dev/null +++ b/third_party/infozip/zip/crypt.h @@ -0,0 +1,169 @@ +/* clang-format off */ +/* + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + crypt.h (full version) by Info-ZIP. Last revised: [see CR_VERSION_DATE] + + The main encryption/decryption source code for Info-Zip software was + originally written in Europe. To the best of our knowledge, it can + be freely distributed in both source and object forms from any country, + including the USA under License Exception TSU of the U.S. Export + Administration Regulations (section 740.13(e)) of 6 June 2002. + + NOTE on copyright history: + Previous versions of this source package (up to version 2.8) were + not copyrighted and put in the public domain. If you cannot comply + with the Info-Zip LICENSE, you may want to look for one of those + public domain versions. + */ + +#ifndef __crypt_h /* don't include more than once */ +#define __crypt_h + +#ifdef CRYPT +# undef CRYPT +#endif +/* + Logic of selecting "full crypt" code: + a) default behaviour: + - dummy crypt code when compiling UnZipSFX stub, to minimize size + - full crypt code when used to compile Zip, UnZip and fUnZip + b) USE_CRYPT defined: + - always full crypt code + c) NO_CRYPT defined: + - never full crypt code + NO_CRYPT takes precedence over USE_CRYPT + */ +#if defined(NO_CRYPT) +# define CRYPT 0 /* dummy version */ +#else +#if defined(USE_CRYPT) +# define CRYPT 1 /* full version */ +#else +#if !defined(SFX) +# define CRYPT 1 /* full version for zip and main unzip */ +#else +# define CRYPT 0 /* dummy version for unzip sfx */ +#endif +#endif /* ?USE_CRYPT */ +#endif /* ?NO_CRYPT */ + +#if CRYPT +/* full version */ + +#ifdef CR_BETA +# undef CR_BETA /* this is not a beta release */ +#endif + +#define CR_MAJORVER 2 +#define CR_MINORVER 91 +#ifdef CR_BETA +# define CR_BETA_VER "c BETA" +# define CR_VERSION_DATE "05 Jan 2007" /* last real code change */ +#else +# define CR_BETA_VER "" +# define CR_VERSION_DATE "05 Jan 2007" /* last public release date */ +# define CR_RELEASE +#endif + +#ifndef __G /* UnZip only, for now (DLL stuff) */ +# define __G +# define __G__ +# define __GDEF +# define __GPRO void +# define __GPRO__ +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) +# ifndef DOS_OS2_W32 +# define DOS_OS2_W32 +# endif +#endif + +#if defined(DOS_OS2_W32) || defined(__human68k__) +# ifndef DOS_H68_OS2_W32 +# define DOS_H68_OS2_W32 +# endif +#endif + +#if defined(VM_CMS) || defined(MVS) +# ifndef CMS_MVS +# define CMS_MVS +# endif +#endif + +/* To allow combining of Zip and UnZip static libraries in a single binary, + * the Zip and UnZip versions of the crypt core functions have to be named + * differently. + */ +#ifdef ZIP +# ifdef REALLY_SHORT_SYMS +# define decrypt_byte zdcrby +# else +# define decrypt_byte zp_decrypt_byte +# endif +# define update_keys zp_update_keys +# define init_keys zp_init_keys +#else /* !ZIP */ +# ifdef REALLY_SHORT_SYMS +# define decrypt_byte dcrbyt +# endif +#endif /* ?ZIP */ + +#define IZ_PWLEN 80 /* input buffer size for reading encryption key */ +#ifndef PWLEN /* for compatibility with previous zcrypt release... */ +# define PWLEN IZ_PWLEN +#endif +#define RAND_HEAD_LEN 12 /* length of encryption random header */ + +/* the crc_32_tab array has to be provided externally for the crypt calculus */ + +/* encode byte c, using temp t. Warning: c must not have side effects. */ +#define zencode(c,t) (t=decrypt_byte(__G), update_keys(c), t^(c)) + +/* decode byte c in place */ +#define zdecode(c) update_keys(__G__ c ^= decrypt_byte(__G)) + +int decrypt_byte OF((__GPRO)); +int update_keys OF((__GPRO__ int c)); +void init_keys OF((__GPRO__ ZCONST char *passwd)); + +#ifdef ZIP + void crypthead OF((ZCONST char *, ulg)); +# ifdef UTIL + int zipcloak OF((struct zlist far *, ZCONST char *)); + int zipbare OF((struct zlist far *, ZCONST char *)); +# else + unsigned zfwrite OF((zvoid *, extent, extent)); +# endif +#endif /* ZIP */ + +#if (defined(UNZIP) && !defined(FUNZIP)) + int decrypt OF((__GPRO__ ZCONST char *passwrd)); +#endif + +#ifdef FUNZIP + extern int encrypted; +# ifdef NEXTBYTE +# undef NEXTBYTE +# endif +# define NEXTBYTE \ + (encrypted? update_keys(__G__ getc(G.in)^decrypt_byte(__G)) : getc(G.in)) +#endif /* FUNZIP */ + +#else /* !CRYPT */ +/* dummy version */ + +#define zencode +#define zdecode + +#define zfwrite(b,s,c) bfwrite(b,s,c,BFWRITE_DATA) + +#endif /* ?CRYPT */ +#endif /* !__crypt_h */ diff --git a/third_party/infozip/zip/deflate.c b/third_party/infozip/zip/deflate.c new file mode 100644 index 000000000..c6b4b83f6 --- /dev/null +++ b/third_party/infozip/zip/deflate.c @@ -0,0 +1,930 @@ +/* clang-format off */ +/* + deflate.c - Zip 3 + + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * deflate.c by Jean-loup Gailly. + * + * PURPOSE + * + * Identify new text as repetitions of old text within a fixed- + * length sliding window trailing behind the new text. + * + * DISCUSSION + * + * The "deflation" process depends on being able to identify portions + * of the input text which are identical to earlier input (within a + * sliding window trailing behind the input currently being processed). + * + * The most straightforward technique turns out to be the fastest for + * most input files: try all possible matches and select the longest. + * The key feature of this algorithm is that insertions into the string + * dictionary are very simple and thus fast, and deletions are avoided + * completely. Insertions are performed at each input character, whereas + * string matches are performed only when the previous match ends. So it + * is preferable to spend more time in matches to allow very fast string + * insertions and avoid deletions. The matching algorithm for small + * strings is inspired from that of Rabin & Karp. A brute force approach + * is used to find longer strings when a small match has been found. + * A similar algorithm is used in comic (by Jan-Mark Wams) and freeze + * (by Leonid Broukhis). + * A previous version of this file used a more sophisticated algorithm + * (by Fiala and Greene) which is guaranteed to run in linear amortized + * time, but has a larger average cost, uses more memory and is patented. + * However the F&G algorithm may be faster for some highly redundant + * files if the parameter max_chain_length (described below) is too large. + * + * ACKNOWLEDGEMENTS + * + * The idea of lazy evaluation of matches is due to Jan-Mark Wams, and + * I found it in 'freeze' written by Leonid Broukhis. + * Thanks to many info-zippers for bug reports and testing. + * + * REFERENCES + * + * APPNOTE.TXT documentation file in PKZIP 1.93a distribution. + * + * A description of the Rabin and Karp algorithm is given in the book + * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. + * + * Fiala,E.R., and Greene,D.H. + * Data Compression with Finite Windows, Comm.ACM, 32,4 (1989) 490-595 + * + * INTERFACE + * + * void lm_init (int pack_level, ush *flags) + * Initialize the "longest match" routines for a new file + * + * ulg deflate (void) + * Processes a new input file and return its compressed length. Sets + * the compressed length, crc, deflate flags and internal file + * attributes. + */ + +#define __DEFLATE_C + +#include "third_party/infozip/zip/zip.h" + +#ifndef USE_ZLIB + +/* =========================================================================== + * Configuration parameters + */ + +/* Compile with MEDIUM_MEM to reduce the memory requirements or + * with SMALL_MEM to use as little memory as possible. Use BIG_MEM if the + * entire input file can be held in memory (not possible on 16 bit systems). + * Warning: defining these symbols affects HASH_BITS (see below) and thus + * affects the compression ratio. The compressed output + * is still correct, and might even be smaller in some cases. + */ + +#ifdef SMALL_MEM +# define HASH_BITS 13 /* Number of bits used to hash strings */ +#endif +#ifdef MEDIUM_MEM +# define HASH_BITS 14 +#endif +#ifndef HASH_BITS +# define HASH_BITS 15 + /* For portability to 16 bit machines, do not use values above 15. */ +#endif + +#define HASH_SIZE (unsigned)(1<= HASH_BITS + */ + +unsigned int near prev_length; +/* Length of the best match at previous step. Matches not greater than this + * are discarded. This is used in the lazy match evaluation. + */ + + unsigned near strstart; /* start of string to insert */ + unsigned near match_start; /* start of matching string */ +local int eofile; /* flag set at end of input file */ +local unsigned lookahead; /* number of valid bytes ahead in window */ + +unsigned near max_chain_length; +/* To speed up deflation, hash chains are never searched beyond this length. + * A higher limit improves compression ratio but degrades the speed. + */ + +local unsigned int max_lazy_match; +/* Attempt to find a better match only when the current match is strictly + * smaller than this value. This mechanism is used only for compression + * levels >= 4. + */ +#define max_insert_length max_lazy_match +/* Insert new strings in the hash table only if the match length + * is not greater than this length. This saves time but degrades compression. + * max_insert_length is used only for compression levels <= 3. + */ + +unsigned near good_match; +/* Use a faster search when the previous match is longer than this */ + +#ifdef FULL_SEARCH +# define nice_match MAX_MATCH +#else + int near nice_match; /* Stop searching when current match exceeds this */ +#endif + + +/* Values for max_lazy_match, good_match, nice_match and max_chain_length, + * depending on the desired pack level (0..9). The values given below have + * been tuned to exclude worst case performance for pathological files. + * Better values may be found for specific files. + */ + +typedef struct config { + ush good_length; /* reduce lazy search above this match length */ + ush max_lazy; /* do not perform lazy search above this match length */ + ush nice_length; /* quit search above this match length */ + ush max_chain; +} config; + +local config configuration_table[10] = { +/* good lazy nice chain */ +/* 0 */ {0, 0, 0, 0}, /* store only */ +/* 1 */ {4, 4, 8, 4}, /* maximum speed, no lazy matches */ +/* 2 */ {4, 5, 16, 8}, +/* 3 */ {4, 6, 32, 32}, + +/* 4 */ {4, 4, 16, 16}, /* lazy matches */ +/* 5 */ {8, 16, 32, 32}, +/* 6 */ {8, 16, 128, 128}, +/* 7 */ {8, 32, 128, 256}, +/* 8 */ {32, 128, 258, 1024}, +/* 9 */ {32, 258, 258, 4096}}; /* maximum compression */ + +/* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4 + * For deflate_fast() (levels <= 3) good is ignored and lazy has a different + * meaning. + */ + +#define EQUAL 0 +/* result of memcmp for equal strings */ + +/* =========================================================================== + * Prototypes for local functions. + */ + +local void fill_window OF((void)); + +local uzoff_t deflate_fast OF((void)); /* now use uzoff_t 7/24/04 EG */ + + int longest_match OF((IPos cur_match)); +#if defined(ASMV) && !defined(RISCOS) + void match_init OF((void)); /* asm code initialization */ +#endif + +#ifdef DEBUG +local void check_match OF((IPos start, IPos match, int length)); +#endif + +/* =========================================================================== + * Update a hash value with the given input byte + * IN assertion: all calls to to UPDATE_HASH are made with consecutive + * input characters, so that a running hash key can be computed from the + * previous key instead of complete recalculation each time. + */ +#define UPDATE_HASH(h,c) (h = (((h)< 0 if the input file is already read or + * mmap'ed in the window[] array, 0 otherwise. In the first case, + * window_size is sufficient to contain the whole input file plus + * MIN_LOOKAHEAD bytes (to avoid referencing memory beyond the end + * of window[] when looking for matches towards the end). + */ +void lm_init (pack_level, flags) + int pack_level; /* 0: store, 1: best speed, 9: best compression */ + ush *flags; /* general purpose bit flag */ +{ + register unsigned j; + + if (pack_level < 1 || pack_level > 9) error("bad pack level"); + + /* Do not slide the window if the whole input is already in memory + * (window_size > 0) + */ + sliding = 0; + if (window_size == 0L) { + sliding = 1; + window_size = (ulg)2L*WSIZE; + } + + /* Use dynamic allocation if compiler does not like big static arrays: */ +#ifdef DYN_ALLOC + if (window == NULL) { + window = (uch far *) zcalloc(WSIZE, 2*sizeof(uch)); + if (window == NULL) ziperr(ZE_MEM, "window allocation"); + } + if (prev == NULL) { + prev = (Pos far *) zcalloc(WSIZE, sizeof(Pos)); + head = (Pos far *) zcalloc(HASH_SIZE, sizeof(Pos)); + if (prev == NULL || head == NULL) { + ziperr(ZE_MEM, "hash table allocation"); + } + } +#endif /* DYN_ALLOC */ + + /* Initialize the hash table (avoiding 64K overflow for 16 bit systems). + * prev[] will be initialized on the fly. + */ + head[HASH_SIZE-1] = NIL; + memset((char*)head, NIL, (unsigned)(HASH_SIZE-1)*sizeof(*head)); + + /* Set the default configuration parameters: + */ + max_lazy_match = configuration_table[pack_level].max_lazy; + good_match = configuration_table[pack_level].good_length; +#ifndef FULL_SEARCH + nice_match = configuration_table[pack_level].nice_length; +#endif + max_chain_length = configuration_table[pack_level].max_chain; + if (pack_level <= 2) { + *flags |= FAST; + } else if (pack_level >= 8) { + *flags |= SLOW; + } + /* ??? reduce max_chain_length for binary files */ + + strstart = 0; + block_start = 0L; +#if defined(ASMV) && !defined(RISCOS) + match_init(); /* initialize the asm code */ +#endif + + j = WSIZE; +#ifndef MAXSEG_64K + if (sizeof(int) > 2) j <<= 1; /* Can read 64K in one step */ +#endif + lookahead = (*read_buf)((char*)window, j); + + if (lookahead == 0 || lookahead == (unsigned)EOF) { + eofile = 1, lookahead = 0; + return; + } + eofile = 0; + /* Make sure that we always have enough lookahead. This is important + * if input comes from a device such as a tty. + */ + if (lookahead < MIN_LOOKAHEAD) fill_window(); + + ins_h = 0; + for (j=0; j= 1 + */ +#ifndef ASMV +/* For 80x86 and 680x0 and ARM, an optimized version is in match.asm or + * match.S. The code is functionally equivalent, so you can use the C version + * if desired. + */ +int longest_match(cur_match) + IPos cur_match; /* current match */ +{ + unsigned chain_length = max_chain_length; /* max hash chain length */ + register uch far *scan = window + strstart; /* current string */ + register uch far *match; /* matched string */ + register int len; /* length of current match */ + int best_len = prev_length; /* best match length so far */ + IPos limit = strstart > (IPos)MAX_DIST ? strstart - (IPos)MAX_DIST : NIL; + /* Stop when cur_match becomes <= limit. To simplify the code, + * we prevent matches with the string of window index 0. + */ + +/* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. + * It is easy to get rid of this optimization if necessary. + */ +#if HASH_BITS < 8 || MAX_MATCH != 258 + error: Code too clever +#endif + +#ifdef UNALIGNED_OK + /* Compare two bytes at a time. Note: this is not always beneficial. + * Try with and without -DUNALIGNED_OK to check. + */ + register uch far *strend = window + strstart + MAX_MATCH - 1; + register ush scan_start = *(ush far *)scan; + register ush scan_end = *(ush far *)(scan+best_len-1); +#else + register uch far *strend = window + strstart + MAX_MATCH; + register uch scan_end1 = scan[best_len-1]; + register uch scan_end = scan[best_len]; +#endif + + /* Do not waste too much time if we already have a good match: */ + if (prev_length >= good_match) { + chain_length >>= 2; + } + + Assert(strstart <= window_size-MIN_LOOKAHEAD, "insufficient lookahead"); + + do { + Assert(cur_match < strstart, "no future"); + match = window + cur_match; + + /* Skip to next match if the match length cannot increase + * or if the match length is less than 2: + */ +#if (defined(UNALIGNED_OK) && MAX_MATCH == 258) + /* This code assumes sizeof(unsigned short) == 2. Do not use + * UNALIGNED_OK if your compiler uses a different size. + */ + if (*(ush far *)(match+best_len-1) != scan_end || + *(ush far *)match != scan_start) continue; + + /* It is not necessary to compare scan[2] and match[2] since they are + * always equal when the other bytes match, given that the hash keys + * are equal and that HASH_BITS >= 8. Compare 2 bytes at a time at + * strstart+3, +5, ... up to strstart+257. We check for insufficient + * lookahead only every 4th comparison; the 128th check will be made + * at strstart+257. If MAX_MATCH-2 is not a multiple of 8, it is + * necessary to put more guard bytes at the end of the window, or + * to check more often for insufficient lookahead. + */ + scan++, match++; + do { + } while (*(ush far *)(scan+=2) == *(ush far *)(match+=2) && + *(ush far *)(scan+=2) == *(ush far *)(match+=2) && + *(ush far *)(scan+=2) == *(ush far *)(match+=2) && + *(ush far *)(scan+=2) == *(ush far *)(match+=2) && + scan < strend); + /* The funny "do {}" generates better code on most compilers */ + + /* Here, scan <= window+strstart+257 */ + Assert(scan <= window+(unsigned)(window_size-1), "wild scan"); + if (*scan == *match) scan++; + + len = (MAX_MATCH - 1) - (int)(strend-scan); + scan = strend - (MAX_MATCH-1); + +#else /* UNALIGNED_OK */ + + if (match[best_len] != scan_end || + match[best_len-1] != scan_end1 || + *match != *scan || + *++match != scan[1]) continue; + + /* The check at best_len-1 can be removed because it will be made + * again later. (This heuristic is not always a win.) + * It is not necessary to compare scan[2] and match[2] since they + * are always equal when the other bytes match, given that + * the hash keys are equal and that HASH_BITS >= 8. + */ + scan += 2, match++; + + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + do { + } while (*++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + scan < strend); + + Assert(scan <= window+(unsigned)(window_size-1), "wild scan"); + + len = MAX_MATCH - (int)(strend - scan); + scan = strend - MAX_MATCH; + +#endif /* UNALIGNED_OK */ + + if (len > best_len) { + match_start = cur_match; + best_len = len; + if (len >= nice_match) break; +#ifdef UNALIGNED_OK + scan_end = *(ush far *)(scan+best_len-1); +#else + scan_end1 = scan[best_len-1]; + scan_end = scan[best_len]; +#endif + } + } while ((cur_match = prev[cur_match & WMASK]) > limit + && --chain_length != 0); + + return best_len; +} +#endif /* ASMV */ + +#ifdef DEBUG +/* =========================================================================== + * Check that the match at match_start is indeed a match. + */ +local void check_match(start, match, length) + IPos start, match; + int length; +{ + /* check that the match is indeed a match */ + if (memcmp((char*)window + match, + (char*)window + start, length) != EQUAL) { + fprintf(mesg, + " start %d, match %d, length %d\n", + start, match, length); + error("invalid match"); + } + if (verbose > 1) { + fprintf(mesg,"\\[%d,%d]", start-match, length); +#ifndef WINDLL + do { putc(window[start++], mesg); } while (--length != 0); +#else + do { fprintf(stdout,"%c",window[start++]); } while (--length != 0); +#endif + } +} +#else +# define check_match(start, match, length) +#endif + +/* =========================================================================== + * Flush the current block, with given end-of-file flag. + * IN assertion: strstart is set to the end of the current match. + */ +#define FLUSH_BLOCK(eof) \ + flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \ + (char*)NULL, (ulg)strstart - (ulg)block_start, (eof)) + +/* =========================================================================== + * Fill the window when the lookahead becomes insufficient. + * Updates strstart and lookahead, and sets eofile if end of input file. + * + * IN assertion: lookahead < MIN_LOOKAHEAD && strstart + lookahead > 0 + * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD + * At least one byte has been read, or eofile is set; file reads are + * performed for at least two bytes (required for the translate_eol option). + */ +local void fill_window() +{ + register unsigned n, m; + unsigned more; /* Amount of free space at the end of the window. */ + + do { + more = (unsigned)(window_size - (ulg)lookahead - (ulg)strstart); + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (more == (unsigned)EOF) { + /* Very unlikely, but possible on 16 bit machine if strstart == 0 + * and lookahead == 1 (input done one byte at time) + */ + more--; + + /* For MMAP or BIG_MEM, the whole input file is already in memory so + * we must not perform sliding. We must however call (*read_buf)() in + * order to compute the crc, update lookahead and possibly set eofile. + */ + } else if (strstart >= WSIZE+MAX_DIST && sliding) { + +#ifdef FORCE_METHOD + /* When methods "stored" or "store_block" are requested, the + * current block must be flushed before sliding the window. + */ + if (level <= 2) FLUSH_BLOCK(0), block_start = strstart; +#endif + /* By the IN assertion, the window is not empty so we can't confuse + * more == 0 with more == 64K on a 16 bit machine. + */ + memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE); + match_start -= WSIZE; + strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */ + + block_start -= (long) WSIZE; + + for (n = 0; n < HASH_SIZE; n++) { + m = head[n]; + head[n] = (Pos)(m >= WSIZE ? m-WSIZE : NIL); + } + for (n = 0; n < WSIZE; n++) { + m = prev[n]; + prev[n] = (Pos)(m >= WSIZE ? m-WSIZE : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } + more += WSIZE; + if (dot_size > 0 && !display_globaldots) { + /* initial space */ + if (noisy && dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + dot_count++; + } + dot_count++; + if (dot_size <= (dot_count + 1) * WSIZE) dot_count = 0; + } + if ((verbose || noisy) && dot_size && !dot_count) { +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + if (eofile) return; + + /* If there was no sliding: + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && + * more == window_size - lookahead - strstart + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) + * => more >= window_size - 2*WSIZE + 2 + * In the MMAP or BIG_MEM case (not yet supported in gzip), + * window_size == input_size + MIN_LOOKAHEAD && + * strstart + lookahead <= input_size => more >= MIN_LOOKAHEAD. + * Otherwise, window_size == 2*WSIZE so more >= 2. + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. + */ + Assert(more >= 2, "more < 2"); + + n = (*read_buf)((char*)window+strstart+lookahead, more); + if (n == 0 || n == (unsigned)EOF) { + eofile = 1; + } else { + lookahead += n; + } + } while (lookahead < MIN_LOOKAHEAD && !eofile); +} + +/* =========================================================================== + * Processes a new input file and return its compressed length. This + * function does not perform lazy evaluation of matches and inserts + * new strings in the dictionary only for unmatched strings or for short + * matches. It is used only for the fast compression options. + */ +local uzoff_t deflate_fast() +{ + IPos hash_head = NIL; /* head of the hash chain */ + int flush; /* set if current block must be flushed */ + unsigned match_length = 0; /* length of best match */ + + prev_length = MIN_MATCH-1; + while (lookahead != 0) { + /* Insert the string window[strstart .. strstart+2] in the + * dictionary, and set hash_head to the head of the hash chain: + */ +#ifndef DEFL_UNDETERM + if (lookahead >= MIN_MATCH) +#endif + INSERT_STRING(strstart, hash_head); + + /* Find the longest match, discarding those <= prev_length. + * At this point we have always match_length < MIN_MATCH + */ + if (hash_head != NIL && strstart - hash_head <= MAX_DIST) { + /* To simplify the code, we prevent matches with the string + * of window index 0 (in particular we have to avoid a match + * of the string with itself at the start of the input file). + */ +#ifndef HUFFMAN_ONLY +# ifndef DEFL_UNDETERM + /* Do not look for matches beyond the end of the input. + * This is necessary to make deflate deterministic. + */ + if ((unsigned)nice_match > lookahead) nice_match = (int)lookahead; +# endif + match_length = longest_match (hash_head); + /* longest_match() sets match_start */ + if (match_length > lookahead) match_length = lookahead; +#endif + } + if (match_length >= MIN_MATCH) { + check_match(strstart, match_start, match_length); + + flush = ct_tally(strstart-match_start, match_length - MIN_MATCH); + + lookahead -= match_length; + + /* Insert new strings in the hash table only if the match length + * is not too large. This saves time but degrades compression. + */ + if (match_length <= max_insert_length +#ifndef DEFL_UNDETERM + && lookahead >= MIN_MATCH +#endif + ) { + match_length--; /* string at strstart already in hash table */ + do { + strstart++; + INSERT_STRING(strstart, hash_head); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. + */ +#ifdef DEFL_UNDETERM + /* If lookahead < MIN_MATCH these bytes are garbage, + * but it does not matter since the next lookahead bytes + * will be emitted as literals. + */ +#endif + } while (--match_length != 0); + strstart++; + } else { + strstart += match_length; + match_length = 0; + ins_h = window[strstart]; + UPDATE_HASH(ins_h, window[strstart+1]); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + } + } else { + /* No match, output a literal byte */ + Tracevv((stderr,"%c",window[strstart])); + flush = ct_tally (0, window[strstart]); + lookahead--; + strstart++; + } + if (flush) FLUSH_BLOCK(0), block_start = strstart; + + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the next match, plus MIN_MATCH bytes to insert the + * string following the next match. + */ + if (lookahead < MIN_LOOKAHEAD) fill_window(); + } + return FLUSH_BLOCK(1); /* eof */ +} + +/* =========================================================================== + * Same as above, but achieves better compression. We use a lazy + * evaluation for matches: a match is finally adopted only if there is + * no better match at the next window position. + */ +uzoff_t deflate() +{ + IPos hash_head = NIL; /* head of hash chain */ + IPos prev_match; /* previous match */ + int flush; /* set if current block must be flushed */ + int match_available = 0; /* set if previous match exists */ + register unsigned match_length = MIN_MATCH-1; /* length of best match */ +#ifdef DEBUG + extern uzoff_t isize; /* byte length of input file, for debug only */ +#endif + + if (level <= 3) return deflate_fast(); /* optimized for speed */ + + /* Process the input block. */ + while (lookahead != 0) { + /* Insert the string window[strstart .. strstart+2] in the + * dictionary, and set hash_head to the head of the hash chain: + */ +#ifndef DEFL_UNDETERM + if (lookahead >= MIN_MATCH) +#endif + INSERT_STRING(strstart, hash_head); + + /* Find the longest match, discarding those <= prev_length. + */ + prev_length = match_length, prev_match = match_start; + match_length = MIN_MATCH-1; + + if (hash_head != NIL && prev_length < max_lazy_match && + strstart - hash_head <= MAX_DIST) { + /* To simplify the code, we prevent matches with the string + * of window index 0 (in particular we have to avoid a match + * of the string with itself at the start of the input file). + */ +#ifndef HUFFMAN_ONLY +# ifndef DEFL_UNDETERM + /* Do not look for matches beyond the end of the input. + * This is necessary to make deflate deterministic. + */ + if ((unsigned)nice_match > lookahead) nice_match = (int)lookahead; +# endif + match_length = longest_match (hash_head); + /* longest_match() sets match_start */ + if (match_length > lookahead) match_length = lookahead; +#endif + +#ifdef FILTERED + /* Ignore matches of length <= 5 */ + if (match_length <= 5) { +#else + /* Ignore a length 3 match if it is too distant: */ + if (match_length == MIN_MATCH && strstart-match_start > TOO_FAR){ +#endif + /* If prev_match is also MIN_MATCH, match_start is garbage + * but we will ignore the current match anyway. + */ + match_length = MIN_MATCH-1; + } + } + /* If there was a match at the previous step and the current + * match is not better, output the previous match: + */ + if (prev_length >= MIN_MATCH && match_length <= prev_length) { +#ifndef DEFL_UNDETERM + unsigned max_insert = strstart + lookahead - MIN_MATCH; + +#endif + check_match(strstart-1, prev_match, prev_length); + + flush = ct_tally(strstart-1-prev_match, prev_length - MIN_MATCH); + + /* Insert in hash table all strings up to the end of the match. + * strstart-1 and strstart are already inserted. + */ + lookahead -= prev_length-1; + prev_length -= 2; +#ifndef DEFL_UNDETERM + do { + if (++strstart <= max_insert) { + INSERT_STRING(strstart, hash_head); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. + */ + } + } while (--prev_length != 0); + strstart++; +#else /* DEFL_UNDETERM */ + do { + strstart++; + INSERT_STRING(strstart, hash_head); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. If lookahead < MIN_MATCH + * these bytes are garbage, but it does not matter since the + * next lookahead bytes will always be emitted as literals. + */ + } while (--prev_length != 0); + strstart++; +#endif /* ?DEFL_UNDETERM */ + match_available = 0; + match_length = MIN_MATCH-1; + + if (flush) FLUSH_BLOCK(0), block_start = strstart; + + } else if (match_available) { + /* If there was no match at the previous position, output a + * single literal. If there was a match but the current match + * is longer, truncate the previous match to a single literal. + */ + Tracevv((stderr,"%c",window[strstart-1])); + if (ct_tally (0, window[strstart-1])) { + FLUSH_BLOCK(0), block_start = strstart; + } + strstart++; + lookahead--; + } else { + /* There is no previous match to compare with, wait for + * the next step to decide. + */ + match_available = 1; + strstart++; + lookahead--; + } + Assert(strstart <= isize && lookahead <= isize, "a bit too far"); + + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the next match, plus MIN_MATCH bytes to insert the + * string following the next match. + */ + if (lookahead < MIN_LOOKAHEAD) fill_window(); + } + if (match_available) ct_tally (0, window[strstart-1]); + + return FLUSH_BLOCK(1); /* eof */ +} +#endif /* !USE_ZLIB */ diff --git a/third_party/infozip/zip/ebcdic.h b/third_party/infozip/zip/ebcdic.h new file mode 100644 index 000000000..044097965 --- /dev/null +++ b/third_party/infozip/zip/ebcdic.h @@ -0,0 +1,329 @@ +/* clang-format off */ +/* + ebcdic.h + + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, both of these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/licen +*/ +/*--------------------------------------------------------------------------- + + ebcdic.h + + The CECP 1047 (Extended de-facto EBCDIC) <-> ISO 8859-1 conversion tables, + from ftp://aix1.segi.ulg.ac.be/pub/docs/iso8859/iso8859.networking + + NOTES: + (OS/390 port 12/97) + These table no longer represent the standard mappings (for example in the + OS/390 iconv utility). In order to follow current standards I remapped + ebcdic x0a to ascii x15 and + ebcdic x85 to ascii x25 (and vice-versa) + Without these changes, newlines in auto-convert text files appeared + as literal \045. + I'm not sure what effect this remap would have on the MVS and CMS ports, so + I ifdef'd these changes. Hopefully these ifdef's can be removed when the + MVS/CMS folks test the new mappings. + + Christian Spieler , 27-Apr-1998 + The problem mentioned by Paul von Behren was already observed previously + on VM/CMS, during the preparation of the CMS&MVS port of UnZip 5.20 in + 1996. At that point, the ebcdic tables were not changed since they seemed + to be an adopted standard (to my knowledge, these tables are still used + as presented in mainfraime KERMIT). Instead, the "end-of-line" conversion + feature of Zip's and UnZip's "text-translation" mode was used to force + correct mappings between ASCII and EBCDIC newline markers. + Before interchanging the ASCII mappings of the EBCDIC control characters + "NL" 0x25 and "LF" 0x15 according to the OS/390 setting, we have to + make sure that EBCDIC 0x15 is never used as line termination. + + ---------------------------------------------------------------------------*/ + +#ifndef __ebcdic_h /* prevent multiple inclusions */ +#define __ebcdic_h + + +#ifndef ZCONST +# define ZCONST const +#endif + +#ifdef EBCDIC +#ifndef MTS /* MTS uses a slightly "special" EBCDIC code page */ + +ZCONST uch ebcdic[] = { + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, /* 00 - 07 */ +#ifdef OS390 + 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 08 - 0F */ +#else + 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 08 - 0F */ +#endif + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, /* 10 - 17 */ + 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, /* 18 - 1F */ + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, /* 20 - 27 */ + 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, /* 28 - 2F */ + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, /* 30 - 37 */ + 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, /* 38 - 3F */ + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, /* 40 - 47 */ + 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, /* 48 - 4F */ + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, /* 50 - 57 */ + 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, /* 58 - 5F */ + 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 60 - 67 */ + 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, /* 68 - 6F */ + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, /* 70 - 77 */ + 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, /* 78 - 7F */ +#ifdef OS390 + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, /* 80 - 87 */ +#else + 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, /* 80 - 87 */ +#endif + 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, /* 88 - 8F */ + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, /* 90 - 97 */ + 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, /* 98 - 9F */ + 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, /* A0 - A7 */ + 0xBB, 0xB4, 0x9A, 0x8A, 0xB0, 0xCA, 0xAF, 0xBC, /* A8 - AF */ + 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, /* B0 - B7 */ + 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, /* B8 - BF */ + 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, /* C0 - C7 */ + 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, /* C8 - CF */ + 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, /* D0 - D7 */ + 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xBA, 0xAE, 0x59, /* D8 - DF */ + 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, /* E0 - E7 */ + 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, /* E8 - EF */ + 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, /* F0 - F7 */ + 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF /* F8 - FF */ +}; + +#if (defined(ZIP) || CRYPT) +ZCONST uch ascii[] = { + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, /* 00 - 07 */ + 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 08 - 0F */ +#ifdef OS390 + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, /* 10 - 17 */ +#else + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, /* 10 - 17 */ +#endif + 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, /* 18 - 1F */ +#ifdef OS390 + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, /* 20 - 27 */ +#else + 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, /* 20 - 27 */ +#endif + 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, /* 28 - 2F */ + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, /* 30 - 37 */ + 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, /* 38 - 3F */ + 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, /* 40 - 47 */ + 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, /* 48 - 4F */ + 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, /* 50 - 57 */ + 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, /* 58 - 5F */ + 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, /* 60 - 67 */ + 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, /* 68 - 6F */ + 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, /* 70 - 77 */ + 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, /* 78 - 7F */ + 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 80 - 87 */ + 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, /* 88 - 8F */ + 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, /* 90 - 97 */ + 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, /* 98 - 9F */ + 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, /* A0 - A7 */ + 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE, /* A8 - AF */ + 0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, /* B0 - B7 */ + 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7, /* B8 - BF */ + 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* C0 - C7 */ + 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, /* C8 - CF */ + 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, /* D0 - D7 */ + 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, /* D8 - DF */ + 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, /* E0 - E7 */ + 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, /* E8 - EF */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* F0 - F7 */ + 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F /* F8 - FF */ +}; +#endif /* ZIP || CRYPT */ + +#else /* MTS */ + +/* + * This is the MTS ASCII->EBCDIC translation table. It provides a 1-1 + * translation from ISO 8859/1 8-bit ASCII to IBM Code Page 37 EBCDIC. + */ + +ZCONST uch ebcdic[] = { + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, /* 00 - 07 */ + 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 08 - 0F */ + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, /* 10 - 17 */ + 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, /* 18 - 1F */ + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, /* 20 - 27 */ + 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, /* 28 - 2F */ + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, /* 30 - 37 */ + 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, /* 38 - 3F */ + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, /* 40 - 47 */ + 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, /* 48 - 4F */ + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, /* 50 - 57 */ + 0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D, /* 58 - 5F */ + 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 60 - 67 */ + 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, /* 68 - 6F */ + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, /* 70 - 77 */ + 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, /* 78 - 7F */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, /* 80 - 87 */ + 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, /* 88 - 8F */ + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, /* 90 - 97 */ + 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, /* 98 - 9F */ + 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, /* A0 - A7 */ + 0xBD, 0xB4, 0x9A, 0x8A, 0x5F, 0xCA, 0xAF, 0xBC, /* A8 - AF */ + 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, /* B0 - B7 */ + 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, /* B8 - BF */ + 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, /* C0 - C7 */ + 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, /* C8 - CF */ + 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, /* D0 - D7 */ + 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xAD, 0xAE, 0x59, /* D8 - DF */ + 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, /* E0 - E7 */ + 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, /* E8 - EF */ + 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, /* F0 - F7 */ + 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF /* F8 - FF */ +}; + +#if (defined(ZIP) || CRYPT) +ZCONST uch ascii[] = { + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, /* 00 - 07 */ + 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 08 - 0F */ + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, /* 10 - 17 */ + 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, /* 18 - 1F */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, /* 20 - 27 */ + 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, /* 28 - 2F */ + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, /* 30 - 37 */ + 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, /* 38 - 3F */ + 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, /* 40 - 47 */ + 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, /* 48 - 4F */ + 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, /* 50 - 57 */ + 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0xAC, /* 58 - 5F */ + 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, /* 60 - 67 */ + 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, /* 68 - 6F */ + 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, /* 70 - 77 */ + 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, /* 78 - 7F */ + 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 80 - 87 */ + 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, /* 88 - 8F */ + 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, /* 90 - 97 */ + 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, /* 98 - 9F */ + 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, /* A0 - A7 */ + 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE, /* A8 - AF */ + 0x5E, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, /* B0 - B7 */ + 0xBD, 0xBE, 0x5B, 0x5D, 0xAF, 0xA8, 0xB4, 0xD7, /* B8 - BF */ + 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* C0 - C7 */ + 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, /* C8 - CF */ + 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, /* D0 - D7 */ + 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, /* D8 - DF */ + 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, /* E0 - E7 */ + 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, /* E8 - EF */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* F0 - F7 */ + 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F /* F8 - FF */ +}; +#endif /* ZIP || CRYPT */ + +#endif /* ?MTS */ +#endif /* EBCDIC */ + +/*--------------------------------------------------------------------------- + + The following conversion tables translate between IBM PC CP 850 + (OEM codepage) and the "Western Europe & America" Windows codepage 1252. + The Windows codepage 1252 contains the ISO 8859-1 "Latin 1" codepage, + with some additional printable characters in the range (0x80 - 0x9F), + that is reserved to control codes in the ISO 8859-1 character table. + + The ISO <--> OEM conversion tables were constructed with the help + of the WIN32 (Win16?) API's OemToAnsi() and AnsiToOem() conversion + functions and have been checked against the CP850 and LATIN1 tables + provided in the MS-Kermit 3.14 distribution. + + ---------------------------------------------------------------------------*/ + +#ifdef IZ_ISO2OEM_ARRAY +#ifdef OEM_RUSS +ZCONST uch Far iso2oem[] = { + 0x3F, 0x3F, 0x27, 0x9F, 0x22, 0x2E, 0xC5, 0xCE, /* 80 - 87 */ + 0xFD, 0x25, 0x53, 0x3C, 0x4F, 0x3F, 0x3F, 0x3F, /* 88 - 8F */ + 0x3F, 0x27, 0x27, 0x22, 0x22, 0xF9, 0x2D, 0x2D, /* 90 - 97 */ + 0x7E, 0x54, 0x73, 0x3E, 0x6F, 0x3F, 0x3F, 0x59, /* 98 - 9F */ + 0xFF, 0xF6, 0xF7, 0x9C, 0xCF, 0xBE, 0xFE, 0xF5, /* A0 - A7 */ + 0xF0, 0xB8, 0xF2, 0xAE, 0xAA, 0xF0, 0xA9, 0xEE, /* A8 - AF */ + 0xF8, 0xFB, 0xF4, 0xF5, 0xEF, 0xE6, 0xF4, 0xFA, /* B0 - B7 */ + 0xF1, 0xFC, 0xF3, 0xAF, 0xAC, 0xAB, 0xF3, 0xA8, /* B8 - BF */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x86, 0x86, 0x87, /* C0 - C7 */ + 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, /* C8 - CF */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* D0 - D7 */ + 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, /* D8 - DF */ + 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, /* E0 - E7 */ + 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, /* E8 - EF */ + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, /* F0 - F7 */ + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF /* F8 - FF */ +}; +#else /* OEM_RUS */ +ZCONST uch Far iso2oem[] = { + 0x3F, 0x3F, 0x27, 0x9F, 0x22, 0x2E, 0xC5, 0xCE, /* 80 - 87 */ + 0x5E, 0x25, 0x53, 0x3C, 0x4F, 0x3F, 0x3F, 0x3F, /* 88 - 8F */ + 0x3F, 0x27, 0x27, 0x22, 0x22, 0x07, 0x2D, 0x2D, /* 90 - 97 */ + 0x7E, 0x54, 0x73, 0x3E, 0x6F, 0x3F, 0x3F, 0x59, /* 98 - 9F */ + 0xFF, 0xAD, 0xBD, 0x9C, 0xCF, 0xBE, 0xDD, 0xF5, /* A0 - A7 */ + 0xF9, 0xB8, 0xA6, 0xAE, 0xAA, 0xF0, 0xA9, 0xEE, /* A8 - AF */ + 0xF8, 0xF1, 0xFD, 0xFC, 0xEF, 0xE6, 0xF4, 0xFA, /* B0 - B7 */ + 0xF7, 0xFB, 0xA7, 0xAF, 0xAC, 0xAB, 0xF3, 0xA8, /* B8 - BF */ + 0xB7, 0xB5, 0xB6, 0xC7, 0x8E, 0x8F, 0x92, 0x80, /* C0 - C7 */ + 0xD4, 0x90, 0xD2, 0xD3, 0xDE, 0xD6, 0xD7, 0xD8, /* C8 - CF */ + 0xD1, 0xA5, 0xE3, 0xE0, 0xE2, 0xE5, 0x99, 0x9E, /* D0 - D7 */ + 0x9D, 0xEB, 0xE9, 0xEA, 0x9A, 0xED, 0xE8, 0xE1, /* D8 - DF */ + 0x85, 0xA0, 0x83, 0xC6, 0x84, 0x86, 0x91, 0x87, /* E0 - E7 */ + 0x8A, 0x82, 0x88, 0x89, 0x8D, 0xA1, 0x8C, 0x8B, /* E8 - EF */ + 0xD0, 0xA4, 0x95, 0xA2, 0x93, 0xE4, 0x94, 0xF6, /* F0 - F7 */ + 0x9B, 0x97, 0xA3, 0x96, 0x81, 0xEC, 0xE7, 0x98 /* F8 - FF */ +}; +#endif /* OEM_RUS */ +#endif /* IZ_ISO2OEM_ARRAY */ + +#ifdef IZ_OEM2ISO_ARRAY +#ifdef OEM_RUSS +ZCONST uch Far oem2iso[] = { + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, /* 80 - 87 */ + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, /* 88 - 8F */ + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, /* 90 - 97 */ + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, /* 98 - 9F */ + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, /* A0 - A7 */ + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, /* A8 - AF */ + 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xC1, 0xC2, 0xC0, /* B0 - B7 */ + 0xA9, 0xA6, 0xA6, 0x2B, 0x2B, 0xA2, 0xA5, 0x2B, /* B8 - BF */ + 0x2B, 0x2D, 0x2D, 0x2B, 0x2D, 0x2B, 0xE3, 0xC3, /* C0 - C7 */ + 0x2B, 0x2B, 0x2D, 0x2D, 0xA6, 0x2D, 0x2B, 0xA4, /* C8 - CF */ + 0xF0, 0xD0, 0xCA, 0xCB, 0xC8, 0x69, 0xCD, 0xCE, /* D0 - D7 */ + 0xCF, 0x2B, 0x2B, 0xA6, 0x5F, 0xA6, 0xCC, 0xAF, /* D8 - DF */ + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, /* E0 - E7 */ + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, /* E8 - EF */ + 0xA8, 0xB8, 0xAA, 0xBA, 0xB2, 0xB3, 0xA1, 0xA2, /* F0 - F7 */ + 0xB0, 0x95, 0xB7, 0xB1, 0xB9, 0x88, 0xA6, 0xA0 /* F8 - FF */ +}; +#else /* OEM_RUS */ +ZCONST uch Far oem2iso[] = { + 0xC7, 0xFC, 0xE9, 0xE2, 0xE4, 0xE0, 0xE5, 0xE7, /* 80 - 87 */ + 0xEA, 0xEB, 0xE8, 0xEF, 0xEE, 0xEC, 0xC4, 0xC5, /* 88 - 8F */ + 0xC9, 0xE6, 0xC6, 0xF4, 0xF6, 0xF2, 0xFB, 0xF9, /* 90 - 97 */ + 0xFF, 0xD6, 0xDC, 0xF8, 0xA3, 0xD8, 0xD7, 0x83, /* 98 - 9F */ + 0xE1, 0xED, 0xF3, 0xFA, 0xF1, 0xD1, 0xAA, 0xBA, /* A0 - A7 */ + 0xBF, 0xAE, 0xAC, 0xBD, 0xBC, 0xA1, 0xAB, 0xBB, /* A8 - AF */ + 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xC1, 0xC2, 0xC0, /* B0 - B7 */ + 0xA9, 0xA6, 0xA6, 0x2B, 0x2B, 0xA2, 0xA5, 0x2B, /* B8 - BF */ + 0x2B, 0x2D, 0x2D, 0x2B, 0x2D, 0x2B, 0xE3, 0xC3, /* C0 - C7 */ + 0x2B, 0x2B, 0x2D, 0x2D, 0xA6, 0x2D, 0x2B, 0xA4, /* C8 - CF */ + 0xF0, 0xD0, 0xCA, 0xCB, 0xC8, 0x69, 0xCD, 0xCE, /* D0 - D7 */ + 0xCF, 0x2B, 0x2B, 0xA6, 0x5F, 0xA6, 0xCC, 0xAF, /* D8 - DF */ + 0xD3, 0xDF, 0xD4, 0xD2, 0xF5, 0xD5, 0xB5, 0xFE, /* E0 - E7 */ + 0xDE, 0xDA, 0xDB, 0xD9, 0xFD, 0xDD, 0xAF, 0xB4, /* E8 - EF */ + 0xAD, 0xB1, 0x3D, 0xBE, 0xB6, 0xA7, 0xF7, 0xB8, /* F0 - F7 */ + 0xB0, 0xA8, 0xB7, 0xB9, 0xB3, 0xB2, 0xA6, 0xA0 /* F8 - FF */ +}; +#endif /* OEM_RUS */ +#endif /* IZ_OEM2ISO_ARRAY */ + +#if defined(THEOS) || defined(THEOS_SUPPORT) +# include "theos/charconv.h" +#endif + +#endif /* __ebcdic_h */ diff --git a/third_party/infozip/zip/fileio.c b/third_party/infozip/zip/fileio.c new file mode 100644 index 000000000..6450d5477 --- /dev/null +++ b/third_party/infozip/zip/fileio.c @@ -0,0 +1,4909 @@ +/* clang-format off */ +/* + fileio.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * fileio.c by Mark Adler + */ +#define __FILEIO_C + +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/crc32.h" + +#ifdef MACOS +# include "helpers.h" +#endif + +#ifdef VMS +# include "vms/vms.h" +#endif /* def VMS */ + +#include "libc/fmt/fmt.h" +#include "libc/alg/alg.h" +#include "libc/stdio/temp.h" +#include "libc/time/time.h" +#include "libc/time/struct/tm.h" +#include "libc/limits.h" + +#ifdef NO_MKTIME +time_t mktime OF((struct tm *)); +#endif + +#ifdef OSF +#define EXDEV 18 /* avoid a bug in the DEC OSF/1 header files. */ +#else +#include "libc/errno.h" +#endif + +#ifdef NO_ERRNO +extern int errno; +#endif + +/* ----------------------- + For long option support + ----------------------- */ +#include "libc/str/str.h" + + +#if defined(VMS) || defined(TOPS20) +# define PAD 5 +#else +# define PAD 0 +#endif + +#ifdef NO_RENAME +int rename OF((ZCONST char *, ZCONST char *)); +#endif + + +/* Local functions */ +local int optionerr OF((char *, ZCONST char *, int, int)); +local unsigned long get_shortopt OF((char **, int, int *, int *, char **, int *, int)); +local unsigned long get_longopt OF((char **, int, int *, int *, char **, int *, int)); + +#ifdef UNICODE_SUPPORT +local int utf8_char_bytes OF((ZCONST char *utf8)); +local long ucs4_char_from_utf8 OF((ZCONST char **utf8 )); +local int utf8_from_ucs4_char OF((char *utf8buf, ulg ch)); +local int utf8_to_ucs4_string OF((ZCONST char *utf8, ulg *usc4buf, + int buflen)); +local int ucs4_string_to_utf8 OF((ZCONST ulg *ucs4, char *utf8buf, + int buflen)); +#if 0 + local int utf8_chars OF((ZCONST char *utf8)); +#endif +#endif /* UNICODE_SUPPORT */ + +#ifndef UTIL /* the companion #endif is a bit of ways down ... */ + +local int fqcmp OF((ZCONST zvoid *, ZCONST zvoid *)); +local int fqcmpz OF((ZCONST zvoid *, ZCONST zvoid *)); + + +/* Local module level variables. */ +char *label = NULL; /* global, but only used in `system'.c */ +local z_stat zipstatb; /* now use z_stat globally - 7/24/04 EG */ +#if defined(UNICODE_SUPPORT) && defined(WIN32) + local zw_stat zipstatbw; +#endif +#if (!defined(MACOS) && !defined(WINDLL)) +local int zipstate = -1; +#else +int zipstate; +#endif +/* -1 unknown, 0 old zip file exists, 1 new zip file */ + +#if 0 +char *getnam(n, fp) +char *n; /* where to put name (must have >=FNMAX+1 bytes) */ +#endif + +/* converted to return string pointer from malloc to avoid + size limitation - 11/8/04 EG */ +#define GETNAM_MAX 9000 /* hopefully big enough for now */ +char *getnam(fp) + FILE *fp; + /* Read a \n or \r delimited name from stdin into n, and return + n. If EOF, then return NULL. Also, if problem return NULL. */ +{ + char name[GETNAM_MAX + 1]; + int c; /* last character read */ + char *p; /* pointer into name area */ + + + p = name; + while ((c = getc(fp)) == '\n' || c == '\r') + ; + if (c == EOF) + return NULL; + do { + if (p - name >= GETNAM_MAX) + return NULL; + *p++ = (char) c; + c = getc(fp); + } while (c != EOF && (c != '\n' && c != '\r')); +#ifdef WIN32 +/* + * WIN32 strips off trailing spaces and periods in filenames + * XXX what about a filename that only consists of spaces ? + * Answer: on WIN32, a filename must contain at least one non-space char + */ + while (p > name) { + if ((c = p[-1]) != ' ' && c != '.') + break; + --p; + } +#endif + *p = 0; + /* malloc a copy */ + if ((p = malloc(strlen(name) + 1)) == NULL) { + return NULL; + } + strcpy(p, name); + return p; +} + +struct flist far *fexpel(f) +struct flist far *f; /* entry to delete */ +/* Delete the entry *f in the doubly-linked found list. Return pointer to + next entry to allow stepping through list. */ +{ + struct flist far *t; /* temporary variable */ + + t = f->nxt; + *(f->lst) = t; /* point last to next, */ + if (t != NULL) + t->lst = f->lst; /* and next to last */ + if (f->name != NULL) /* free memory used */ + free((zvoid *)(f->name)); + if (f->zname != NULL) + free((zvoid *)(f->zname)); + if (f->iname != NULL) + free((zvoid *)(f->iname)); +#ifdef UNICODE_SUPPORT + if (f->uname) + free((zvoid *)f->uname); +# ifdef WIN32 + if (f->namew) + free((zvoid *)f->namew); + if (f->inamew) + free((zvoid *)f->inamew); + if (f->znamew) + free((zvoid *)f->znamew); +# endif +#endif + farfree((zvoid far *)f); + fcount--; /* decrement count */ + return t; /* return pointer to next */ +} + +local int fqcmp(a, b) + ZCONST zvoid *a, *b; /* pointers to pointers to found entries */ +/* Used by qsort() to compare entries in the found list by name. */ +{ + return strcmp((*(struct flist far **)a)->name, + (*(struct flist far **)b)->name); +} + +local int fqcmpz(a, b) + ZCONST zvoid *a, *b; /* pointers to pointers to found entries */ +/* Used by qsort() to compare entries in the found list by iname. */ +{ + return strcmp((*(struct flist far **)a)->iname, + (*(struct flist far **)b)->iname); +} + +char *last(p, c) + char *p; /* sequence of path components */ + int c; /* path components separator character */ +/* Return a pointer to the start of the last path component. For a directory + * name terminated by the character in c, the return value is an empty string. + */ +{ + char *t; /* temporary variable */ + + if ((t = strrchr(p, c)) != NULL) + return t + 1; + else +#ifndef AOS_VS + return p; +#else +/* We want to allow finding of end of path in either AOS/VS-style pathnames + * or Unix-style pathnames. This presents a few little problems ... + */ + { + if (*p == '=' || *p == '^') /* like ./ and ../ respectively */ + return p + 1; + else + return p; + } +#endif +} + +#if defined(UNICODE_SUPPORT) && defined(WIN32) +wchar_t *lastw(pw, c) + wchar_t *pw; /* sequence of path components */ + wchar_t c; /* path components separator character */ +/* Return a pointer to the start of the last path component. For a directory + * name terminated by the character in c, the return value is an empty string. + */ +{ + wchar_t *tw; /* temporary variable */ + + if ((tw = wcsrchr(pw, c)) != NULL) + return tw + 1; + else +# ifndef AOS_VS + return pw; +# else +/* We want to allow finding of end of path in either AOS/VS-style pathnames + * or Unix-style pathnames. This presents a few little problems ... + */ + { + if (*pw == (wchar_t)'=' || *pw == (wchar_t)'^') /* like ./ and ../ respectively */ + return pw + 1; + else + return pw; + } +# endif +} +#endif + + +char *msname(n) + char *n; +/* Reduce all path components to MSDOS upper case 8.3 style names. */ +{ + int c; /* current character */ + int f; /* characters in current component */ + char *p; /* source pointer */ + char *q; /* destination pointer */ + + p = q = n; + f = 0; + while ((c = (unsigned char)*POSTINCSTR(p)) != 0) + if (c == ' ' || c == ':' || c == '"' || c == '*' || c == '+' || + c == ',' || c == ';' || c == '<' || c == '=' || c == '>' || + c == '?' || c == '[' || c == ']' || c == '|') + continue; /* char is discarded */ + else if (c == '/') + { + *POSTINCSTR(q) = (char)c; + f = 0; /* new component */ + } +#ifdef __human68k__ + else if (ismbblead(c) && *p) + { + if (f == 7 || f == 11) + f++; + else if (*p && f < 12 && f != 8) + { + *q++ = c; + *q++ = *p++; + f += 2; + } + } +#endif /* __human68k__ */ + else if (c == '.') + { + if (f == 0) + continue; /* leading dots are discarded */ + else if (f < 9) + { + *POSTINCSTR(q) = (char)c; + f = 9; /* now in file type */ + } + else + f = 12; /* now just excess characters */ + } + else + if (f < 12 && f != 8) + { + f += CLEN(p); /* do until end of name or type */ + *POSTINCSTR(q) = (char)(to_up(c)); + } + *q = 0; + return n; +} + +#ifdef UNICODE_SUPPORT +wchar_t *msnamew(nw) + wchar_t *nw; +/* Reduce all path components to MSDOS upper case 8.3 style names. */ +{ + wchar_t c; /* current character */ + int f; /* characters in current component */ + wchar_t *pw; /* source pointer */ + wchar_t *qw; /* destination pointer */ + + pw = qw = nw; + f = 0; + while ((c = (unsigned char)*pw++) != 0) + if (c == ' ' || c == ':' || c == '"' || c == '*' || c == '+' || + c == ',' || c == ';' || c == '<' || c == '=' || c == '>' || + c == '?' || c == '[' || c == ']' || c == '|') + continue; /* char is discarded */ + else if (c == '/') + { + *qw++ = c; + f = 0; /* new component */ + } +#ifdef __human68k__ + else if (ismbblead(c) && *pw) + { + if (f == 7 || f == 11) + f++; + else if (*pw && f < 12 && f != 8) + { + *qw++ = c; + *qw++ = *pw++; + f += 2; + } + } +#endif /* __human68k__ */ + else if (c == '.') + { + if (f == 0) + continue; /* leading dots are discarded */ + else if (f < 9) + { + *qw++ = c; + f = 9; /* now in file type */ + } + else + f = 12; /* now just excess characters */ + } + else + if (f < 12 && f != 8) + { + f++; /* do until end of name or type */ + *qw++ = towupper(c); + } + *qw = 0; + return nw; +} +#endif + + +int proc_archive_name(n, caseflag) + char *n; /* name to process */ + int caseflag; /* true to force case-sensitive match */ +/* Process a name or sh expression in existing archive to operate + on (or exclude). Return an error code in the ZE_ class. */ +{ + int m; /* matched flag */ + char *p; /* path for recursion */ + struct zlist far *z; /* steps through zfiles list */ + + if (strcmp(n, "-") == 0) { /* if compressing stdin */ + zipwarn("Cannot select stdin when selecting archive entries", ""); + return ZE_MISS; + } + else + { + /* Search for shell expression in zip file */ + p = ex2in(n, 0, (int *)NULL); /* shouldn't affect matching chars */ + m = 1; + for (z = zfiles; z != NULL; z = z->nxt) { + if (MATCH(p, z->iname, caseflag)) + { + z->mark = pcount ? filter(z->zname, caseflag) : 1; + if (verbose) + fprintf(mesg, "zip diagnostic: %scluding %s\n", + z->mark ? "in" : "ex", z->oname); + m = 0; + } + } +#ifdef UNICODE_SUPPORT + /* also check escaped Unicode names */ + for (z = zfiles; z != NULL; z = z->nxt) { + if (z->zuname) { +#ifdef WIN32 + /* It seems something is lost in going from a listed + name from zip -su in a console window to using that + name in a command line. This kluge may fix it + and just takes zuname, converts to oem (i.e. ouname), + then converts it back which ends up not the same as + started with. + */ + char *zuname = z->wuname; +#else + char *zuname = z->zuname; +#endif + if (MATCH(p, zuname, caseflag)) + { + z->mark = pcount ? filter(zuname, caseflag) : 1; + if (verbose) { + fprintf(mesg, "zip diagnostic: %scluding %s\n", + z->mark ? "in" : "ex", z->oname); + fprintf(mesg, " Escaped Unicode: %s\n", + z->ouname); + } + m = 0; + } + } + } +#endif + free((zvoid *)p); + return m ? ZE_MISS : ZE_OK; + } +} + + +int check_dup() +/* Sort the found list and remove duplicates. + Return an error code in the ZE_ class. */ +{ + struct flist far *f; /* steps through found linked list */ + extent j, k; /* indices for s */ + struct flist far **s; /* sorted table */ + struct flist far **nodup; /* sorted table without duplicates */ + + /* sort found list, remove duplicates */ + if (fcount) + { + extent fl_size = fcount * sizeof(struct flist far *); + if ((fl_size / sizeof(struct flist far *)) != fcount || + (s = (struct flist far **)malloc(fl_size)) == NULL) + return ZE_MEM; + for (j = 0, f = found; f != NULL; f = f->nxt) + s[j++] = f; + /* Check names as given (f->name) */ + qsort((char *)s, fcount, sizeof(struct flist far *), fqcmp); + for (k = j = fcount - 1; j > 0; j--) + if (strcmp(s[j - 1]->name, s[j]->name) == 0) + /* remove duplicate entry from list */ + fexpel(s[j]); /* fexpel() changes fcount */ + else + /* copy valid entry into destination position */ + s[k--] = s[j]; + s[k] = s[0]; /* First entry is always valid */ + nodup = &s[k]; /* Valid entries are at end of array s */ + + /* sort only valid items and check for unique internal names (f->iname) */ + qsort((char *)nodup, fcount, sizeof(struct flist far *), fqcmpz); + for (j = 1; j < fcount; j++) + if (strcmp(nodup[j - 1]->iname, nodup[j]->iname) == 0) + { + char tempbuf[FNMAX+4081]; + + sprintf(errbuf, " first full name: %s\n", nodup[j - 1]->name); + sprintf(tempbuf, " second full name: %s\n", nodup[j]->name); + strcat(errbuf, " "); + strcat(errbuf, tempbuf); +#ifdef EBCDIC + strtoebc(nodup[j]->iname, nodup[j]->iname); +#endif + sprintf(tempbuf, "name in zip file repeated: %s", nodup[j]->iname); + strcat(errbuf, " "); + strcat(errbuf, tempbuf); + if (pathput == 0) { + strcat(errbuf, "\n this may be a result of using -j"); + } +#ifdef EBCDIC + strtoasc(nodup[j]->iname, nodup[j]->iname); +#endif + zipwarn(errbuf, ""); + return ZE_PARMS; + } + free((zvoid *)s); + } + return ZE_OK; +} + +int filter(name, casesensitive) + char *name; + int casesensitive; + /* Scan the -R, -i and -x lists for matches to the given name. + Return TRUE if the name must be included, FALSE otherwise. + Give precedence to -x over -i and -R. + Note that if both R and i patterns are given then must + have a match for both. + This routine relies on the following global variables: + patterns array of match pattern structures + pcount total number of patterns + icount number of -i patterns + Rcount number of -R patterns + These data are set up by the command line parsing code. + */ +{ + unsigned int n; + int slashes; + char *p, *q; + /* without -i patterns, every name matches the "-i select rules" */ + int imatch = (icount == 0); + /* without -R patterns, every name matches the "-R select rules" */ + int Rmatch = (Rcount == 0); + + if (pcount == 0) return TRUE; + + for (n = 0; n < pcount; n++) { + if (!patterns[n].zname[0]) /* it can happen... */ + continue; + p = name; + switch (patterns[n].select) { + case 'R': + if (Rmatch) + /* one -R match is sufficient, skip this pattern */ + continue; + /* With -R patterns, if the pattern has N path components (that is, + N-1 slashes), then we test only the last N components of name. + */ + slashes = 0; + for (q = patterns[n].zname; (q = MBSCHR(q, '/')) != NULL; MB_NEXTCHAR(q)) + slashes++; + /* The name may have M path components (M-1 slashes) */ + for (q = p; (q = MBSCHR(q, '/')) != NULL; MB_NEXTCHAR(q)) + slashes--; + /* Now, "slashes" contains the difference "N-M" between the number + of path components in the pattern (N) and in the name (M). + */ + if (slashes < 0) + /* We found "M > N" + --> skip the first (M-N) path components of the name. + */ + for (q = p; (q = MBSCHR(q, '/')) != NULL; MB_NEXTCHAR(q)) + if (++slashes == 0) { + p = q + 1; /* q points at '/', mblen("/") is 1 */ + break; + } + break; + case 'i': + if (imatch) + /* one -i match is sufficient, skip this pattern */ + continue; + break; + } + if (MATCH(patterns[n].zname, p, casesensitive)) { + switch (patterns[n].select) { + case 'x': + /* The -x match takes precedence over everything else */ + return FALSE; + case 'R': + Rmatch = TRUE; + break; + default: + /* this must be a type -i match */ + imatch = TRUE; + break; + } + } + } + return imatch && Rmatch; +} + + +#ifdef UNICODE_SUPPORT +# ifdef WIN32 + +int newnamew(namew, isdir, casesensitive) + wchar_t *namew; /* name to add (or exclude) */ + int isdir; /* true for a directory */ + int casesensitive; /* true for case-sensitive matching */ +/* Add (or exclude) the name of an existing disk file. Return an error + code in the ZE_ class. */ +{ + wchar_t *inamew = NULL; /* internal name */ + wchar_t *znamew = NULL; /* external version of iname */ + wchar_t *undosmw = NULL; /* zname version with "-j" and "-k" options disabled */ + char *oname = NULL; /* iname converted for display */ + char *name = NULL; + char *iname = NULL; + char *zname = NULL; + char *zuname = NULL; + char *undosm = NULL; + struct flist far *f; /* where in found, or new found entry */ + struct zlist far *z; /* where in zfiles (if found) */ + int dosflag; + + /* Scanning files ... + * + * After 5 seconds output Scanning files... + * then a dot every 2 seconds + */ + if (noisy) { + /* If find files then output message after delay */ + if (scan_count == 0) { + time_t current = time(NULL); + scan_start = current; + } + scan_count++; + if (scan_count % 100 == 0) { + time_t current = time(NULL); + + if (current - scan_start > scan_delay) { + if (scan_last == 0) { + zipmessage_nl("Scanning files ", 0); + scan_last = current; + } + if (current - scan_last > scan_dot_time) { + scan_last = current; + fprintf(mesg, "."); + fflush(mesg); + } + } + } + } + + /* Search for name in zip file. If there, mark it, else add to + list of new names to do (or remove from that list). */ + if ((inamew = ex2inw(namew, isdir, &dosflag)) == NULL) + return ZE_MEM; + + /* Discard directory names with zip -rj */ + if (*inamew == (wchar_t)'\0') { + + /* If extensions needs to be swapped, we will have empty directory names + instead of the original directory. For example, zipping 'c.', 'c.main' + should zip only 'main.c' while 'c.' will be converted to '\0' by ex2in. */ + + if (pathput && !recurse) error("empty name without -j or -r"); + free((zvoid *)inamew); + return ZE_OK; + } + + if (dosflag || !pathput) { + int save_dosify = dosify, save_pathput = pathput; + dosify = 0; + pathput = 1; + /* zname is temporarly mis-used as "undosmode" iname pointer */ + if ((znamew = ex2inw(namew, isdir, NULL)) != NULL) { + undosmw = in2exw(znamew); + free(znamew); + } + dosify = save_dosify; + pathput = save_pathput; + } + if ((znamew = in2exw(inamew)) == NULL) + return ZE_MEM; + + /* Convert names from wchar_t to char */ + + name = wchar_to_local_string(namew); + iname = wchar_to_local_string(inamew); + zname = wchar_to_local_string(znamew); + + oname = local_to_display_string(zname); + + zuname = wchar_to_local_string(znamew); + + if (undosmw == NULL) + undosmw = znamew; + undosm = wchar_to_local_string(undosmw); + + if ((z = zsearch(zuname)) != NULL) { + if (pcount && !filter(undosm, casesensitive)) { + /* Do not clear z->mark if "exclude", because, when "dosify || !pathput" + * is in effect, two files with different filter options may hit the + * same z entry. + */ + if (verbose) + fprintf(mesg, "excluding %s\n", oname); + } else { + z->mark = 1; + if ((z->name = malloc(strlen(name) + 1 + PAD)) == NULL) { + if (undosmw != znamew) + free(undosmw); + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_MEM; + } + strcpy(z->name, name); + z->oname = oname; + oname = NULL; + z->dosflag = dosflag; + +#ifdef FORCE_NEWNAME + free((zvoid *)(z->iname)); + z->iname = iname; + iname = NULL; +#else + /* Better keep the old name. Useful when updating on MSDOS a zip file + * made on Unix. + */ +#endif /* ? FORCE_NEWNAME */ + } + + if ((z->namew = (wchar_t *)malloc((wcslen(namew) + 1) * sizeof(wchar_t))) == NULL) { + if (undosmw != znamew) + free(undosmw); + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_MEM; + } + wcscpy(z->namew, namew); + z->inamew = inamew; + inamew = NULL; + z->znamew = znamew; + znamew = NULL; + z->uname = wchar_to_utf8_string(z->inamew); + if (name == label) { + label = z->name; + } + } else if (pcount == 0 || filter(undosm, casesensitive)) { + + /* Check that we are not adding the zip file to itself. This + * catches cases like "zip -m foo ../dir/foo.zip". + */ +/* Version of stat() for CMS/MVS isn't complete enough to see if */ +/* files match. Just let ZIP.C compare the filenames. That's good */ +/* enough for CMS anyway since there aren't paths to worry about. */ + zw_stat statbw; /* need for wide stat */ + wchar_t *zipfilew = local_to_wchar_string(zipfile); + + if (zipstate == -1) + zipstate = strcmp(zipfile, "-") != 0 && + zwstat(zipfilew, &zipstatbw) == 0; + free(zipfilew); + + if (zipstate == 1 && (statbw = zipstatbw, zwstat(namew, &statbw) == 0 + && zipstatbw.st_mode == statbw.st_mode + && zipstatbw.st_ino == statbw.st_ino + && zipstatbw.st_dev == statbw.st_dev + && zipstatbw.st_uid == statbw.st_uid + && zipstatbw.st_gid == statbw.st_gid + && zipstatbw.st_size == statbw.st_size + && zipstatbw.st_mtime == statbw.st_mtime + && zipstatbw.st_ctime == statbw.st_ctime)) { + /* Don't compare a_time since we are reading the file */ + if (verbose) + fprintf(mesg, "file matches zip file -- skipping\n"); + if (undosmw != znamew) + free(undosmw); + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_OK; + } + + /* allocate space and add to list */ + if ((f = (struct flist far *)farmalloc(sizeof(struct flist))) == NULL || + fcount + 1 < fcount || + (f->name = malloc(strlen(name) + 1 + PAD)) == NULL) + { + if (f != NULL) + farfree((zvoid far *)f); + if (undosmw != znamew) + free(undosmw); + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_MEM; + } + if (undosmw != znamew) + free((zvoid *)undosmw); + strcpy(f->name, name); + f->iname = iname; + iname = NULL; + f->zname = zname; + zname = NULL; + /* Unicode */ + if ((f->namew = (wchar_t *)malloc((wcslen(namew) + 1) * sizeof(wchar_t))) == NULL) { + if (f != NULL) + farfree((zvoid far *)f); + if (undosmw != znamew) + free(undosmw); + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_MEM; + } + wcscpy(f->namew, namew); + f->znamew = znamew; + znamew = NULL; + f->uname = wchar_to_utf8_string(inamew); + f->inamew = inamew; + inamew = NULL; + f->oname = oname; + oname = NULL; + f->dosflag = dosflag; + *fnxt = f; + f->lst = fnxt; + f->nxt = NULL; + fnxt = &f->nxt; + fcount++; + if (name == label) { + label = f->name; + } + } + if (undosm) free(undosm); + if (inamew) free(inamew); + if (znamew) free(znamew); + if (name) free(name); + if (iname) free(iname); + if (zname) free(zname); + if (oname) free(oname); + if (zuname) free(zuname); + return ZE_OK; +} + +# endif +#endif + +int newname(name, isdir, casesensitive) + char *name; /* name to add (or exclude) */ + int isdir; /* true for a directory */ + int casesensitive; /* true for case-sensitive matching */ +/* Add (or exclude) the name of an existing disk file. Return an error + code in the ZE_ class. */ +{ + char *iname, *zname; /* internal name, external version of iname */ + char *undosm; /* zname version with "-j" and "-k" options disabled */ + char *oname; /* iname converted for display */ + struct flist far *f; /* where in found, or new found entry */ + struct zlist far *z; /* where in zfiles (if found) */ + int dosflag; + + /* Scanning files ... + * + * After 5 seconds output Scanning files... + * then a dot every 2 seconds + */ + if (noisy) { + /* If find files then output message after delay */ + if (scan_count == 0) { + time_t current = time(NULL); + scan_start = current; + } + scan_count++; + if (scan_count % 100 == 0) { + time_t current = time(NULL); + + if (current - scan_start > scan_delay) { + if (scan_last == 0) { + zipmessage_nl("Scanning files ", 0); + scan_last = current; + } + if (current - scan_last > scan_dot_time) { + scan_last = current; + fprintf(mesg, "."); + fflush(mesg); + } + } + } + } + + /* Search for name in zip file. If there, mark it, else add to + list of new names to do (or remove from that list). */ + if ((iname = ex2in(name, isdir, &dosflag)) == NULL) + return ZE_MEM; + + /* Discard directory names with zip -rj */ + if (*iname == '\0') { +#ifndef AMIGA +/* A null string is a legitimate external directory name in AmigaDOS; also, + * a command like "zip -r zipfile FOO:" produces an empty internal name. + */ +# ifndef RISCOS + /* If extensions needs to be swapped, we will have empty directory names + instead of the original directory. For example, zipping 'c.', 'c.main' + should zip only 'main.c' while 'c.' will be converted to '\0' by ex2in. */ + + if (pathput && !recurse) error("empty name without -j or -r"); + +# endif /* !RISCOS */ +#endif /* !AMIGA */ + free((zvoid *)iname); + return ZE_OK; + } + undosm = NULL; + if (dosflag || !pathput) { + int save_dosify = dosify, save_pathput = pathput; + dosify = 0; + pathput = 1; + /* zname is temporarly mis-used as "undosmode" iname pointer */ + if ((zname = ex2in(name, isdir, NULL)) != NULL) { + undosm = in2ex(zname); + free(zname); + } + dosify = save_dosify; + pathput = save_pathput; + } + if ((zname = in2ex(iname)) == NULL) + return ZE_MEM; +#ifdef UNICODE_SUPPORT + /* Convert name to display or OEM name */ + oname = local_to_display_string(iname); +#else + if ((oname = malloc(strlen(zname) + 1)) == NULL) + return ZE_MEM; + strcpy(oname, zname); +#endif + if (undosm == NULL) + undosm = zname; + if ((z = zsearch(zname)) != NULL) { + if (pcount && !filter(undosm, casesensitive)) { + /* Do not clear z->mark if "exclude", because, when "dosify || !pathput" + * is in effect, two files with different filter options may hit the + * same z entry. + */ + if (verbose) + fprintf(mesg, "excluding %s\n", oname); + free((zvoid *)iname); + free((zvoid *)zname); + } else { + z->mark = 1; + if ((z->name = malloc(strlen(name) + 1 + PAD)) == NULL) { + if (undosm != zname) + free((zvoid *)undosm); + free((zvoid *)iname); + free((zvoid *)zname); + return ZE_MEM; + } + strcpy(z->name, name); + z->oname = oname; + z->dosflag = dosflag; + +#ifdef FORCE_NEWNAME + free((zvoid *)(z->iname)); + z->iname = iname; +#else + /* Better keep the old name. Useful when updating on MSDOS a zip file + * made on Unix. + */ + free((zvoid *)iname); + free((zvoid *)zname); +#endif /* ? FORCE_NEWNAME */ + } +#if defined(UNICODE_SUPPORT) && defined(WIN32) + z->namew = NULL; + z->inamew = NULL; + z->znamew = NULL; +#endif + if (name == label) { + label = z->name; + } + } else if (pcount == 0 || filter(undosm, casesensitive)) { + + /* Check that we are not adding the zip file to itself. This + * catches cases like "zip -m foo ../dir/foo.zip". + */ +#ifndef CMS_MVS +/* Version of stat() for CMS/MVS isn't complete enough to see if */ +/* files match. Just let ZIP.C compare the filenames. That's good */ +/* enough for CMS anyway since there aren't paths to worry about. */ + z_stat statb; /* now use structure z_stat and function zstat globally 7/24/04 EG */ + + if (zipstate == -1) + zipstate = strcmp(zipfile, "-") != 0 && + zstat(zipfile, &zipstatb) == 0; + + if (zipstate == 1 && (statb = zipstatb, zstat(name, &statb) == 0 + && zipstatb.st_mode == statb.st_mode +#ifdef VMS + && memcmp(zipstatb.st_ino, statb.st_ino, sizeof(statb.st_ino)) == 0 + && strcmp(zipstatb.st_dev, statb.st_dev) == 0 + && zipstatb.st_uid == statb.st_uid +#else /* !VMS */ + && zipstatb.st_ino == statb.st_ino + && zipstatb.st_dev == statb.st_dev + && zipstatb.st_uid == statb.st_uid + && zipstatb.st_gid == statb.st_gid +#endif /* ?VMS */ + && zipstatb.st_size == statb.st_size + && zipstatb.st_mtime == statb.st_mtime + && zipstatb.st_ctime == statb.st_ctime)) { + /* Don't compare a_time since we are reading the file */ + if (verbose) + fprintf(mesg, "file matches zip file -- skipping\n"); + if (undosm != zname) + free((zvoid *)zname); + if (undosm != iname) + free((zvoid *)undosm); + free((zvoid *)iname); + free(oname); + return ZE_OK; + } +#endif /* CMS_MVS */ + + /* allocate space and add to list */ + if ((f = (struct flist far *)farmalloc(sizeof(struct flist))) == NULL || + fcount + 1 < fcount || + (f->name = malloc(strlen(name) + 1 + PAD)) == NULL) + { + if (f != NULL) + farfree((zvoid far *)f); + if (undosm != zname) + free((zvoid *)undosm); + free((zvoid *)iname); + free((zvoid *)zname); + free(oname); + return ZE_MEM; + } + strcpy(f->name, name); + f->iname = iname; + f->zname = zname; +#ifdef UNICODE_SUPPORT + /* Unicode */ + f->uname = local_to_utf8_string(iname); +#ifdef WIN32 + f->namew = NULL; + f->inamew = NULL; + f->znamew = NULL; + if (strcmp(f->name, "-") == 0) { + f->namew = local_to_wchar_string(f->name); + } +#endif + +#endif + f->oname = oname; + f->dosflag = dosflag; + + *fnxt = f; + f->lst = fnxt; + f->nxt = NULL; + fnxt = &f->nxt; + fcount++; + if (name == label) { + label = f->name; + } + } + if (undosm != zname) + free((zvoid *)undosm); + return ZE_OK; +} + +ulg dostime(y, n, d, h, m, s) +int y; /* year */ +int n; /* month */ +int d; /* day */ +int h; /* hour */ +int m; /* minute */ +int s; /* second */ +/* Convert the date y/n/d and time h:m:s to a four byte DOS date and + time (date in high two bytes, time in low two bytes allowing magnitude + comparison). */ +{ + return y < 1980 ? DOSTIME_MINIMUM /* dostime(1980, 1, 1, 0, 0, 0) */ : + (((ulg)y - 1980) << 25) | ((ulg)n << 21) | ((ulg)d << 16) | + ((ulg)h << 11) | ((ulg)m << 5) | ((ulg)s >> 1); +} + + +ulg unix2dostime(t) +time_t *t; /* unix time to convert */ +/* Return the Unix time t in DOS format, rounded up to the next two + second boundary. */ +{ + time_t t_even; + struct tm *s; /* result of localtime() */ + + t_even = (time_t)(((unsigned long)(*t) + 1) & (~1)); + /* Round up to even seconds. */ + s = localtime(&t_even); /* Use local time since MSDOS does. */ + if (s == (struct tm *)NULL) { + /* time conversion error; use current time as emergency value + (assuming that localtime() does at least accept this value!) */ + t_even = (time_t)(((unsigned long)time(NULL) + 1) & (~1)); + s = localtime(&t_even); + } + return dostime(s->tm_year + 1900, s->tm_mon + 1, s->tm_mday, + s->tm_hour, s->tm_min, s->tm_sec); +} + +int issymlnk(a) +ulg a; /* Attributes returned by filetime() */ +/* Return true if the attributes are those of a symbolic link */ +{ +#ifndef QDOS +#ifdef S_IFLNK +#ifdef __human68k__ + int *_dos_importlnenv(void); + + if (_dos_importlnenv() == NULL) + return 0; +#endif + return ((a >> 16) & S_IFMT) == S_IFLNK; +#else /* !S_IFLNK */ + return (int)a & 0; /* avoid warning on unused parameter */ +#endif /* ?S_IFLNK */ +#else + return 0; +#endif +} + +#endif /* !UTIL */ + + +#if (!defined(UTIL) && !defined(ZP_NEED_GEN_D2U_TIME)) + /* There is no need for dos2unixtime() in the ZipUtils' code. */ +# define ZP_NEED_GEN_D2U_TIME +#endif +#if ((defined(OS2) || defined(VMS)) && defined(ZP_NEED_GEN_D2U_TIME)) + /* OS/2 and VMS use a special solution to handle time-stams of files. */ +# undef ZP_NEED_GEN_D2U_TIME +#endif +#if (defined(W32_STATROOT_FIX) && !defined(ZP_NEED_GEN_D2U_TIME)) + /* The Win32 stat()-bandaid to fix stat'ing root directories needs + * dos2unixtime() to calculate the time-stamps. */ +# define ZP_NEED_GEN_D2U_TIME +#endif + +#ifdef ZP_NEED_GEN_D2U_TIME + +time_t dos2unixtime(dostime) +ulg dostime; /* DOS time to convert */ +/* Return the Unix time_t value (GMT/UTC time) for the DOS format (local) + * time dostime, where dostime is a four byte value (date in most significant + * word, time in least significant word), see dostime() function. + */ +{ + struct tm *t; /* argument for mktime() */ + ZCONST time_t clock = time(NULL); + + t = localtime(&clock); + t->tm_isdst = -1; /* let mktime() determine if DST is in effect */ + /* Convert DOS time to UNIX time_t format */ + t->tm_sec = (((int)dostime) << 1) & 0x3e; + t->tm_min = (((int)dostime) >> 5) & 0x3f; + t->tm_hour = (((int)dostime) >> 11) & 0x1f; + t->tm_mday = (int)(dostime >> 16) & 0x1f; + t->tm_mon = ((int)(dostime >> 21) & 0x0f) - 1; + t->tm_year = ((int)(dostime >> 25) & 0x7f) + 80; + + return mktime(t); +} + +#undef ZP_NEED_GEN_D2U_TIME +#endif /* ZP_NEED_GEN_D2U_TIME */ + + +#ifndef MACOS +int destroy(f) + char *f; /* file to delete */ +/* Delete the file *f, returning non-zero on failure. */ +{ + return unlink(f); +} + + +int replace(d, s) +char *d, *s; /* destination and source file names */ +/* Replace file *d by file *s, removing the old *s. Return an error code + in the ZE_ class. This function need not preserve the file attributes, + this will be done by setfileattr() later. + */ +{ + z_stat t; /* results of stat() */ +#if defined(CMS_MVS) + /* cmsmvs.h defines FOPW_TEMP as memory(hiperspace). Since memory is + * lost at end of run, always do copy instead of rename. + */ + int copy = 1; +#else + int copy = 0; +#endif + int d_exists; + +#if defined(VMS) || defined(CMS_MVS) + /* stat() is broken on VMS remote files (accessed through Decnet). + * This patch allows creation of remote zip files, but is not sufficient + * to update them or compress remote files */ + unlink(d); +#else /* !(VMS || CMS_MVS) */ + d_exists = (LSTAT(d, &t) == 0); + if (d_exists) + { + /* + * respect existing soft and hard links! + */ + if (t.st_nlink > 1 +# ifdef S_IFLNK + || (t.st_mode & S_IFMT) == S_IFLNK +# endif + ) + copy = 1; + else if (unlink(d)) + return ZE_CREAT; /* Can't erase zip file--give up */ + } +#endif /* ?(VMS || CMS_MVS) */ +#ifndef CMS_MVS + if (!copy) { + if (rename(s, d)) { /* Just move s on top of d */ + copy = 1; /* failed ? */ +#if !defined(VMS) && !defined(ATARI) && !defined(AZTEC_C) +#if !defined(CMS_MVS) && !defined(RISCOS) && !defined(QDOS) + /* For VMS, ATARI, AMIGA Aztec, VM_CMS, MVS, RISCOS, + always assume that failure is EXDEV */ + if (errno != EXDEV +# ifdef THEOS + && errno != EEXIST +# else +# ifdef ENOTSAM + && errno != ENOTSAM /* Used at least on Turbo C */ +# endif +# endif + ) return ZE_CREAT; +#endif /* !CMS_MVS && !RISCOS */ +#endif /* !VMS && !ATARI && !AZTEC_C */ + } + } +#endif /* !CMS_MVS */ + + if (copy) { + FILE *f, *g; /* source and destination files */ + int r; /* temporary variable */ + +#ifdef RISCOS + if (SWI_OS_FSControl_26(s,d,0xA1)!=NULL) { +#endif + + /* Use zfopen for almost all opens where fopen is used. For + most OS that support large files we use the 64-bit file + environment and zfopen maps to fopen, but this allows + tweeking ports that don't do that. 7/24/04 */ + if ((f = zfopen(s, FOPR)) == NULL) { + fprintf(mesg," replace: can't open %s\n", s); + return ZE_TEMP; + } + if ((g = zfopen(d, FOPW)) == NULL) + { + fclose(f); + return ZE_CREAT; + } + + r = fcopy(f, g, (ulg)-1L); + fclose(f); + if (fclose(g) || r != ZE_OK) + { + unlink(d); + return r ? (r == ZE_TEMP ? ZE_WRITE : r) : ZE_WRITE; + } + unlink(s); +#ifdef RISCOS + } +#endif + } + return ZE_OK; +} +#endif /* !MACOS */ + + +int getfileattr(f) +char *f; /* file path */ +/* Return the file attributes for file f or 0 if failure */ +{ +#ifdef __human68k__ + struct _filbuf buf; + + return _dos_files(&buf, f, 0xff) < 0 ? 0x20 : buf.atr; +#else + z_stat s; + + return SSTAT(f, &s) == 0 ? (int) s.st_mode : 0; +#endif +} + + +int setfileattr(f, a) +char *f; /* file path */ +int a; /* attributes returned by getfileattr() */ +/* Give the file f the attributes a, return non-zero on failure */ +{ +#if defined(TOPS20) || defined (CMS_MVS) + return 0; +#else +#ifdef __human68k__ + return _dos_chmod(f, a) < 0 ? -1 : 0; +#else + return chmod(f, a); +#endif +#endif +} + + +/* tempname */ + +#ifndef VMS /* VMS-specific function is in VMS.C. */ + +char *tempname(zip) + char *zip; /* path name of zip file to generate temp name for */ + +/* Return a temporary file name in its own malloc'ed space, using tempath. */ +{ + char *t = zip; /* malloc'ed space for name (use zip to avoid warning) */ + +# ifdef CMS_MVS + if ((t = malloc(strlen(tempath) + L_tmpnam + 2)) == NULL) + return NULL; + +# ifdef VM_CMS + tmpnam(t); + /* Remove filemode and replace with tempath, if any. */ + /* Otherwise A-disk is used by default */ + *(strrchr(t, ' ')+1) = '\0'; + if (tempath!=NULL) + strcat(t, tempath); + return t; +# else /* !VM_CMS */ + /* For MVS */ + tmpnam(t); + if (tempath != NULL) + { + int l1 = strlen(t); + char *dot; + if (*t == '\'' && *(t+l1-1) == '\'' && (dot = strchr(t, '.'))) + { + /* MVS and not OE. tmpnam() returns quoted string of 5 qualifiers. + * First is HLQ, rest are timestamps. User can only replace HLQ. + */ + int l2 = strlen(tempath); + if (strchr(tempath, '.') || l2 < 1 || l2 > 8) + ziperr(ZE_PARMS, "On MVS and not OE, tempath (-b) can only be HLQ"); + memmove(t+1+l2, dot, l1+1-(dot-t)); /* shift dot ready for new hlq */ + memcpy(t+1, tempath, l2); /* insert new hlq */ + } + else + { + /* MVS and probably OE. tmpnam() returns filename based on TMPDIR, + * no point in even attempting to change it. User should modify TMPDIR + * instead. + */ + zipwarn("MVS, assumed to be OE, change TMPDIR instead of option -b: ", + tempath); + } + } + return t; +# endif /* !VM_CMS */ + +# else /* !CMS_MVS */ + +# ifdef TANDEM + char cur_subvol [FILENAME_MAX]; + char temp_subvol [FILENAME_MAX]; + char *zptr; + char *ptr; + char *cptr = &cur_subvol[0]; + char *tptr = &temp_subvol[0]; + short err; + FILE *tempf; + int attempts; + + t = (char *)malloc(NAMELEN); /* malloc here as you cannot free */ + /* tmpnam allocated storage later */ + + zptr = strrchr(zip, TANDEM_DELIMITER); + + if (zptr != NULL) { + /* ZIP file specifies a Subvol so make temp file there so it can just + be renamed at end */ + + *tptr = *cptr = '\0'; + strcat(cptr, getenv("DEFAULTS")); + + strncat(tptr, zip, _min(FILENAME_MAX, (zptr - zip)) ); /* temp subvol */ + strncat(t, zip, _min(NAMELEN, ((zptr - zip) + 1)) ); /* temp stem */ + + err = chvol(tptr); + ptr = t + strlen(t); /* point to end of stem */ + } + else + ptr = t; + + /* If two zips are running in same subvol then we can get contention problems + with the temporary filename. As a work around we attempt to create + the file here, and if it already exists we get a new temporary name */ + + attempts = 0; + do { + attempts++; + tmpnam(ptr); /* Add filename */ + tempf = zfopen(ptr, FOPW_TMP); /* Attempt to create file */ + } while (tempf == NULL && attempts < 100); + + if (attempts >= 100) { + ziperr(ZE_TEMP, "Could not get unique temp file name"); + } + + fclose(tempf); + + if (zptr != NULL) { + err = chvol(cptr); /* Put ourself back to where we came in */ + } + + return t; + +# else /* !CMS_MVS && !TANDEM */ +/* + * Do something with TMPDIR, TMP, TEMP ???? + */ + if (tempath != NULL) + { + if ((t = malloc(strlen(tempath) + 12)) == NULL) + return NULL; + strcpy(t, tempath); + +# if (!defined(VMS) && !defined(TOPS20)) +# ifdef MSDOS + { + char c = (char)lastchar(t); + if (c != '/' && c != ':' && c != '\\') + strcat(t, "/"); + } +# else + +# ifdef AMIGA + { + char c = (char)lastchar(t); + if (c != '/' && c != ':') + strcat(t, "/"); + } +# else /* !AMIGA */ +# ifdef RISCOS + if (lastchar(t) != '.') + strcat(t, "."); +# else /* !RISCOS */ + +# ifdef QDOS + if (lastchar(t) != '_') + strcat(t, "_"); +# else + if (lastchar(t) != '/') + strcat(t, "/"); +# endif /* ?QDOS */ +# endif /* ?RISCOS */ +# endif /* ?AMIGA */ +# endif /* ?MSDOS */ +# endif /* !VMS && !TOPS20 */ + } + else + { + if ((t = malloc(12)) == NULL) + return NULL; + *t = 0; + } +# ifdef NO_MKTEMP + { + char *p = t + strlen(t); + sprintf(p, "%08lx", (ulg)time(NULL)); + return t; + } +# else + strcat(t, "ziXXXXXX"); /* must use lowercase for Linux dos file system */ +# if defined(UNIX) && !defined(NO_MKSTEMP) + /* tempname should not be called */ + return t; +# else + return mktemp(t); +# endif +# endif /* NO_MKTEMP */ +# endif /* TANDEM */ +# endif /* CMS_MVS */ +} +#endif /* !VMS */ + +int fcopy(f, g, n) + FILE *f, *g; /* source and destination files */ + /* now use uzoff_t for all file sizes 5/14/05 CS */ + uzoff_t n; /* number of bytes to copy or -1 for all */ +/* Copy n bytes from file *f to file *g, or until EOF if (zoff_t)n == -1. + Return an error code in the ZE_ class. */ +{ + char *b; /* malloc'ed buffer for copying */ + extent k; /* result of fread() */ + uzoff_t m; /* bytes copied so far */ + + if ((b = malloc(CBSZ)) == NULL) + return ZE_MEM; + m = 0; + while (n == (uzoff_t)(-1L) || m < n) + { + if ((k = fread(b, 1, n == (uzoff_t)(-1) ? + CBSZ : (n - m < CBSZ ? (extent)(n - m) : CBSZ), f)) == 0) + { + if (ferror(f)) + { + free((zvoid *)b); + return ZE_READ; + } + else + break; + } + if (fwrite(b, 1, k, g) != k) + { + free((zvoid *)b); + fprintf(mesg," fcopy: write error\n"); + return ZE_TEMP; + } + m += k; + } + free((zvoid *)b); + return ZE_OK; +} + + +/* from zipfile.c */ + +#ifdef THEOS + /* Macros cause stack overflow in compiler */ + ush SH(uch* p) { return ((ush)(uch)((p)[0]) | ((ush)(uch)((p)[1]) << 8)); } + ulg LG(uch* p) { return ((ulg)(SH(p)) | ((ulg)(SH((p)+2)) << 16)); } +#else /* !THEOS */ + /* Macros for converting integers in little-endian to machine format */ +# define SH(a) ((ush)(((ush)(uch)(a)[0]) | (((ush)(uch)(a)[1]) << 8))) +# define LG(a) ((ulg)SH(a) | ((ulg)SH((a)+2) << 16)) +# ifdef ZIP64_SUPPORT /* zip64 support 08/31/2003 R.Nausedat */ +# define LLG(a) ((zoff_t)LG(a) | ((zoff_t)LG((a)+4) << 32)) +# endif +#endif /* ?THEOS */ + + +/* always copies from global in_file to global output file y */ +int bfcopy(n) + /* now use uzoff_t for all file sizes 5/14/05 CS */ + uzoff_t n; /* number of bytes to copy or -1 for all */ +/* Copy n bytes from in_file to out_file, or until EOF if (zoff_t)n == -1. + + Normally we have the compressed size from either the central directory + entry or the local header. + + If n != -1 and EOF, close current split and open next and continue + copying. + + If n == -2, copy until find the extended header (data descriptor). Only + used for -FF when no size available. + + If fix == 1 calculate CRC of input entry and verify matches. + + If fix == 2 and this entry using data descriptor keep a sliding + window in the buffer for looking for signature. + + Return an error code in the ZE_ class. */ +{ + char *b; /* malloc'ed buffer for copying */ + extent k; /* result of fread() */ + uzoff_t m; /* bytes copied so far */ + extent brd; /* bytes to read */ + zoff_t data_start = 0; + zoff_t des_start = 0; + char *split_path; + extent kk; + int i; + char sbuf[4]; /* buffer for sliding signature window for fix = 2 */ + int des = 0; /* this entry has data descriptor to find */ + + if ((b = malloc(CBSZ)) == NULL) + return ZE_MEM; + + if (copy_only && !display_globaldots) { + /* initialize dot count */ + dot_count = -1; + } + + if (fix == 2 && n == (uzoff_t) -2) { + data_start = zftello(in_file); + for (kk = 0; kk < 4; kk++) + sbuf[kk] = 0; + des = 1; + } + + des_good = 0; + + m = 0; + while (des || n == (uzoff_t)(-1L) || m < n) + { + if (des || n == (uzoff_t)(-1)) + brd = CBSZ; + else + brd = (n - m < CBSZ ? (extent)(n - m) : CBSZ); + + des_start = zftello(in_file); + + if ((k = fread(b, 1, brd, in_file)) == 0) + { + if (fix == 2 && k < brd) { + free((zvoid *)b); + return ZE_READ; + } + else if (ferror(in_file)) + { + free((zvoid *)b); + return ZE_READ; + } + else { + break; + } + } + + + /* end at extended local header (data descriptor) signature */ + if (des) { + des_crc = 0; + des_csize = 0; + des_usize = 0; + + /* If first 4 bytes in buffer are data descriptor signature then + try to read the data descriptor. + If not, scan for signature and break if found, let bfwrite flush + the data and then next read should put the data descriptor at + the beginning of the buffer. + */ + + if ( + (b[0] != 0x50 /*'P' except EBCDIC*/ || + b[1] != 0x4b /*'K' except EBCDIC*/ || + b[2] != '\07' || + b[3] != '\010')) { + /* buffer is not start of data descriptor */ + + for (kk = 0; kk < k; kk++) { + /* add byte to end of sbuf */ + for (i = 0; i < 3; i++) + sbuf[i] = sbuf[i + 1]; + sbuf[3] = b[kk]; + + /* see if this is signature */ + if ( + (sbuf[0] == 0x50 /*'P' except EBCDIC*/ && + sbuf[1] == 0x4b /*'K' except EBCDIC*/ && + sbuf[2] == '\07' && + sbuf[3] == '\010')) { + kk -= 3; + if (zfseeko(in_file, bytes_this_split + kk, SEEK_SET) != 0) { + /* seek error */ + ZIPERR(ZE_READ, "seek failed reading descriptor"); + } + des_start = zftello(in_file); + k = kk; + break; + } + } + } + else + + /* signature at start of buffer */ + { + des_good = 0; + +#ifdef ZIP64_SUPPORT + if (zip64_entry) { + + /* read Zip64 data descriptor */ + if (k < 24) { + /* not enough bytes, so can't be data descriptor + as data descriptors can't be split across splits + */ + } + else + { + /* read the Zip64 descriptor */ + + des_crc = LG(b + 4); + des_csize = LLG(b + 8); + des_usize = LLG(b + 16); + + /* if this is the right data descriptor then the sizes should match */ + if ((uzoff_t)des_start - (uzoff_t)data_start != des_csize) { + /* apparently this signature does not go with this data so skip */ + + /* write out signature as data */ + k = 4; + if (zfseeko(in_file, des_start + k, SEEK_SET) != 0) { + /* seek error */ + ZIPERR(ZE_READ, "seek failed reading descriptor"); + } + if (bfwrite(b, 1, k, BFWRITE_DATA) != k) + { + free((zvoid *)b); + fprintf(mesg," fcopy: write error\n"); + return ZE_TEMP; + } + m += k; + continue; + } + else + { + /* apparently this is the correct data descriptor */ + + /* we should check the CRC but would need to inflate + the data */ + + /* skip descriptor as will write out later */ + des_good = 1; + k = 24; + data_start = zftello(in_file); + if (zfseeko(in_file, des_start + k, SEEK_SET) != 0) { + /* seek error */ + ZIPERR(ZE_READ, "seek failed reading descriptor"); + } + data_start = zftello(in_file); + } + } + + } + else +#endif + { + /* read standard data descriptor */ + + if (k < 16) { + /* not enough bytes, so can't be data descriptor + as data descriptors can't be split across splits + */ + } + else + { + /* read the descriptor */ + + des_crc = LG(b + 4); + des_csize = LG(b + 8); + des_usize = LG(b + 12); + + /* if this is the right data descriptor then the sizes should match */ + if ((uzoff_t)des_start - (uzoff_t)data_start != des_csize) { + /* apparently this signature does not go with this data so skip */ + + /* write out signature as data */ + k = 4; + if (zfseeko(in_file, des_start + k, SEEK_SET) != 0) { + /* seek error */ + ZIPERR(ZE_READ, "seek failed reading descriptor"); + } + if (bfwrite(b, 1, k, BFWRITE_DATA) != k) + { + free((zvoid *)b); + fprintf(mesg," fcopy: write error\n"); + return ZE_TEMP; + } + m += k; + continue; + } + else + { + /* apparently this is the correct data descriptor */ + + /* we should check the CRC but this does not work for + encrypted data */ + + /* skip descriptor as will write out later */ + des_good = 1; + data_start = zftello(in_file); + k = 16; + if (zfseeko(in_file, des_start + k, SEEK_SET) != 0) { + /* seek error */ + ZIPERR(ZE_READ, "seek failed reading descriptor"); + } + data_start = zftello(in_file); + } + } + + + } + } + } + + + if (des_good) { + /* skip descriptor as will write out later */ + } else { + /* write out apparently wrong descriptor as data */ + if (bfwrite(b, 1, k, BFWRITE_DATA) != k) + { + free((zvoid *)b); + fprintf(mesg," fcopy: write error\n"); + return ZE_TEMP; + } + m += k; + } + + if (copy_only && !display_globaldots) { + if (dot_size > 0) { + /* initial space */ + if (noisy && dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + dot_count++; + } + dot_count += k; + if (dot_size <= dot_count) dot_count = 0; + } + if ((verbose || noisy) && dot_size && !dot_count) { +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + + if (des_good) + break; + + if (des) + continue; + + if ((des || n != (uzoff_t)(-1L)) && m < n && feof(in_file)) { + /* open next split */ + current_in_disk++; + + if (current_in_disk >= total_disks) { + /* done */ + break; + + } else if (current_in_disk == total_disks - 1) { + /* last disk is archive.zip */ + if ((split_path = malloc(strlen(in_path) + 1)) == NULL) { + zipwarn("reading archive: ", in_path); + return ZE_MEM; + } + strcpy(split_path, in_path); + } else { + /* other disks are archive.z01, archive.z02, ... */ + split_path = get_in_split_path(in_path, current_in_disk); + } + + fclose(in_file); + + /* open the split */ + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + int r = 0; + + /* could not open split */ + + if (fix == 1 && skip_this_disk) { + free(split_path); + free((zvoid *)b); + return ZE_FORM; + } + + /* Ask for directory with split. Updates in_path */ + r = ask_for_split_read_path(current_in_disk); + if (r == ZE_ABORT) { + zipwarn("could not find split: ", split_path); + free(split_path); + free((zvoid *)b); + return ZE_ABORT; + } + if (r == ZE_EOF) { + zipmessage_nl("", 1); + zipwarn("user ended reading - closing archive", ""); + free(split_path); + free((zvoid *)b); + return ZE_EOF; + } + if (fix == 2 && skip_this_disk) { + /* user asked to skip this disk */ + zipwarn("skipping split file: ", split_path); + current_in_disk++; + } + + if (current_in_disk == total_disks - 1) { + /* last disk is archive.zip */ + if ((split_path = malloc(strlen(in_path) + 1)) == NULL) { + zipwarn("reading archive: ", in_path); + return ZE_MEM; + } + strcpy(split_path, in_path); + } else { + /* other disks are archive.z01, archive.z02, ... */ + split_path = get_in_split_path(zipfile, current_in_disk); + } + } + if (fix == 2 && skip_this_disk) { + /* user asked to skip this disk */ + free(split_path); + free((zvoid *)b); + return ZE_FORM; + } + free(split_path); + } + } + free((zvoid *)b); + return ZE_OK; +} + + + +#ifdef NO_RENAME +int rename(from, to) +ZCONST char *from; +ZCONST char *to; +{ + unlink(to); + if (link(from, to) == -1) + return -1; + if (unlink(from) == -1) + return -1; + return 0; +} + +#endif /* NO_RENAME */ + + +#ifdef ZMEM + +/************************/ +/* Function memset() */ +/************************/ + +/* + * memset - for systems without it + * bill davidsen - March 1990 + */ + +char * +memset(buf, init, len) +register char *buf; /* buffer loc */ +register int init; /* initializer */ +register unsigned int len; /* length of the buffer */ +{ + char *start; + + start = buf; + while (len--) *(buf++) = init; + return(start); +} + + +/************************/ +/* Function memcpy() */ +/************************/ + +char * +memcpy(dst,src,len) /* v2.0f */ +register char *dst, *src; +register unsigned int len; +{ + char *start; + + start = dst; + while (len--) + *dst++ = *src++; + return(start); +} + + +/************************/ +/* Function memcmp() */ +/************************/ + +int +memcmp(b1,b2,len) /* jpd@usl.edu -- 11/16/90 */ +register char *b1, *b2; +register unsigned int len; +{ + + if (len) do { /* examine each byte (if any) */ + if (*b1++ != *b2++) + return (*((uch *)b1-1) - *((uch *)b2-1)); /* exit when miscompare */ + } while (--len); + + return(0); /* no miscompares, yield 0 result */ +} + +#endif /* ZMEM */ + + +/*------------------------------------------------------------------ + * Split archives + */ + + +/* ask_for_split_read_path + * + * If the next split file is not in the current directory, ask + * the user where it is. + * + * in_path is the base path for reading splits and is usually + * the same as zipfile. The path in in_path must be the archive + * file ending in .zip as this is assumed by get_in_split_path(). + * + * Updates in_path if changed. Returns ZE_OK if OK or ZE_ABORT if + * user cancels reading archive. + * + * If fix = 1 then allow skipping disk (user may not have it). + */ + +#define SPLIT_MAXPATH (FNMAX + 4010) + +int ask_for_split_read_path(current_disk) + ulg current_disk; +{ + FILE *f; + int is_readable = 0; + int i; + char *split_dir = NULL; + char *archive_name = NULL; + char *split_name = NULL; + char *split_path = NULL; + char buf[SPLIT_MAXPATH + 100]; + + /* get split path */ + split_path = get_in_split_path(in_path, current_disk); + + /* get the directory */ + if ((split_dir = malloc(strlen(in_path) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(split_dir, in_path); + + /* remove any name at end */ + for (i = strlen(split_dir) - 1; i >= 0; i--) { + if (split_dir[i] == '/' || split_dir[i] == '\\' + || split_dir[i] == ':') { + split_dir[i + 1] = '\0'; + break; + } + } + if (i < 0) + split_dir[0] = '\0'; + + /* get the name of the archive */ + if ((archive_name = malloc(strlen(in_path) + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + if (strlen(in_path) == strlen(split_dir)) { + archive_name[0] = '\0'; + } else { + strcpy(archive_name, in_path + strlen(split_dir)); + } + + /* get the name of the split */ + if ((split_name = malloc(strlen(split_path) + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + if (strlen(in_path) == strlen(split_dir)) { + split_name[0] = '\0'; + } else { + strcpy(split_name, split_path + strlen(split_dir)); + } + if (i < 0) { + strcpy(split_dir, "(current directory)"); + } + + fprintf(mesg, "\n\nCould not find:\n"); + fprintf(mesg, " %s\n", split_path); + /* + fprintf(mesg, "Please enter the path directory (. for cur dir) where\n"); + fprintf(mesg, " %s\n", split_name); + fprintf(mesg, "is located\n"); + */ + for (;;) { + if (is_readable) { + fprintf(mesg, "\nHit c (change path to where this split file is)"); + fprintf(mesg, "\n q (abort archive - quit)"); + fprintf(mesg, "\n or ENTER (continue with this split): "); + } else { + if (fix == 1) { + fprintf(mesg, "\nHit c (change path to where this split file is)"); + fprintf(mesg, "\n s (skip this split)"); + fprintf(mesg, "\n q (abort archive - quit)"); + fprintf(mesg, "\n or ENTER (try reading this split again): "); + } else if (fix == 2) { + fprintf(mesg, "\nHit c (change path to where this split file is)"); + fprintf(mesg, "\n s (skip this split)"); + fprintf(mesg, "\n q (abort archive - quit)"); + fprintf(mesg, "\n e (end this archive - no more splits)"); + fprintf(mesg, "\n z (look for .zip split - the last split)"); + fprintf(mesg, "\n or ENTER (try reading this split again): "); + } else { + fprintf(mesg, "\nHit c (change path to where this split file is)"); + fprintf(mesg, "\n q (abort archive - quit)"); + fprintf(mesg, "\n or ENTER (try reading this split again): "); + } + } + fflush(mesg); + fgets(buf, SPLIT_MAXPATH, stdin); + /* remove any newline */ + for (i = 0; buf[i]; i++) { + if (buf[i] == '\n') { + buf[i] = '\0'; + break; + } + } + if (toupper(buf[0]) == 'Q') { + return ZE_ABORT; + } else if ((fix == 1 || fix == 2) && toupper(buf[0]) == 'S') { + /* + fprintf(mesg, "\nSkip this split/disk? (files in this split will not be recovered) [n/y] "); + fflush(mesg); + fgets(buf, SPLIT_MAXPATH, stdin); + if (buf[0] == 'y' || buf[0] == 'Y') { + */ + skip_this_disk = current_in_disk + 1; + return ZE_FORM; + } else if (toupper(buf[0]) == 'C') { + fprintf(mesg, "\nEnter path where this split is (ENTER = same dir, . = current dir)"); + fprintf(mesg, "\n: "); + fflush(mesg); + fgets(buf, SPLIT_MAXPATH, stdin); + is_readable = 0; + /* remove any newline */ + for (i = 0; buf[i]; i++) { + if (buf[i] == '\n') { + buf[i] = '\0'; + break; + } + } + if (buf[0] == '\0') { + /* Hit ENTER so try old path again - could be removable media was changed */ + strcpy(buf, split_path); + } + } else if (fix == 2 && toupper(buf[0]) == 'E') { + /* no more splits to read */ + return ZE_EOF; + } else if (fix == 2 && toupper(buf[0]) == 'Z') { + total_disks = current_disk + 1; + free(split_path); + split_path = get_in_split_path(in_path, current_disk); + buf[0] = '\0'; + strncat(buf, split_path, SPLIT_MAXPATH); + } + if (strlen(buf) > 0) { + /* changing path */ + + /* check if user wants current directory */ + if (buf[0] == '.' && buf[1] == '\0') { + buf[0] = '\0'; + } + /* remove any name at end */ + for (i = strlen(buf); i >= 0; i--) { + if (buf[i] == '/' || buf[i] == '\\' + || buf[i] == ':') { + buf[i + 1] = '\0'; + break; + } + } + /* update base_path to newdir/split_name - in_path is the .zip file path */ + free(in_path); + if (i < 0) { + /* just name so current directory */ + strcpy(buf, "(current directory)"); + if (archive_name == NULL) { + i = 0; + } else { + i = strlen(archive_name); + } + if ((in_path = malloc(strlen(archive_name) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(in_path, archive_name); + } else { + /* not the current directory */ + /* remove any name at end */ + for (i = strlen(buf); i >= 0; i--) { + if (buf[i] == '/') { + buf[i + 1] = '\0'; + break; + } + } + if (i < 0) { + buf[0] = '\0'; + } + if ((in_path = malloc(strlen(buf) + strlen(archive_name) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(in_path, buf); + strcat(in_path, archive_name); + } + + free(split_path); + + /* get split path */ + split_path = get_in_split_path(in_path, current_disk); + + free(split_dir); + if ((split_dir = malloc(strlen(in_path) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(split_dir, in_path); + /* remove any name at end */ + for (i = strlen(split_dir); i >= 0; i--) { + if (split_dir[i] == '/') { + split_dir[i + 1] = '\0'; + break; + } + } + + /* try to open it */ + if ((f = fopen(split_path, "r")) == NULL) { + fprintf(mesg, "\nCould not find or open\n"); + fprintf(mesg, " %s\n", split_path); + /* + fprintf(mesg, "Please enter the path (. for cur dir) where\n"); + fprintf(mesg, " %s\n", split_name); + fprintf(mesg, "is located\n"); + */ + continue; + } + fclose(f); + is_readable = 1; + fprintf(mesg, "Found: %s\n", split_path); + } else { + /* try to open it */ + if ((f = fopen(split_path, "r")) == NULL) { + fprintf(mesg, "\nCould not find or open\n"); + fprintf(mesg, " %s\n", split_path); + /* + fprintf(mesg, "Please enter the path (. for cur dir) where\n"); + fprintf(mesg, " %s\n", split_name); + fprintf(mesg, "is located\n"); + */ + continue; + } + fclose(f); + is_readable = 1; + fprintf(mesg, "\nFound: %s\n", split_path); + break; + } + } + free(archive_name); + free(split_dir); + free(split_name); + + return ZE_OK; +} + + +/* ask_for_split_write_path + * + * Verify the directory for the next split. Called + * when -sp is used to pause between writing splits. + * + * Updates out_path and return 1 if OK or 0 if cancel + */ +int ask_for_split_write_path(current_disk) + ulg current_disk; +{ + unsigned int num = (unsigned int)current_disk + 1; + int i; + char *split_dir = NULL; + char *split_name = NULL; + char buf[FNMAX + 40]; + + /* get the directory */ + if ((split_dir = malloc(strlen(out_path) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(split_dir, out_path); + + /* remove any name at end */ + for (i = strlen(split_dir); i >= 0; i--) { + if (split_dir[i] == '/' || split_dir[i] == '\\' + || split_dir[i] == ':') { + split_dir[i + 1] = '\0'; + break; + } + } + + /* get the name of the split */ + if ((split_name = malloc(strlen(out_path) + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + if (strlen(out_path) == strlen(split_dir)) { + split_name[0] = '\0'; + } else { + strcpy(split_name, out_path + strlen(split_dir)); + } + if (i < 0) { + strcpy(split_dir, "(current directory)"); + } + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "\nOpening disk %d\n", num); + fprintf(mesg, "Hit ENTER to write to default path of\n"); + fprintf(mesg, " %s\n", split_dir); + fprintf(mesg, "or enter a new directory path (. for cur dir) and hit ENTER\n"); + for (;;) { + fprintf(mesg, "\nPath (or hit ENTER to continue): "); + fflush(mesg); + fgets(buf, FNMAX, stdin); + /* remove any newline */ + for (i = 0; buf[i]; i++) { + if (buf[i] == '\n') { + buf[i] = '\0'; + break; + } + } + if (strlen(buf) > 0) { + /* changing path */ + + /* current directory */ + if (buf[0] == '.' && buf[1] == '\0') { + buf[0] = '\0'; + } + /* remove any name at end */ + for (i = strlen(buf); i >= 0; i--) { + if (buf[i] == '/' || buf[i] == '\\' + || buf[i] == ':') { + buf[i + 1] = '\0'; + break; + } + } + /* update out_path to newdir/split_name */ + free(out_path); + if (i < 0) { + /* just name so current directory */ + strcpy(buf, "(current directory)"); + if (split_name == NULL) { + i = 0; + } else { + i = strlen(split_name); + } + if ((out_path = malloc(strlen(split_name) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(out_path, split_name); + } else { + /* not the current directory */ + /* remove any name at end */ + for (i = strlen(buf); i >= 0; i--) { + if (buf[i] == '/') { + buf[i + 1] = '\0'; + break; + } + } + if (i < 0) { + buf[0] = '\0'; + } + if ((out_path = malloc(strlen(buf) + strlen(split_name) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(out_path, buf); + strcat(out_path, split_name); + } + fprintf(mesg, "Writing to:\n %s\n", buf); + free(split_name); + free(split_dir); + if ((split_dir = malloc(strlen(out_path) + 40)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(split_dir, out_path); + /* remove any name at end */ + for (i = strlen(split_dir); i >= 0; i--) { + if (split_dir[i] == '/') { + split_dir[i + 1] = '\0'; + break; + } + } + if ((split_name = malloc(strlen(out_path) + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + strcpy(split_name, out_path + strlen(split_dir)); + } else { + break; + } + } + free(split_dir); + free(split_name); + + /* for now no way out except Ctrl C */ + return 1; +} + + +/* split_name + * + * get name of split being read + */ +char *get_in_split_path(base_path, disk_number) + char *base_path; + ulg disk_number; +{ + char *split_path = NULL; + int base_len = 0; + int path_len = 0; + ulg num = disk_number + 1; + char ext[6]; +#ifdef VMS + int vers_len; /* File version length. */ + char *vers_ptr; /* File version string. */ +#endif /* def VMS */ + + /* + * A split has extension z01, z02, ..., z99, z100, z101, ... z999 + * We currently support up to .z99999 + * WinZip will also read .100, .101, ... but AppNote 6.2.2 uses above + * so use that. Means on DOS can only have 100 splits. + */ + + if (num == total_disks) { + /* last disk is base path */ + if ((split_path = malloc(strlen(base_path) + 1)) == NULL) { + ZIPERR(ZE_MEM, "base path"); + } + strcpy(split_path, base_path); + + return split_path; + } else { + if (num > 99999) { + ZIPERR(ZE_BIG, "More than 99999 splits needed"); + } + sprintf(ext, "z%02lu", num); + } + + /* create path for this split - zip.c checked for .zip extension */ + base_len = strlen(base_path) - 3; + path_len = base_len + strlen(ext); + +#ifdef VMS + /* On VMS, locate the file version, and adjust base_len accordingly. + Note that path_len is correct, as-is. + */ + vers_ptr = vms_file_version( base_path); + vers_len = strlen( vers_ptr); + base_len -= vers_len; +#endif /* def VMS */ + + if ((split_path = malloc(path_len + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + /* copy base_path except for end zip */ + strcpy(split_path, base_path); + split_path[base_len] = '\0'; + /* add extension */ + strcat(split_path, ext); + +#ifdef VMS + /* On VMS, append (preserve) the file version. */ + strcat(split_path, vers_ptr); +#endif /* def VMS */ + + return split_path; +} + + +/* split_name + * + * get name of split being written + */ +char *get_out_split_path(base_path, disk_number) + char *base_path; + ulg disk_number; +{ + char *split_path = NULL; + int base_len = 0; + int path_len = 0; + ulg num = disk_number + 1; + char ext[6]; +#ifdef VMS + int vers_len; /* File version length. */ + char *vers_ptr; /* File version string. */ +#endif /* def VMS */ + + /* + * A split has extension z01, z02, ..., z99, z100, z101, ... z999 + * We currently support up to .z99999 + * WinZip will also read .100, .101, ... but AppNote 6.2.2 uses above + * so use that. Means on DOS can only have 100 splits. + */ + + if (num > 99999) { + ZIPERR(ZE_BIG, "More than 99999 splits needed"); + } + sprintf(ext, "z%02lu", num); + + /* create path for this split - zip.c checked for .zip extension */ + base_len = strlen(base_path) - 3; + path_len = base_len + strlen(ext); + +#ifdef VMS + /* On VMS, locate the file version, and adjust base_len accordingly. + Note that path_len is correct, as-is. + */ + vers_ptr = vms_file_version( base_path); + vers_len = strlen( vers_ptr); + base_len -= vers_len; +#endif /* def VMS */ + + if ((split_path = malloc(path_len + 1)) == NULL) { + ZIPERR(ZE_MEM, "split path"); + } + /* copy base_path except for end zip */ + strcpy(split_path, base_path); + split_path[base_len] = '\0'; + /* add extension */ + strcat(split_path, ext); + +#ifdef VMS + /* On VMS, append (preserve) the file version. */ + strcat(split_path, vers_ptr); +#endif /* def VMS */ + + return split_path; +} + +/* close_split + * + * close a split - assume that the paths needed for the splits are + * available. + */ +int close_split(disk_number, tempfile, temp_name) + ulg disk_number; + FILE *tempfile; + char *temp_name; +{ + char *split_path = NULL; + + split_path = get_out_split_path(out_path, disk_number); + + if (noisy_splits) { + zipmessage("\tClosing split ", split_path); + } + + fclose(tempfile); + + rename_split(temp_name, split_path); + set_filetype(split_path); + + return ZE_OK; +} + +/* bfwrite + Does the fwrite but also counts bytes and does splits */ +size_t bfwrite(buffer, size, count, mode) + ZCONST void *buffer; + size_t size; + size_t count; + int mode; +{ + size_t bytes_written = 0; + size_t r; + size_t b = size * count; + uzoff_t bytes_left_in_split = 0; + size_t bytes_to_write = b; + + + /* -------------------------------- */ + /* local header */ + if (mode == BFWRITE_LOCALHEADER) { + /* writing local header - reset entry data count */ + bytes_this_entry = 0; + /* save start of local header so we can rewrite later */ + current_local_file = y; + current_local_disk = current_disk; + current_local_offset = bytes_this_split; + } + + if (split_size == 0) + bytes_left_in_split = bytes_to_write; + else + bytes_left_in_split = split_size - bytes_this_split; + + if (bytes_to_write > bytes_left_in_split) { + if (mode == BFWRITE_HEADER || + mode == BFWRITE_LOCALHEADER || + mode == BFWRITE_CENTRALHEADER) { + /* if can't write entire header save for next split */ + bytes_to_write = 0; + } else { + /* normal data so fill the split */ + bytes_to_write = (size_t)bytes_left_in_split; + } + } + + /* -------------------------------- */ + /* central header */ + if (mode == BFWRITE_CENTRALHEADER) { + /* set start disk for CD */ + if (cd_start_disk == (ulg)-1) { + cd_start_disk = current_disk; + cd_start_offset = bytes_this_split; + } + cd_entries_this_disk++; + total_cd_entries++; + } + + /* -------------------------------- */ + if (bytes_to_write > 0) { + /* write out the bytes for this split */ + r = fwrite(buffer, size, bytes_to_write, y); + bytes_written += r; + bytes_to_write = b - r; + bytes_this_split += r; + if (mode == BFWRITE_DATA) + /* if data descriptor do not include in count */ + bytes_this_entry += r; + } else { + bytes_to_write = b; + } + + if (bytes_to_write > 0) { + if (split_method) { + /* still bytes to write so close split and open next split */ + bytes_prev_splits += bytes_this_split; + + if (split_method == 1 && ferror(y)) { + /* if writing all splits to same place and have problem then bad */ + ZIPERR(ZE_WRITE, "Could not write split"); + } + + if (split_method == 2 && ferror(y)) { + /* A split must be at least 64K except last .zip split */ + if (bytes_this_split < 64 * (uzoff_t)0x400) { + ZIPERR(ZE_WRITE, "Not enough space to write split"); + } + } + + /* close this split */ + if (split_method == 1 && current_local_disk == current_disk) { + /* keep split open so can update it */ + current_local_tempname = tempzip; + } else { + /* close split */ + close_split(current_disk, y, tempzip); + y = NULL; + free(tempzip); + tempzip = NULL; + } + cd_entries_this_disk = 0; + bytes_this_split = 0; + + /* increment disk - disks are numbered 0, 1, 2, ... and + splits are 01, 02, ... */ + current_disk++; + + if (split_method == 2 && split_bell) { + /* bell when pause to ask for next split */ + putc('\007', mesg); + fflush(mesg); + } + + for (;;) { + /* if method 2 pause and allow changing path */ + if (split_method == 2) { + if (ask_for_split_write_path(current_disk) == 0) { + ZIPERR(ZE_ABORT, "could not write split"); + } + } + + /* open next split */ +#if defined(UNIX) && !defined(NO_MKSTEMP) + { + int yd; + int i; + + /* use mkstemp to avoid race condition and compiler warning */ + + if (tempath != NULL) + { + /* if -b used to set temp file dir use that for split temp */ + if ((tempzip = malloc(strlen(tempath) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, tempath); + if (lastchar(tempzip) != '/') + strcat(tempzip, "/"); + } + else + { + /* create path by stripping name and appending template */ + if ((tempzip = malloc(strlen(zipfile) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, zipfile); + for(i = strlen(tempzip); i > 0; i--) { + if (tempzip[i - 1] == '/') + break; + } + tempzip[i] = '\0'; + } + strcat(tempzip, "ziXXXXXX"); + + if ((yd = mkstemp(tempzip)) == EOF) { + ZIPERR(ZE_TEMP, tempzip); + } + if ((y = fdopen(yd, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } + } +#else + if ((tempzip = tempname(zipfile)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + if ((y = zfopen(tempzip, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } +#endif + + r = fwrite((char *)buffer + bytes_written, 1, bytes_to_write, y); + bytes_written += r; + bytes_this_split += r; + if (!(mode == BFWRITE_HEADER || + mode == BFWRITE_LOCALHEADER || + mode == BFWRITE_CENTRALHEADER)) { + bytes_this_entry += r; + } + if (bytes_to_write > r) { + /* buffer bigger than split */ + if (split_method == 2) { + /* let user choose another disk */ + zipwarn("Not enough room on disk", ""); + continue; + } else { + ZIPERR(ZE_WRITE, "Not enough room on disk"); + } + } + if (mode == BFWRITE_LOCALHEADER || + mode == BFWRITE_HEADER || + mode == BFWRITE_CENTRALHEADER) { + if (split_method == 1 && current_local_file && + current_local_disk != current_disk) { + /* We're opening a new split because the next header + did not fit on the last split. We need to now close + the last split and update the pointers for + the current split. */ + close_split(current_local_disk, current_local_file, + current_local_tempname); + free(current_local_tempname); + } + current_local_tempname = tempzip; + current_local_file = y; + current_local_offset = 0; + current_local_disk = current_disk; + } + break; + } + } + else + { + /* likely have more than fits but no splits */ + + /* probably already have error "no space left on device" */ + /* could let flush_outbuf() handle error but bfwrite() is called for + headers also */ + if (ferror(y)) + ziperr(ZE_WRITE, "write error on zip file"); + } + } + + + /* display dots for archive instead of for each file */ + if (display_globaldots) { + if (dot_size > 0) { + /* initial space */ + if (dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + /* assume a header will be written first, so avoid 0 */ + dot_count = 1; + } + /* skip incrementing dot count for small buffers like for headers */ + if (size * count > 1000) { + dot_count++; + if (dot_size <= dot_count * (zoff_t)size * (zoff_t)count) dot_count = 0; + } + } + if (dot_size && !dot_count) { + dot_count++; +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + + + return bytes_written; +} + + +#ifdef UNICODE_SUPPORT + +/*--------------------------------------------- + * Unicode conversion functions + * + * Provided by Paul Kienitz + * + * Some modifications to work with Zip + * + *--------------------------------------------- + */ + +/* + NOTES APPLICABLE TO ALL STRING FUNCTIONS: + + All of the x_to_y functions take parameters for an output buffer and + its available length, and return an int. The value returned is the + length of the string that the input produces, which may be larger than + the provided buffer length. If the returned value is less than the + buffer length, then the contents of the buffer will be null-terminated; + otherwise, it will not be terminated and may be invalid, possibly + stopping in the middle of a multibyte sequence. + + In all cases you may pass NULL as the buffer and/or 0 as the length, if + you just want to learn how much space the string is going to require. + + The functions will return -1 if the input is invalid UTF-8 or cannot be + encoded as UTF-8. +*/ + +/* utility functions for managing UTF-8 and UCS-4 strings */ + + +/* utf8_char_bytes + * + * Returns the number of bytes used by the first character in a UTF-8 + * string, or -1 if the UTF-8 is invalid or null. + */ +local int utf8_char_bytes(utf8) + ZCONST char *utf8; +{ + int t, r; + unsigned lead; + + if (!utf8) + return -1; /* no input */ + lead = (unsigned char) *utf8; + if (lead < 0x80) + r = 1; /* an ascii-7 character */ + else if (lead < 0xC0) + return -1; /* error: trailing byte without lead byte */ + else if (lead < 0xE0) + r = 2; /* an 11 bit character */ + else if (lead < 0xF0) + r = 3; /* a 16 bit character */ + else if (lead < 0xF8) + r = 4; /* a 21 bit character (the most currently used) */ + else if (lead < 0xFC) + r = 5; /* a 26 bit character (shouldn't happen) */ + else if (lead < 0xFE) + r = 6; /* a 31 bit character (shouldn't happen) */ + else + return -1; /* error: invalid lead byte */ + for (t = 1; t < r; t++) + if ((unsigned char) utf8[t] < 0x80 || (unsigned char) utf8[t] >= 0xC0) + return -1; /* error: not enough valid trailing bytes */ + return r; +} + + +/* ucs4_char_from_utf8 + * + * Given a reference to a pointer into a UTF-8 string, returns the next + * UCS-4 character and advances the pointer to the next character sequence. + * Returns ~0 and does not advance the pointer when input is ill-formed. + * + * Since the Unicode standard says 32-bit values won't be used (just + * up to the current 21-bit mappings) changed this to signed to allow -1 to + * be returned. + */ +long ucs4_char_from_utf8(utf8) + ZCONST char **utf8; +{ + ulg ret; + int t, bytes; + + if (!utf8) + return -1; /* no input */ + bytes = utf8_char_bytes(*utf8); + if (bytes <= 0) + return -1; /* invalid input */ + if (bytes == 1) + ret = **utf8; /* ascii-7 */ + else + ret = **utf8 & (0x7F >> bytes); /* lead byte of a multibyte sequence */ + (*utf8)++; + for (t = 1; t < bytes; t++) /* consume trailing bytes */ + ret = (ret << 6) | (*((*utf8)++) & 0x3F); + return (long) ret; +} + + +/* utf8_from_ucs4_char - Convert UCS char to UTF-8 + * + * Returns the number of bytes put into utf8buf to represent ch, from 1 to 6, + * or -1 if ch is too large to represent. utf8buf must have room for 6 bytes. + */ +local int utf8_from_ucs4_char(utf8buf, ch) + char *utf8buf; + ulg ch; +{ + int trailing = 0; + int leadmask = 0x80; + int leadbits = 0x3F; + ulg tch = ch; + int ret; + + if (ch > 0x7FFFFFFF) + return -1; /* UTF-8 can represent 31 bits */ + if (ch < 0x7F) + { + *utf8buf++ = (char) ch; /* ascii-7 */ + return 1; + } + do { + trailing++; + leadmask = (leadmask >> 1) | 0x80; + leadbits >>= 1; + tch >>= 6; + } while (tch & ~leadbits); + ret = trailing + 1; + /* produce lead byte */ + *utf8buf++ = (char) (leadmask | (ch >> (6 * trailing))); + /* produce trailing bytes */ + while (--trailing >= 0) + *utf8buf++ = (char) (0x80 | ((ch >> (6 * trailing)) & 0x3F)); + return ret; +} + + +/*===================================================================*/ + +/* utf8_to_ucs4_string - convert UTF-8 string to UCS string + * + * Return UCS count. Now returns int so can return -1. + */ +local int utf8_to_ucs4_string(utf8, ucs4buf, buflen) + ZCONST char *utf8; + ulg *ucs4buf; + int buflen; +{ + int count = 0; + + for (;;) + { + long ch = ucs4_char_from_utf8(&utf8); + if (ch == -1) + return -1; + else + { + if (ucs4buf && count < buflen) + ucs4buf[count] = ch; + if (ch == 0) + return count; + count++; + } + } +} + + +/* ucs4_string_to_utf8 + * + * + */ +local int ucs4_string_to_utf8(ucs4, utf8buf, buflen) + ZCONST ulg *ucs4; + char *utf8buf; + int buflen; +{ + char mb[6]; + int count = 0; + + if (!ucs4) + return -1; + for (;;) + { + int mbl = utf8_from_ucs4_char(mb, *ucs4++); + int c; + if (mbl <= 0) + return -1; + /* We could optimize this a bit by passing utf8buf + count */ + /* directly to utf8_from_ucs4_char when buflen >= count + 6... */ + c = buflen - count; + if (mbl < c) + c = mbl; + if (utf8buf && count < buflen) + strncpy(utf8buf + count, mb, c); + if (mbl == 1 && !mb[0]) + return count; /* terminating nul */ + count += mbl; + } +} + + +#if 0 /* currently unused */ +/* utf8_chars + * + * Wrapper: counts the actual unicode characters in a UTF-8 string. + */ +local int utf8_chars(utf8) + ZCONST char *utf8; +{ + return utf8_to_ucs4_string(utf8, NULL, 0); +} +#endif + + +/* --------------------------------------------------- */ +/* Unicode Support + * + * These functions common for all Unicode ports. + * + * These functions should allocate and return strings that can be + * freed with free(). + * + * 8/27/05 EG + * + * Use zwchar for wide char which is unsigned long + * in zip.h and 32 bits. This avoids problems with + * different sizes of wchar_t. + */ + +#ifdef WIN32 + +zwchar *wchar_to_wide_string(wchar_string) + wchar_t *wchar_string; +{ + int i; + int wchar_len; + zwchar *wide_string; + + wchar_len = wcslen(wchar_string); + + if ((wide_string = malloc((wchar_len + 1) * sizeof(zwchar))) == NULL) { + ZIPERR(ZE_MEM, "wchar to wide conversion"); + } + for (i = 0; i <= wchar_len; i++) { + wide_string[i] = wchar_string[i]; + } + + return wide_string; +} + +/* is_ascii_stringw + * Checks if a wide string is all ascii + */ +int is_ascii_stringw(wstring) + wchar_t *wstring; +{ + wchar_t *pw; + wchar_t cw; + + if (wstring == NULL) + return 0; + + for (pw = wstring; (cw = *pw) != '\0'; pw++) { + if (cw > 0x7F) { + return 0; + } + } + return 1; +} + +#endif + +/* is_ascii_string + * Checks if a string is all ascii + */ +int is_ascii_string(mbstring) + char *mbstring; +{ + char *p; + uch c; + + if (mbstring == NULL) + return 0; + + for (p = mbstring; (c = (uch)*p) != '\0'; p++) { + if (c > 0x7F) { + return 0; + } + } + return 1; +} + +/* local to UTF-8 */ +char *local_to_utf8_string(local_string) + char *local_string; +{ + zwchar *wide_string = local_to_wide_string(local_string); + char *utf8_string = wide_to_utf8_string(wide_string); + + free(wide_string); + return utf8_string; +} + +/* wide_char_to_escape_string + provides a string that represents a wide char not in local char set + + An initial try at an algorithm. Suggestions welcome. + + If not an ASCII char, probably need 2 bytes at least. So if + a 2-byte wide encode it as 4 hex digits with a leading #U. + Since the Unicode standard has been frozen, it looks like 3 bytes + should be enough for any large Unicode character. In these cases + prefix the string with #L. + So + #U1234 + is a 2-byte wide character with bytes 0x12 and 0x34 while + #L123456 + is a 3-byte wide with bytes 0x12, 0x34, and 0x56. + On Windows, wide that need two wide characters as a surrogate pair + to represent them need to be converted to a single number. + */ + + /* set this to the max bytes an escape can be */ +#define MAX_ESCAPE_BYTES 8 + +char *wide_char_to_escape_string(wide_char) + zwchar wide_char; +{ + int i; + zwchar w = wide_char; + uch b[9]; + char e[7]; + int len; + char *r; + + /* fill byte array with zeros */ + for (len = 0; len < sizeof(zwchar); len++) { + b[len] = 0; + } + /* get bytes in right to left order */ + for (len = 0; w; len++) { + b[len] = (char)(w % 0x100); + w /= 0x100; + } + + if ((r = malloc(MAX_ESCAPE_BYTES + 8)) == NULL) { + ZIPERR(ZE_MEM, "wide_char_to_escape_string"); + } + strcpy(r, "#"); + /* either 2 bytes or 4 bytes */ + if (len < 3) { + len = 2; + strcat(r, "U"); + } else { + len = 3; + strcat(r, "L"); + } + for (i = len - 1; i >= 0; i--) { + sprintf(e, "%02x", b[i]); + strcat(r, e); + } + return r; +} + +#if 0 +/* returns the wide character represented by the escape string */ +zwchar escape_string_to_wide(escape_string) + char *escape_string; +{ + int i; + zwchar w; + char c; + char u; + int len; + char *e = escape_string; + + if (e == NULL) { + return 0; + } + if (e[0] != '#') { + /* no leading # */ + return 0; + } + len = strlen(e); + /* either #U1234 or #L123456 format */ + if (len != 6 && len != 8) { + return 0; + } + w = 0; + if (e[1] == 'L') { + if (len != 8) { + return 0; + } + /* 3 bytes */ + for (i = 2; i < 8; i++) { + c = e[i]; + u = toupper(c); + if (u >= 'A' && u <= 'F') { + w = w * 0x10 + (zwchar)(u + 10 - 'A'); + } else if (c >= '0' && c <= '9') { + w = w * 0x10 + (zwchar)(c - '0'); + } else { + return 0; + } + } + } else if (e[1] == 'U') { + /* 2 bytes */ + for (i = 2; i < 6; i++) { + c = e[i]; + u = toupper(c); + if (u >= 'A' && u <= 'F') { + w = w * 0x10 + (zwchar)(u + 10 - 'A'); + } else if (c >= '0' && c <= '9') { + w = w * 0x10 + (zwchar)(c - '0'); + } else { + return 0; + } + } + } + return w; +} +#endif + + +char *local_to_escape_string(local_string) + char *local_string; +{ + zwchar *wide_string = local_to_wide_string(local_string); + char *escape_string = wide_to_escape_string(wide_string); + + free(wide_string); + return escape_string; +} + +#ifdef WIN32 +char *wchar_to_local_string(wstring) + wchar_t *wstring; +{ + zwchar *wide_string = wchar_to_wide_string(wstring); + char *local_string = wide_to_local_string(wide_string); + + free(wide_string); + + return local_string; +} +#endif + + +#ifndef WIN32 /* The Win32 port uses a system-specific variant. */ +/* convert wide character string to multi-byte character string */ +char *wide_to_local_string(wide_string) + zwchar *wide_string; +{ + int i; + wchar_t wc; + int b; + int state_dependent; + int wsize = 0; + int max_bytes = MB_CUR_MAX; + char buf[9]; + char *buffer = NULL; + char *local_string = NULL; + + for (wsize = 0; wide_string[wsize]; wsize++) ; + + if (MAX_ESCAPE_BYTES > max_bytes) + max_bytes = MAX_ESCAPE_BYTES; + + if ((buffer = (char *)malloc(wsize * max_bytes + 1)) == NULL) { + ZIPERR(ZE_MEM, "wide_to_local_string"); + } + + /* convert it */ + buffer[0] = '\0'; + /* set initial state if state-dependent encoding */ + wc = (wchar_t)'a'; + b = wctomb(NULL, wc); + if (b == 0) + state_dependent = 0; + else + state_dependent = 1; + for (i = 0; i < wsize; i++) { + if (sizeof(wchar_t) < 4 && wide_string[i] > 0xFFFF) { + /* wchar_t probably 2 bytes */ + /* could do surrogates if state_dependent and wctomb can do */ + wc = zwchar_to_wchar_t_default_char; + } else { + wc = (wchar_t)wide_string[i]; + } + b = wctomb(buf, wc); + if (unicode_escape_all) { + if (b == 1 && (uch)buf[0] <= 0x7f) { + /* ASCII */ + strncat(buffer, buf, b); + } else { + /* use escape for wide character */ + char *e = wide_char_to_escape_string(wide_string[i]); + strcat(buffer, e); + free(e); + } + } else if (b > 0) { + /* multi-byte char */ + strncat(buffer, buf, b); + } else { + /* no MB for this wide */ + if (use_wide_to_mb_default) { + /* default character */ + strcat(buffer, wide_to_mb_default_string); + } else { + /* use escape for wide character */ + char *e = wide_char_to_escape_string(wide_string[i]); + strcat(buffer, e); + free(e); + } + } + } + if ((local_string = (char *)malloc(strlen(buffer) + 1)) == NULL) { + free(buffer); + ZIPERR(ZE_MEM, "wide_to_local_string"); + } + strcpy(local_string, buffer); + free(buffer); + + return local_string; +} +#endif /* !WIN32 */ + + +/* convert wide character string to escaped string */ +char *wide_to_escape_string(wide_string) + zwchar *wide_string; +{ + int i; + int wsize = 0; + char buf[9]; + char *buffer = NULL; + char *escape_string = NULL; + + for (wsize = 0; wide_string[wsize]; wsize++) ; + + if ((buffer = (char *)malloc(wsize * MAX_ESCAPE_BYTES + 1)) == NULL) { + ZIPERR(ZE_MEM, "wide_to_escape_string"); + } + + /* convert it */ + buffer[0] = '\0'; + for (i = 0; i < wsize; i++) { + if (wide_string[i] <= 0x7f && isprint((char)wide_string[i])) { + /* ASCII */ + buf[0] = (char)wide_string[i]; + buf[1] = '\0'; + strcat(buffer, buf); + } else { + /* use escape for wide character */ + char *e = wide_char_to_escape_string(wide_string[i]); + strcat(buffer, e); + free(e); + } + } + if ((escape_string = (char *)malloc(strlen(buffer) + 1)) == NULL) { + ZIPERR(ZE_MEM, "wide_to_escape_string"); + } + strcpy(escape_string, buffer); + free(buffer); + + return escape_string; +} + + +/* convert local string to display character set string */ +char *local_to_display_string(local_string) + char *local_string; +{ + char *temp_string; + char *display_string; + + /* For Windows, OEM string should never be bigger than ANSI string, says + CharToOem description. + On UNIX, non-printable characters (0x00 - 0xFF) will be replaced by + "^x", so more space may be needed. Note that "^" itself is a valid + name character, so this leaves an ambiguity, but UnZip displays + names this way, too. (0x00 is not possible, I hope.) + For all other ports, just make a copy of local_string. + */ + +#ifdef UNIX + char *cp_dst; /* Character pointers used in the */ + char *cp_src; /* copying/changing procedure. */ +#endif + + if ((temp_string = (char *)malloc(2 * strlen(local_string) + 1)) == NULL) { + ZIPERR(ZE_MEM, "local_to_display_string"); + } + +#ifdef WIN32 + /* convert to OEM display character set */ + local_to_oem_string(temp_string, local_string); +#else +# ifdef UNIX + /* Copy source string, expanding non-printable characters to "^x". */ + cp_dst = temp_string; + cp_src = local_string; + while (*cp_src != '\0') { + if ((unsigned char)*cp_src < ' ') { + *cp_dst++ = '^'; + *cp_dst++ = '@'+ *cp_src++; + } + else { + *cp_dst++ = *cp_src++; + } + } + *cp_dst = '\0'; +# else /* not UNIX */ + strcpy(temp_string, local_string); +# endif /* UNIX */ +#endif + +#ifdef EBCDIC + { + char *ebc; + + if ((ebc = malloc(strlen(display_string) + 1)) == NULL) { + ZIPERR(ZE_MEM, "local_to_display_string"); + } + strtoebc(ebc, display_string); + free(display_string); + display_string = ebc; + } +#endif + + if ((display_string = (char *)malloc(strlen(temp_string) + 1)) == NULL) { + ZIPERR(ZE_MEM, "local_to_display_string"); + } + strcpy(display_string, temp_string); + free(temp_string); + + return display_string; +} + +/* UTF-8 to local */ +char *utf8_to_local_string(utf8_string) + char *utf8_string; +{ + zwchar *wide_string = utf8_to_wide_string(utf8_string); + char *loc = wide_to_local_string(wide_string); + if (wide_string) + free(wide_string); + return loc; +} + +/* UTF-8 to local */ +char *utf8_to_escape_string(utf8_string) + char *utf8_string; +{ + zwchar *wide_string = utf8_to_wide_string(utf8_string); + char *escape_string = wide_to_escape_string(wide_string); + free(wide_string); + return escape_string; +} + +#ifndef WIN32 /* The Win32 port uses a system-specific variant. */ +/* convert multi-byte character string to wide character string */ +zwchar *local_to_wide_string(local_string) + char *local_string; +{ + int wsize; + wchar_t *wc_string; + zwchar *wide_string; + + /* for now try to convert as string - fails if a bad char in string */ + wsize = mbstowcs(NULL, local_string, MB_CUR_MAX ); + if (wsize == (size_t)-1) { + /* could not convert */ + return NULL; + } + + /* convert it */ + if ((wc_string = (wchar_t *)malloc((wsize + 1) * sizeof(wchar_t))) == NULL) { + ZIPERR(ZE_MEM, "local_to_wide_string"); + } + wsize = mbstowcs(wc_string, local_string, strlen(local_string) + 1); + wc_string[wsize] = (wchar_t) 0; + + /* in case wchar_t is not zwchar */ + if ((wide_string = (zwchar *)malloc((wsize + 1) * sizeof(zwchar))) == NULL) { + ZIPERR(ZE_MEM, "local_to_wide_string"); + } + for (wsize = 0; (wide_string[wsize] = (zwchar)wc_string[wsize]); wsize++) ; + wide_string[wsize] = (zwchar)0; + free(wc_string); + + return wide_string; +} +#endif /* !WIN32 */ + + +#if 0 +/* All wchar functions are only used by Windows and are + now in win32zip.c so that the Windows functions can + be used and multiple character wide characters can + be handled easily. */ +# ifndef WIN32 +char *wchar_to_utf8_string(wstring) + wchar_t *wstring; +{ + zwchar *wide_string = wchar_to_wide_string(wstring); + char *local_string = wide_to_utf8_string(wide_string); + + free(wide_string); + + return local_string; +} +# endif +#endif + + +/* convert wide string to UTF-8 */ +char *wide_to_utf8_string(wide_string) + zwchar *wide_string; +{ + int mbcount; + char *utf8_string; + + /* get size of utf8 string */ + mbcount = ucs4_string_to_utf8(wide_string, NULL, 0); + if (mbcount == -1) + return NULL; + if ((utf8_string = (char *) malloc(mbcount + 1)) == NULL) { + ZIPERR(ZE_MEM, "wide_to_utf8_string"); + } + mbcount = ucs4_string_to_utf8(wide_string, utf8_string, mbcount + 1); + if (mbcount == -1) + return NULL; + + return utf8_string; +} + +/* convert UTF-8 string to wide string */ +zwchar *utf8_to_wide_string(utf8_string) + char *utf8_string; +{ + int wcount; + zwchar *wide_string; + + wcount = utf8_to_ucs4_string(utf8_string, NULL, 0); + if (wcount == -1) + return NULL; + if ((wide_string = (zwchar *) malloc((wcount + 2) * sizeof(zwchar))) == NULL) { + ZIPERR(ZE_MEM, "utf8_to_wide_string"); + } + wcount = utf8_to_ucs4_string(utf8_string, wide_string, wcount + 1); + + return wide_string; +} + + +#endif /* UNICODE_SUPPORT */ + + +/*--------------------------------------------------------------- + * Long option support + * 8/23/2003 + * + * Defines function get_option() to get and process the command + * line options and arguments from argv[]. The caller calls + * get_option() in a loop to get either one option and possible + * value or a non-option argument each loop. + * + * This version does not include argument file support and can + * work directly on argv. The argument file code complicates things and + * it seemed best to leave it out for now. If argument file support (reading + * in command line arguments stored in a file and inserting into + * command line where @filename is found) is added later the arguments + * can change and a freeable copy of argv will be needed and can be + * created using copy_args in the left out code. + * + * Supports short and long options as defined in the array options[] + * in zip.c, multiple short options in an argument (like -jlv), long + * option abbreviation (like --te for --temp-file if --te unique), + * short and long option values (like -b filename or --temp-file filename + * or --temp-file=filename), optional and required values, option negation + * by trailing - (like -S- to not include hidden and system files in MSDOS), + * value lists (like -x a b c), argument permuting (returning all options + * and values before any non-option arguments), and argument files (where any + * non-option non-value argument in form @path gets substituted with the + * white space separated arguments in the text file at path). In this + * version argument file support has been removed to simplify development but + * may be added later. + * + * E. Gordon + */ + + +/* message output - char casts are needed to handle constants */ +#define oWARN(message) zipwarn((char *) message, "") +#define oERR(err,message) ZIPERR(err, (char *) message) + + +/* Although the below provides some support for multibyte characters + the proper thing to do may be to use wide characters and support + Unicode. May get to it soon. EG + */ + +/* For now stay with muti-byte characters. May support wide characters + in Zip 3.1. + */ + +/* multibyte character set support + Multibyte characters use typically two or more sequential bytes + to represent additional characters than can fit in a single byte + character set. The code used here is based on the ANSI mblen function. */ +#ifdef MULTIBYTE_GETOPTNS + int mb_clen(ptr) + ZCONST char *ptr; + { + /* return the number of bytes that the char pointed to is. Return 1 if + null character or error like not start of valid multibyte character. */ + int cl; + + cl = mblen(ptr, MB_CUR_MAX); + return (cl > 0) ? cl : 1; + } +#endif + + + /* moved to zip.h */ +#if 0 +#ifdef UNICODE_SUPPORT +# define MB_CLEN(ptr) (1) +# define MB_NEXTCHAR(ptr) ((ptr)++) +# ifdef MULTIBYTE_GETOPTNS +# undef MULTIBYTE_GETOPTNS +# endif +#else +# ifdef _MBCS +# ifndef MULTIBYTE_GETOPTNS +# define MULTIBYTE_GETOPTNS +# endif +# endif +/* multibyte character set support + Multibyte characters use typically two or more sequential bytes + to represent additional characters than can fit in a single byte + character set. The code used here is based on the ANSI mblen function. */ +# ifdef MULTIBYTE_GETOPTNS + local int mb_clen OF((ZCONST char *)); /* declare proto first */ + local int mb_clen(ptr) + ZCONST char *ptr; + { + /* return the number of bytes that the char pointed to is. Return 1 if + null character or error like not start of valid multibyte character. */ + int cl; + + cl = mblen(ptr, MB_CUR_MAX); + return (cl > 0) ? cl : 1; + } +# define MB_CLEN(ptr) mb_clen(ptr) +# define MB_NEXTCHAR(ptr) ((ptr) += MB_CLEN(ptr)) +# else +# define MB_CLEN(ptr) (1) +# define MB_NEXTCHAR(ptr) ((ptr)++) +# endif +#endif +#endif + + +/* constants */ + +/* function get_args_from_arg_file() can return this in depth parameter */ +#define ARG_FILE_ERR -1 + +/* internal settings for optchar */ +#define SKIP_VALUE_ARG -1 +#define THIS_ARG_DONE -2 +#define START_VALUE_LIST -3 +#define IN_VALUE_LIST -4 +#define NON_OPTION_ARG -5 +#define STOP_VALUE_LIST -6 +/* 7/25/04 EG */ +#define READ_REST_ARGS_VERBATIM -7 + + +/* global veriables */ + +int enable_permute = 1; /* yes - return options first */ +/* 7/25/04 EG */ +int doubledash_ends_options = 1; /* when -- what follows are not options */ + +/* buffer for error messages (this sizing is a guess but must hold 2 paths) */ +#define OPTIONERR_BUF_SIZE (FNMAX * 2 + 4000) +local char Far optionerrbuf[OPTIONERR_BUF_SIZE + 1]; + +/* error messages */ +static ZCONST char Far op_not_neg_err[] = "option %s not negatable"; +static ZCONST char Far op_req_val_err[] = "option %s requires a value"; +static ZCONST char Far op_no_allow_val_err[] = "option %s does not allow a value"; +static ZCONST char Far sh_op_not_sup_err[] = "short option '%c' not supported"; +static ZCONST char Far oco_req_val_err[] = "option %s requires one character value"; +static ZCONST char Far oco_no_mbc_err[] = "option %s does not support multibyte values"; +static ZCONST char Far num_req_val_err[] = "option %s requires number value"; +static ZCONST char Far long_op_ambig_err[] = "long option '%s' ambiguous"; +static ZCONST char Far long_op_not_sup_err[] = "long option '%s' not supported"; + +static ZCONST char Far no_arg_files_err[] = "argument files not enabled\n"; + + +/* below removed as only used for processing argument files */ + +/* get_nextarg */ +/* get_args_from_string */ +/* insert_args */ +/* get_args_from_arg_file */ + + +/* copy error, option name, and option description if any to buf */ +local int optionerr(buf, err, optind, islong) + char *buf; + ZCONST char *err; + int optind; + int islong; +{ + char optname[50]; + + if (options[optind].name && options[optind].name[0] != '\0') { + if (islong) + sprintf(optname, "'%s' (%s)", options[optind].longopt, options[optind].name); + else + sprintf(optname, "'%s' (%s)", options[optind].shortopt, options[optind].name); + } else { + if (islong) + sprintf(optname, "'%s'", options[optind].longopt); + else + sprintf(optname, "'%s'", options[optind].shortopt); + } + sprintf(buf, err, optname); + return 0; +} + + +/* copy_args + * + * Copy arguments in args, allocating storage with malloc. + * Copies until a NULL argument is found or until max_args args + * including args[0] are copied. Set max_args to 0 to copy + * until NULL. Always terminates returned args[] with NULL arg. + * + * Any argument in the returned args can be freed with free(). Any + * freed argument should be replaced with either another string + * allocated with malloc or by NULL if last argument so that free_args + * will properly work. + */ +char **copy_args(args, max_args) + char **args; + int max_args; +{ + int j; + char **new_args; + + if (args == NULL) { + return NULL; + } + + /* count args */ + for (j = 0; args[j] && (max_args == 0 || j < max_args); j++) ; + + if ((new_args = (char **) malloc((j + 1) * sizeof(char *))) == NULL) { + oERR(ZE_MEM, "ca"); + } + + for (j = 0; args[j] && (max_args == 0 || j < max_args); j++) { + if (args[j] == NULL) { + /* null argument is end of args */ + new_args[j] = NULL; + break; + } + if ((new_args[j] = malloc(strlen(args[j]) + 1)) == NULL) { + free_args(new_args); + oERR(ZE_MEM, "ca"); + } + strcpy(new_args[j], args[j]); + } + new_args[j] = NULL; + + return new_args; +} + + +/* free args - free args created with one of these functions */ +int free_args(args) + char **args; +{ + int i; + + if (args == NULL) { + return 0; + } + + for (i = 0; args[i]; i++) { + free(args[i]); + } + free(args); + return i; +} + + +/* insert_arg + * + * Insert the argument arg into the array args before argument at_arg. + * Return the new count of arguments (argc). + * + * If free_args is true, this function frees the old args array + * (but not the component strings). DO NOT set free_args on original + * argv but only on args allocated with malloc. + */ + +int insert_arg(pargs, arg, at_arg, free_args) + char ***pargs; + ZCONST char *arg; + int at_arg; + int free_args; +{ + char *newarg = NULL; + char **args; + char **newargs = NULL; + int argnum; + int newargnum; + int argcnt; + int newargcnt; + + if (pargs == NULL) { + return 0; + } + args = *pargs; + + /* count args */ + if (args == NULL) { + argcnt = 0; + } else { + for (argcnt = 0; args[argcnt]; argcnt++) ; + } + if (arg == NULL) { + /* done */ + return argcnt; + } + newargcnt = argcnt + 1; + + /* get storage for new args */ + if ((newargs = (char **) malloc((newargcnt + 1) * sizeof(char *))) == NULL) { + oERR(ZE_MEM, "ia"); + } + + /* copy argument pointers from args to position at_arg, copy arg, then rest args */ + argnum = 0; + newargnum = 0; + if (args) { + for (; args[argnum] && argnum < at_arg; argnum++) { + newargs[newargnum++] = args[argnum]; + } + } + /* copy new arg */ + if ((newarg = (char *) malloc(strlen(arg) + 1)) == NULL) { + oERR(ZE_MEM, "ia"); + } + strcpy(newarg, arg); + + newargs[newargnum++] = newarg; + if (args) { + for ( ; args[argnum]; argnum++) { + newargs[newargnum++] = args[argnum]; + } + } + newargs[newargnum] = NULL; + + /* free old args array but not component strings - this assumes that + args was allocated with malloc as copy_args does. DO NOT DO THIS + on the original argv. + */ + if (free_args) + free(args); + + *pargs = newargs; + + return newargnum; +} + +/* ------------------------------------- */ + + + + +/* get_shortopt + * + * Get next short option from arg. The state is stored in argnum, optchar, and + * option_num so no static storage is used. Returns the option_ID. + * + * parameters: + * args - argv array of arguments + * argnum - index of current arg in args + * optchar - pointer to index of next char to process. Can be 0 or + * const defined at top of this file like THIS_ARG_DONE + * negated - on return pointer to int set to 1 if option negated or 0 otherwise + * value - on return pointer to string set to value of option if any or NULL + * if none. If value is returned then the caller should free() + * it when not needed anymore. + * option_num - pointer to index in options[] of returned option or + * o_NO_OPTION_MATCH if none. Do not change as used by + * value lists. + * depth - recursion depth (0 at top level, 1 or more in arg files) + */ +local unsigned long get_shortopt(args, argnum, optchar, negated, value, + option_num, depth) + char **args; + int argnum; + int *optchar; + int *negated; + char **value; + int *option_num; + int depth; +{ + char *shortopt; + int clen; + char *nextchar; + char *s; + char *start; + int op; + char *arg; + int match = -1; + + + /* get arg */ + arg = args[argnum]; + /* current char in arg */ + nextchar = arg + (*optchar); + clen = MB_CLEN(nextchar); + /* next char in arg */ + (*optchar) += clen; + /* get first char of short option */ + shortopt = arg + (*optchar); + /* no value */ + *value = NULL; + + if (*shortopt == '\0') { + /* no more options in arg */ + *optchar = 0; + *option_num = o_NO_OPTION_MATCH; + return 0; + } + + /* look for match in options */ + clen = MB_CLEN(shortopt); + for (op = 0; options[op].option_ID; op++) { + s = options[op].shortopt; + if (s && s[0] == shortopt[0]) { + if (s[1] == '\0' && clen == 1) { + /* single char match */ + match = op; + } else { + /* 2 wide short opt. Could support more chars but should use long opts instead */ + if (s[1] == shortopt[1]) { + /* match 2 char short opt or 2 byte char */ + match = op; + if (clen == 1) (*optchar)++; + break; + } + } + } + } + + if (match > -1) { + /* match */ + clen = MB_CLEN(shortopt); + nextchar = arg + (*optchar) + clen; + /* check for trailing dash negating option */ + if (*nextchar == '-') { + /* negated */ + if (options[match].negatable == o_NOT_NEGATABLE) { + if (options[match].value_type == o_NO_VALUE) { + optionerr(optionerrbuf, op_not_neg_err, match, 0); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + } else { + *negated = 1; + /* set up to skip negating dash */ + (*optchar) += clen; + clen = 1; + } + } + + /* value */ + clen = MB_CLEN(arg + (*optchar)); + /* optional value, one char value, and number value must follow option */ + if (options[match].value_type == o_ONE_CHAR_VALUE) { + /* one char value */ + if (arg[(*optchar) + clen]) { + /* has value */ + if (MB_CLEN(arg + (*optchar) + clen) > 1) { + /* multibyte value not allowed for now */ + optionerr(optionerrbuf, oco_no_mbc_err, match, 0); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + if ((*value = (char *) malloc(2)) == NULL) { + oERR(ZE_MEM, "gso"); + } + (*value)[0] = *(arg + (*optchar) + clen); + (*value)[1] = '\0'; + *optchar += clen; + clen = 1; + } else { + /* one char values require a value */ + optionerr(optionerrbuf, oco_req_val_err, match, 0); + if (depth > 0) { + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + } else if (options[match].value_type == o_NUMBER_VALUE) { + /* read chars until end of number */ + start = arg + (*optchar) + clen; + if (*start == '+' || *start == '-') { + start++; + } + s = start; + for (; isdigit(*s); MB_NEXTCHAR(s)) ; + if (s == start) { + /* no digits */ + optionerr(optionerrbuf, num_req_val_err, match, 0); + if (depth > 0) { + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + start = arg + (*optchar) + clen; + if ((*value = (char *) malloc((int)(s - start) + 1)) == NULL) { + oERR(ZE_MEM, "gso"); + } + *optchar += (int)(s - start); + strncpy(*value, start, (int)(s - start)); + (*value)[(int)(s - start)] = '\0'; + clen = MB_CLEN(s); + } else if (options[match].value_type == o_OPTIONAL_VALUE) { + /* optional value */ + /* This seemed inconsistent so now if no value attached to argument look + to the next argument if that argument is not an option for option + value - 11/12/04 EG */ + if (arg[(*optchar) + clen]) { + /* has value */ + /* add support for optional = - 2/6/05 EG */ + if (arg[(*optchar) + clen] == '=') { + /* skip = */ + clen++; + } + if (arg[(*optchar) + clen]) { + if ((*value = (char *)malloc(strlen(arg + (*optchar) + clen) + 1)) + == NULL) { + oERR(ZE_MEM, "gso"); + } + strcpy(*value, arg + (*optchar) + clen); + } + *optchar = THIS_ARG_DONE; + } else if (args[argnum + 1] && args[argnum + 1][0] != '-') { + /* use next arg for value */ + if ((*value = (char *)malloc(strlen(args[argnum + 1]) + 1)) == NULL) { + oERR(ZE_MEM, "gso"); + } + /* using next arg as value */ + strcpy(*value, args[argnum + 1]); + *optchar = SKIP_VALUE_ARG; + } + } else if (options[match].value_type == o_REQUIRED_VALUE || + options[match].value_type == o_VALUE_LIST) { + /* see if follows option */ + if (arg[(*optchar) + clen]) { + /* has value following option as -ovalue */ + /* add support for optional = - 6/5/05 EG */ + if (arg[(*optchar) + clen] == '=') { + /* skip = */ + clen++; + } + if ((*value = (char *)malloc(strlen(arg + (*optchar) + clen) + 1)) + == NULL) { + oERR(ZE_MEM, "gso"); + } + strcpy(*value, arg + (*optchar) + clen); + *optchar = THIS_ARG_DONE; + } else { + /* use next arg for value */ + if (args[argnum + 1]) { + if ((*value = (char *)malloc(strlen(args[argnum + 1]) + 1)) == NULL) { + oERR(ZE_MEM, "gso"); + } + strcpy(*value, args[argnum + 1]); + if (options[match].value_type == o_VALUE_LIST) { + *optchar = START_VALUE_LIST; + } else { + *optchar = SKIP_VALUE_ARG; + } + } else { + /* no value found */ + optionerr(optionerrbuf, op_req_val_err, match, 0); + if (depth > 0) { + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + } + } + + *option_num = match; + return options[match].option_ID; + } + sprintf(optionerrbuf, sh_op_not_sup_err, *shortopt); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + return 0; +} + + +/* get_longopt + * + * Get the long option in args array at argnum. + * Parameters same as for get_shortopt. + */ + +local unsigned long get_longopt(args, argnum, optchar, negated, value, + option_num, depth) + char **args; + int argnum; + int *optchar; + int *negated; + char **value; + int *option_num; + int depth; +{ + char *longopt; + char *lastchr; + char *valuestart; + int op; + char *arg; + int match = -1; + *value = NULL; + + if (args == NULL) { + *option_num = o_NO_OPTION_MATCH; + return 0; + } + if (args[argnum] == NULL) { + *option_num = o_NO_OPTION_MATCH; + return 0; + } + /* copy arg so can chop end if value */ + if ((arg = (char *)malloc(strlen(args[argnum]) + 1)) == NULL) { + oERR(ZE_MEM, "glo"); + } + strcpy(arg, args[argnum]); + + /* get option */ + longopt = arg + 2; + /* no value */ + *value = NULL; + + /* find = */ + for (lastchr = longopt, valuestart = longopt; + *valuestart && *valuestart != '='; + lastchr = valuestart, MB_NEXTCHAR(valuestart)) ; + if (*valuestart) { + /* found =value */ + *valuestart = '\0'; + valuestart++; + } else { + valuestart = NULL; + } + + if (*lastchr == '-') { + /* option negated */ + *negated = 1; + *lastchr = '\0'; + } else { + *negated = 0; + } + + /* look for long option match */ + for (op = 0; options[op].option_ID; op++) { + if (options[op].longopt && strcmp(options[op].longopt, longopt) == 0) { + /* exact match */ + match = op; + break; + } + if (options[op].longopt && strncmp(options[op].longopt, longopt, strlen(longopt)) == 0) { + if (match > -1) { + sprintf(optionerrbuf, long_op_ambig_err, longopt); + free(arg); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + match = op; + } + } + + if (match == -1) { + sprintf(optionerrbuf, long_op_not_sup_err, longopt); + free(arg); + if (depth > 0) { + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + + /* one long option an arg */ + *optchar = THIS_ARG_DONE; + + /* if negated then see if allowed */ + if (*negated && options[match].negatable == o_NOT_NEGATABLE) { + optionerr(optionerrbuf, op_not_neg_err, match, 1); + free(arg); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + /* get value */ + if (options[match].value_type == o_OPTIONAL_VALUE) { + /* optional value in form option=value */ + if (valuestart) { + /* option=value */ + if ((*value = (char *)malloc(strlen(valuestart) + 1)) == NULL) { + free(arg); + oERR(ZE_MEM, "glo"); + } + strcpy(*value, valuestart); + } + } else if (options[match].value_type == o_REQUIRED_VALUE || + options[match].value_type == o_NUMBER_VALUE || + options[match].value_type == o_ONE_CHAR_VALUE || + options[match].value_type == o_VALUE_LIST) { + /* handle long option one char and number value as required value */ + if (valuestart) { + /* option=value */ + if ((*value = (char *)malloc(strlen(valuestart) + 1)) == NULL) { + free(arg); + oERR(ZE_MEM, "glo"); + } + strcpy(*value, valuestart); + } else { + /* use next arg */ + if (args[argnum + 1]) { + if ((*value = (char *)malloc(strlen(args[argnum + 1]) + 1)) == NULL) { + free(arg); + oERR(ZE_MEM, "glo"); + } + /* using next arg as value */ + strcpy(*value, args[argnum + 1]); + if (options[match].value_type == o_VALUE_LIST) { + *optchar = START_VALUE_LIST; + } else { + *optchar = SKIP_VALUE_ARG; + } + } else { + /* no value found */ + optionerr(optionerrbuf, op_req_val_err, match, 1); + free(arg); + if (depth > 0) { + /* unwind */ + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + } + } else if (options[match].value_type == o_NO_VALUE) { + /* this option does not accept a value */ + if (valuestart) { + /* --option=value */ + optionerr(optionerrbuf, op_no_allow_val_err, match, 1); + free(arg); + if (depth > 0) { + oWARN(optionerrbuf); + return o_ARG_FILE_ERR; + } else { + oERR(ZE_PARMS, optionerrbuf); + } + } + } + free(arg); + + *option_num = match; + return options[match].option_ID; +} + + + +/* get_option + * + * Main interface for user. Use this function to get options, values and + * non-option arguments from a command line provided in argv form. + * + * To use get_option() first define valid options by setting + * the global variable options[] to an array of option_struct. Also + * either change defaults below or make variables global and set elsewhere. + * Zip uses below defaults. + * + * Call get_option() to get an option (like -b or --temp-file) and any + * value for that option (like filename for -b) or a non-option argument + * (like archive name) each call. If *value* is not NULL after calling + * get_option() it is a returned value and the caller should either store + * the char pointer or free() it before calling get_option() again to avoid + * leaking memory. If a non-option non-value argument is returned get_option() + * returns o_NON_OPTION_ARG and value is set to the entire argument. + * When there are no more arguments get_option() returns 0. + * + * The parameters argnum (after set to 0 on initial call), + * optchar, first_nonopt_arg, option_num, and depth (after initial + * call) are set and maintained by get_option() and should not be + * changed. The parameters argc, negated, and value are outputs and + * can be used by the calling program. get_option() returns either the + * option_ID for the current option, a special value defined in + * zip.h, or 0 when no more arguments. + * + * The value returned by get_option() is the ID value in the options + * table. This value can be duplicated in the table if different + * options are really the same option. The index into the options[] + * table is given by option_num, though the ID should be used as + * option numbers change when the table is changed. The ID must + * not be 0 for any option as this ends the table. If get_option() + * finds an option not in the table it calls oERR to post an + * error and exit. Errors also result if the option requires a + * value that is missing, a value is present but the option does + * not take one, and an option is negated but is not + * negatable. Non-option arguments return o_NON_OPTION_ARG + * with the entire argument in value. + * + * For Zip, permuting is on and all options and their values are + * returned before any non-option arguments like archive name. + * + * The arguments "-" alone and "--" alone return as non-option arguments. + * Note that "-" should not be used as part of a short option + * entry in the table but can be used in the middle of long + * options such as in the long option "a-long-option". Now "--" alone + * stops option processing, returning any arguments following "--" as + * non-option arguments instead of options. + * + * Argument file support is removed from this version. It may be added later. + * + * After each call: + * argc is set to the current size of args[] but should not change + * with argument file support removed, + * argnum is the index of the current arg, + * value is either the value of the returned option or non-option + * argument or NULL if option with no value, + * negated is set if the option was negated by a trailing dash (-) + * option_num is set to either the index in options[] for the option or + * o_NO_OPTION_MATCH if no match. + * Negation is checked before the value is read if the option is negatable so + * that the - is not included in the value. If the option is not negatable + * but takes a value then the - will start the value. If permuting then + * argnum and first_nonopt_arg are unreliable and should not be used. + * + * Command line is read from left to right. As get_option() finds non-option + * arguments (arguments not starting with - and that are not values to options) + * it moves later options and values in front of the non-option arguments. + * This permuting is turned off by setting enable_permute to 0. Then + * get_option() will return options and non-option arguments in the order + * found. Currently permuting is only done after an argument is completely + * processed so that any value can be moved with options they go with. All + * state information is stored in the parameters argnum, optchar, + * first_nonopt_arg and option_num. You should not change these after the + * first call to get_option(). If you need to back up to a previous arg then + * set argnum to that arg (remembering that args may have been permuted) and + * set optchar = 0 and first_nonopt_arg to the first non-option argument if + * permuting. After all arguments are returned the next call to get_option() + * returns 0. The caller can then call free_args(args) if appropriate. + * + * get_option() accepts arguments in the following forms: + * short options + * of 1 and 2 characters, e.g. a, b, cc, d, and ba, after a single + * leading -, as in -abccdba. In this example if 'b' is followed by 'a' + * it matches short option 'ba' else it is interpreted as short option + * b followed by another option. The character - is not legal as a + * short option or as part of a 2 character short option. + * + * If a short option has a value it immediately follows the option or + * if that option is the end of the arg then the next arg is used as + * the value. So if short option e has a value, it can be given as + * -evalue + * or + * -e value + * and now + * -e=value + * but now that = is optional a leading = is stripped for the first. + * This change allows optional short option values to be defaulted as + * -e= + * Either optional or required values can be specified. Optional values + * now use both forms as ignoring the later got confusing. Any + * non-value short options can preceed a valued short option as in + * -abevalue + * Some value types (one_char and number) allow options after the value + * so if oc is an option that takes a character and n takes a number + * then + * -abocVccn42evalue + * returns value V for oc and value 42 for n. All values are strings + * so programs may have to convert the "42" to a number. See long + * options below for how value lists are handled. + * + * Any short option can be negated by following it with -. Any - is + * handled and skipped over before any value is read unless the option + * is not negatable but takes a value and then - starts the value. + * + * If the value for an optional value is just =, then treated as no + * value. + * + * long options + * of arbitrary length are assumed if an arg starts with -- but is not + * exactly --. Long options are given one per arg and can be abbreviated + * if the abbreviation uniquely matches one of the long options. + * Exact matches always match before partial matches. If ambiguous an + * error is generated. + * + * Values are specified either in the form + * --longoption=value + * or can be the following arg if the value is required as in + * --longoption value + * Optional values to long options must be in the first form. + * + * Value lists are specified by o_VALUE_LIST and consist of an option + * that takes a value followed by one or more value arguments. + * The two forms are + * --option=value + * or + * -ovalue + * for a single value or + * --option value1 value2 value3 ... --option2 + * or + * -o value1 value2 value3 ... + * for a list of values. The list ends at the next option, the + * end of the command line, or at a single "@" argument. + * Each value is treated as if it was preceeded by the option, so + * --option1 val1 val2 + * with option1 value_type set to o_VALUE_LIST is the same as + * --option1=val1 --option1=val2 + * + * Long options can be negated by following the option with - as in + * --longoption- + * Long options with values can also be negated if this makes sense for + * the caller as: + * --longoption-=value + * If = is not followed by anything it is treated as no value. + * + * @path + * When an argument in the form @path is encountered, the file at path + * is opened and white space separated arguments read from the file + * and inserted into the command line at that point as if the contents + * of the file were directly typed at that location. The file can + * have options, files to zip, or anything appropriate at that location + * in the command line. Since Zip has permuting enabled, options and + * files will propagate to the appropriate locations in the command + * line. + * + * Argument files support has been removed from this version. It may + * be added back later. + * + * non-option argument + * is any argument not given above. If enable_permute is 1 then + * these are returned after all options, otherwise all options and + * args are returned in order. Returns option ID o_NON_OPTION_ARG + * and sets value to the argument. + * + * + * Arguments to get_option: + * char ***pargs - pointer to arg array in the argv form + * int *argc - returns the current argc for args incl. args[0] + * int *argnum - the index of the current argument (caller + * should set = 0 on first call and not change + * after that) + * int *optchar - index of next short opt in arg or special + * int *first_nonopt_arg - used by get_option to permute args + * int *negated - option was negated (had trailing -) + * char *value - value of option if any (free when done with it) or NULL + * int *option_num - the index in options of the last option returned + * (can be o_NO_OPTION_MATCH) + * int recursion_depth - current depth of recursion + * (always set to 0 by caller) + * (always 0 with argument files support removed) + * + * Caller should only read the returned option ID and the value, negated, + * and option_num (if required) parameters after each call. + * + * Ed Gordon + * 24 August 2003 (last updated 2 April 2008 EG) + * + */ + +unsigned long get_option(pargs, argc, argnum, optchar, value, + negated, first_nonopt_arg, option_num, recursion_depth) + char ***pargs; + int *argc; + int *argnum; + int *optchar; + char **value; + int *negated; + int *first_nonopt_arg; + int *option_num; + int recursion_depth; +{ + char **args; + unsigned long option_ID; + + int argcnt; + int first_nonoption_arg; + char *arg = NULL; + int h; + int optc; + int argn; + int j; + int v; + int read_rest_args_verbatim = 0; /* 7/25/04 - ignore options and arg files for rest args */ + + /* value is outdated. The caller should free value before + calling get_option again. */ + *value = NULL; + + /* if args is NULL then done */ + if (pargs == NULL) { + *argc = 0; + return 0; + } + args = *pargs; + if (args == NULL) { + *argc = 0; + return 0; + } + + /* count args */ + for (argcnt = 0; args[argcnt]; argcnt++) ; + + /* if no provided args then nothing to do */ + if (argcnt < 1 || (recursion_depth == 0 && argcnt < 2)) { + *argc = argcnt; + /* return 0 to note that no args are left */ + return 0; + } + + *negated = 0; + first_nonoption_arg = *first_nonopt_arg; + argn = *argnum; + optc = *optchar; + + if (optc == READ_REST_ARGS_VERBATIM) { + read_rest_args_verbatim = 1; + } + + if (argn == -1 || (recursion_depth == 0 && argn == 0)) { + /* first call */ + /* if depth = 0 then args[0] is argv[0] so skip */ + *option_num = o_NO_OPTION_MATCH; + optc = THIS_ARG_DONE; + first_nonoption_arg = -1; + } + + /* if option_num is set then restore last option_ID in case continuing value list */ + option_ID = 0; + if (*option_num != o_NO_OPTION_MATCH) { + option_ID = options[*option_num].option_ID; + } + + /* get next option if any */ + for (;;) { + if (read_rest_args_verbatim) { + /* rest of args after "--" are non-option args if doubledash_ends_options set */ + argn++; + if (argn > argcnt || args[argn] == NULL) { + /* done */ + option_ID = 0; + break; + } + arg = args[argn]; + if ((*value = (char *)malloc(strlen(arg) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, arg); + *option_num = o_NO_OPTION_MATCH; + option_ID = o_NON_OPTION_ARG; + break; + + /* permute non-option args after option args so options are returned first */ + } else if (enable_permute) { + if (optc == SKIP_VALUE_ARG || optc == THIS_ARG_DONE || + optc == START_VALUE_LIST || optc == IN_VALUE_LIST || + optc == STOP_VALUE_LIST) { + /* moved to new arg */ + if (first_nonoption_arg > -1 && args[first_nonoption_arg]) { + /* do the permuting - move non-options after this option */ + /* if option and value separate args or starting list skip option */ + if (optc == SKIP_VALUE_ARG || optc == START_VALUE_LIST) { + v = 1; + } else { + v = 0; + } + for (h = first_nonoption_arg; h < argn; h++) { + arg = args[first_nonoption_arg]; + for (j = first_nonoption_arg; j < argn + v; j++) { + args[j] = args[j + 1]; + } + args[j] = arg; + } + first_nonoption_arg += 1 + v; + } + } + } else if (optc == NON_OPTION_ARG) { + /* if not permuting then already returned arg */ + optc = THIS_ARG_DONE; + } + + /* value lists */ + if (optc == STOP_VALUE_LIST) { + optc = THIS_ARG_DONE; + } + + if (optc == START_VALUE_LIST || optc == IN_VALUE_LIST) { + if (optc == START_VALUE_LIST) { + /* already returned first value */ + argn++; + optc = IN_VALUE_LIST; + } + argn++; + arg = args[argn]; + /* if end of args and still in list and there are non-option args then + terminate list */ + if (arg == NULL && (optc == START_VALUE_LIST || optc == IN_VALUE_LIST) + && first_nonoption_arg > -1) { + /* terminate value list with @ */ + /* this is only needed for argument files */ + /* but is also good for show command line so command lines with lists + can always be read back in */ + argcnt = insert_arg(&args, "@", first_nonoption_arg, 1); + argn++; + if (first_nonoption_arg > -1) { + first_nonoption_arg++; + } + } + + arg = args[argn]; + if (arg && arg[0] == '@' && arg[1] == '\0') { + /* inserted arguments terminator */ + optc = STOP_VALUE_LIST; + continue; + } else if (arg && arg[0] != '-') { /* not option */ + /* - and -- are not allowed in value lists unless escaped */ + /* another value in value list */ + if ((*value = (char *)malloc(strlen(args[argn]) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, args[argn]); + break; + + } else { + argn--; + optc = THIS_ARG_DONE; + } + } + + /* move to next arg */ + if (optc == SKIP_VALUE_ARG) { + argn += 2; + optc = 0; + } else if (optc == THIS_ARG_DONE) { + argn++; + optc = 0; + } + if (argn > argcnt) { + break; + } + if (args[argn] == NULL) { + /* done unless permuting and non-option args */ + if (first_nonoption_arg > -1 && args[first_nonoption_arg]) { + /* return non-option arguments at end */ + if (optc == NON_OPTION_ARG) { + first_nonoption_arg++; + } + /* after first pass args are permuted but skipped over non-option args */ + /* swap so argn points to first non-option arg */ + j = argn; + argn = first_nonoption_arg; + first_nonoption_arg = j; + } + if (argn > argcnt || args[argn] == NULL) { + /* done */ + option_ID = 0; + break; + } + } + + /* after swap first_nonoption_arg points to end which is NULL */ + if (first_nonoption_arg > -1 && (args[first_nonoption_arg] == NULL)) { + /* only non-option args left */ + if (optc == NON_OPTION_ARG) { + argn++; + } + if (argn > argcnt || args[argn] == NULL) { + /* done */ + option_ID = 0; + break; + } + if ((*value = (char *)malloc(strlen(args[argn]) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, args[argn]); + optc = NON_OPTION_ARG; + option_ID = o_NON_OPTION_ARG; + break; + } + + arg = args[argn]; + + /* is it an option */ + if (arg[0] == '-') { + /* option */ + if (arg[1] == '\0') { + /* arg = - */ + /* treat like non-option arg */ + *option_num = o_NO_OPTION_MATCH; + if (enable_permute) { + /* permute args to move all non-option args to end */ + if (first_nonoption_arg < 0) { + first_nonoption_arg = argn; + } + argn++; + } else { + /* not permute args so return non-option args when found */ + if ((*value = (char *)malloc(strlen(arg) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, arg); + optc = NON_OPTION_ARG; + option_ID = o_NON_OPTION_ARG; + break; + } + + } else if (arg[1] == '-') { + /* long option */ + if (arg[2] == '\0') { + /* arg = -- */ + if (doubledash_ends_options) { + /* Now -- stops permuting and forces the rest of + the command line to be read verbatim - 7/25/04 EG */ + + /* never permute args after -- and return as non-option args */ + if (first_nonoption_arg < 1) { + /* -- is first non-option argument - 8/7/04 EG */ + argn--; + } else { + /* go back to start of non-option args - 8/7/04 EG */ + argn = first_nonoption_arg - 1; + } + + /* disable permuting and treat remaining arguments as not + options */ + read_rest_args_verbatim = 1; + optc = READ_REST_ARGS_VERBATIM; + + } else { + /* treat like non-option arg */ + *option_num = o_NO_OPTION_MATCH; + if (enable_permute) { + /* permute args to move all non-option args to end */ + if (first_nonoption_arg < 0) { + first_nonoption_arg = argn; + } + argn++; + } else { + /* not permute args so return non-option args when found */ + if ((*value = (char *)malloc(strlen(arg) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, arg); + optc = NON_OPTION_ARG; + option_ID = o_NON_OPTION_ARG; + break; + } + } + + } else { + option_ID = get_longopt(args, argn, &optc, negated, value, option_num, recursion_depth); + if (option_ID == o_ARG_FILE_ERR) { + /* unwind as only get this if recursion_depth > 0 */ + return option_ID; + } + break; + } + + } else { + /* short option */ + option_ID = get_shortopt(args, argn, &optc, negated, value, option_num, recursion_depth); + + if (option_ID == o_ARG_FILE_ERR) { + /* unwind as only get this if recursion_depth > 0 */ + return option_ID; + } + + if (optc == 0) { + /* if optc = 0 then ran out of short opts this arg */ + optc = THIS_ARG_DONE; + } else { + break; + } + } + +#if 0 + /* argument file code left out + so for now let filenames start with @ + */ + + } else if (allow_arg_files && arg[0] == '@') { + /* arg file */ + oERR(ZE_PARMS, no_arg_files_err); +#endif + + } else { + /* non-option */ + if (enable_permute) { + /* permute args to move all non-option args to end */ + if (first_nonoption_arg < 0) { + first_nonoption_arg = argn; + } + argn++; + } else { + /* no permute args so return non-option args when found */ + if ((*value = (char *)malloc(strlen(arg) + 1)) == NULL) { + oERR(ZE_MEM, "go"); + } + strcpy(*value, arg); + *option_num = o_NO_OPTION_MATCH; + optc = NON_OPTION_ARG; + option_ID = o_NON_OPTION_ARG; + break; + } + + } + } + + *pargs = args; + *argc = argcnt; + *first_nonopt_arg = first_nonoption_arg; + *argnum = argn; + *optchar = optc; + + return option_ID; +} diff --git a/third_party/infozip/zip/globals.c b/third_party/infozip/zip/globals.c new file mode 100644 index 000000000..5fb193a5c --- /dev/null +++ b/third_party/infozip/zip/globals.c @@ -0,0 +1,254 @@ +/* clang-format off */ +/* + globals.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * globals.c by Mark Adler + */ +#define __GLOBALS_C + +#define GLOBALS /* include definition of errors[] in zip.h */ +#ifndef UTIL +#define UTIL /* do not declare the read_buf variable */ +#endif + +#include "third_party/infozip/zip/zip.h" + + +/* Handy place to build error messages */ +char errbuf[FNMAX+4081]; + +/* Argument processing globals */ +int recurse = 0; /* 1=recurse into directories encountered */ +int dispose = 0; /* 1=remove files after put in zip file */ +int pathput = 1; /* 1=store path with name */ +#ifdef RISCOS +int scanimage = 1; /* 1=scan through image files */ +#endif +int method = BEST; /* one of BEST, DEFLATE (only), or STORE (only) */ +int dosify = 0; /* 1=make new entries look like MSDOS */ +int verbose = 0; /* 1=report oddities in zip file structure */ +int fix = 0; /* 1=fix the zip file, 2=FF, 3=ZipNote */ +int filesync = 0; /* 1=file sync, delete entries not on file system */ +int adjust = 0; /* 1=adjust offsets for sfx'd file (keep preamble) */ +int level = 6; /* 0=fastest compression, 9=best compression */ +int translate_eol = 0; /* Translate end-of-line LF -> CR LF */ +#ifdef VMS + int vmsver = 0; /* 1=append VMS version number to file names */ + int vms_native = 0; /* 1=store in VMS format */ + int vms_case_2 = 0; /* ODS2 file name case in VMS. -1: down. */ + int vms_case_5 = 0; /* ODS5 file name case in VMS. +1: preserve. */ +#endif /* VMS */ +#if defined(OS2) || defined(WIN32) + int use_longname_ea = 0; /* 1=use the .LONGNAME EA as the file's name */ +#endif +/* 9/26/04 */ +int no_wild = 0; /* 1 = wildcards are disabled */ +int allow_regex = 0; /* 1 = allow [list] matching */ +#ifdef WILD_STOP_AT_DIR + int wild_stop_at_dir = 1; /* default wildcards do not include / in matches */ +#else + int wild_stop_at_dir = 0; /* default wildcards do include / in matches */ +#endif + +#ifdef UNICODE_SUPPORT + int using_utf8 = 0; /* 1 if current character set UTF-8 */ +# ifdef WIN32 + int no_win32_wide = -1; /* 1 = no wide functions, like GetFileAttributesW() */ +# endif +#endif + +ulg skip_this_disk = 0; +int des_good = 0; /* Good data descriptor found */ +ulg des_crc = 0; /* Data descriptor CRC */ +uzoff_t des_csize = 0; /* Data descriptor csize */ +uzoff_t des_usize = 0; /* Data descriptor usize */ + +/* dots 10/20/04 */ +zoff_t dot_size = 0; /* bytes processed in deflate per dot, 0 = no dots */ +zoff_t dot_count = 0; /* buffers seen, recyles at dot_size */ +/* status 10/30/04 */ +int display_counts = 0; /* display running file count */ +int display_bytes = 0; /* display running bytes remaining */ +int display_globaldots = 0; /* display dots for archive instead of each file */ +int display_volume = 0; /* display current input and output volume (disk) numbers */ +int display_usize = 0; /* display uncompressed bytes */ +ulg files_so_far = 0; /* files processed so far */ +ulg bad_files_so_far = 0; /* bad files skipped so far */ +ulg files_total = 0; /* files total to process */ +uzoff_t bytes_so_far = 0; /* bytes processed so far (from initial scan) */ +uzoff_t good_bytes_so_far = 0;/* good bytes read so far */ +uzoff_t bad_bytes_so_far = 0; /* bad bytes skipped so far */ +uzoff_t bytes_total = 0; /* total bytes to process (from initial scan) */ + +/* logfile 6/5/05 */ +int logall = 0; /* 0 = warnings/errors, 1 = all */ +FILE *logfile = NULL; /* pointer to open logfile or NULL */ +int logfile_append = 0; /* append to existing logfile */ +char *logfile_path = NULL; /* pointer to path of logfile */ + +int hidden_files = 0; /* process hidden and system files */ +int volume_label = 0; /* add volume label */ +int dirnames = 1; /* include directory entries by default */ +int filter_match_case = 1; /* 1=match case when filter() */ +int diff_mode = 0; /* 1=require --out and only store changed and add */ +#if defined(WIN32) +int only_archive_set = 0; /* include only files with DOS archive bit set */ +int clear_archive_bits = 0; /* clear DOS archive bit of included files */ +#endif +int linkput = 0; /* 1=store symbolic links as such */ +int noisy = 1; /* 0=quiet operation */ +int extra_fields = 1; /* 0=create minimum, 1=don't copy old, 2=keep old */ +int use_descriptors = 0; /* 1=use data descriptors 12/29/04 */ +int zip_to_stdout = 0; /* output zipfile to stdout 12/30/04 */ +int allow_empty_archive = 0; /* if no files, create empty archive anyway 12/28/05 */ +int copy_only = 0; /* 1=copying archive entries only */ +int allow_fifo = 0; /* 1=allow reading Unix FIFOs, waiting if pipe open */ +int show_files = 0; /* show files to operate on and exit (=2 log only) */ + +int output_seekable = 1; /* 1 = output seekable 3/13/05 EG */ + +#ifdef ZIP64_SUPPORT /* zip64 support 10/4/03 */ + int force_zip64 = -1; /* if 1 force entries to be zip64, 0 force not zip64 */ + /* mainly for streaming from stdin */ + int zip64_entry = 0; /* current entry needs Zip64 */ + int zip64_archive = 0; /* if 1 then at least 1 entry needs zip64 */ +#endif + +#ifdef NTSD_EAS + int use_privileges = 0; /* 1=use security privilege overrides */ +#endif +#ifndef RISCOS +#ifndef QDOS +#ifndef TANDEM +char *special = ".Z:.zip:.zoo:.arc:.lzh:.arj"; /* List of special suffixes */ +#else /* TANDEM */ +char *special = " Z: zip: zoo: arc: lzh: arj"; /* List of special suffixes */ +#endif +#else /* QDOS */ +char *special = "_Z:_zip:_zoo:_arc:_lzh:_arj"; /* List of special suffixes */ +#endif +#else /* RISCOS */ +char *special = "DDC:D96:68E"; +#endif /* ?RISCOS */ +char *key = NULL; /* Scramble password if scrambling */ +char *tempath = NULL; /* Path for temporary files */ +FILE *mesg; /* stdout by default, stderr for piping */ + +#ifdef UNICODE_SUPPORT + int utf8_force = 0; /* 1=force storing UTF-8 as standard per AppNote bit 11 */ +#endif +int unicode_escape_all = 0; /* 1=escape all non-ASCII characters in paths */ +int unicode_mismatch = 1; /* unicode mismatch is 0=error, 1=warn, 2=ignore, 3=no */ + +time_t scan_delay = 5; /* seconds before display Scanning files message */ +time_t scan_dot_time = 2; /* time in seconds between Scanning files dots */ +time_t scan_start = 0; /* start of scan */ +time_t scan_last = 0; /* time of last message */ +int scan_started = 0; /* scan has started */ +uzoff_t scan_count = 0; /* Used for Scanning files ... message */ + +ulg before = 0; /* 0=ignore, else exclude files before this time */ +ulg after = 0; /* 0=ignore, else exclude files newer than this time */ + +/* Zip file globals */ +char *zipfile; /* New or existing zip archive (zip file) */ + +/* zip64 support 08/31/2003 R.Nausedat */ +/* all are across splits - subtract bytes_prev_splits to get offsets for current disk */ +uzoff_t zipbeg; /* Starting offset of zip structures */ +uzoff_t cenbeg; /* Starting offset of central dir */ +uzoff_t tempzn; /* Count of bytes written to output zip files */ + +/* 10/28/05 */ +char *tempzip = NULL; /* name of temp file */ +FILE *y = NULL; /* output file now global so can change in splits */ +FILE *in_file = NULL; /* current input file for splits */ +char *in_path = NULL; /* base name of input archive file */ +char *in_split_path = NULL; /* in split path */ +char *out_path = NULL; /* base name of output file, usually same as zipfile */ +int zip_attributes = 0; + +/* in split globals */ + +ulg total_disks = 0; /* total disks in archive */ +ulg current_in_disk = 0; /* current read split disk */ +uzoff_t current_in_offset = 0; /* current offset in current read disk */ +ulg skip_current_disk = 0; /* if != 0 and fix then skip entries on this disk */ + + +/* out split globals */ + +ulg current_local_disk = 0; /* disk with current local header */ + +ulg current_disk = 0; /* current disk number */ +ulg cd_start_disk = (ulg)-1; /* central directory start disk */ +uzoff_t cd_start_offset = 0; /* offset of start of cd on cd start disk */ +uzoff_t cd_entries_this_disk = 0; /* cd entries this disk */ +uzoff_t total_cd_entries = 0; /* total cd entries in new/updated archive */ +ulg zip64_eocd_disk = 0; /* disk with Zip64 End Of Central Directory Record */ +uzoff_t zip64_eocd_offset = 0; /* offset for Zip64 EOCD Record */ + +/* for split method 1 (keep split with local header open and update) */ +char *current_local_tempname = NULL; /* name of temp file */ +FILE *current_local_file = NULL; /* file pointer for current local header */ +uzoff_t current_local_offset = 0; /* offset to start of current local header */ + +/* global */ +uzoff_t bytes_this_split = 0; /* bytes written to the current split */ +int read_split_archive = 0; /* 1=scanzipf_reg detected spanning signature */ +int split_method = 0; /* 0=no splits, 1=seekable, 2=data desc, -1=no */ +uzoff_t split_size = 0; /* how big each split should be */ +int split_bell = 0; /* when pause for next split ring bell */ +uzoff_t bytes_prev_splits = 0; /* total bytes written to all splits before this */ +uzoff_t bytes_this_entry = 0; /* bytes written for this entry across all splits */ +int noisy_splits = 0; /* note when splits are being created */ +int mesg_line_started = 0; /* 1=started writing a line to mesg */ +int logfile_line_started = 0; /* 1=started writing a line to logfile */ + +#ifdef WIN32 + int nonlocal_name = 0; /* Name has non-local characters */ + int nonlocal_path = 0; /* Path has non-local characters */ +#endif +#ifdef UNICODE_SUPPORT + int use_wide_to_mb_default = 0; +#endif + +struct zlist far *zfiles = NULL; /* Pointer to list of files in zip file */ +/* The limit for number of files using the Zip64 format is 2^64 - 1 (8 bytes) + but extent is used for many internal sorts and other tasks and is generally + long on 32-bit systems. Because of that, but more because of various memory + utilization issues limiting the practical number of central directory entries + that can be sorted, the number of actual entries that can be stored probably + can't exceed roughly 2^30 on 32-bit systems so extent is probably sufficient. */ +extent zcount; /* Number of files in zip file */ +int zipfile_exists = 0; /* 1 if zipfile exists */ +ush zcomlen; /* Length of zip file comment */ +char *zcomment = NULL; /* Zip file comment (not zero-terminated) */ +struct zlist far **zsort; /* List of files sorted by name */ +#ifdef UNICODE_SUPPORT + struct zlist far **zusort; /* List of files sorted by zuname */ +#endif + +/* Files to operate on that are not in zip file */ +struct flist far *found = NULL; /* List of names found */ +struct flist far * far *fnxt = &found; + /* Where to put next name in found list */ +extent fcount; /* Count of files in list */ + +/* Patterns to be matched */ +struct plist *patterns = NULL; /* List of patterns to be matched */ +unsigned pcount = 0; /* number of patterns */ +unsigned icount = 0; /* number of include only patterns */ +unsigned Rcount = 0; /* number of -R include patterns */ + +#ifdef IZ_CHECK_TZ +int zp_tz_is_valid; /* signals "timezone info is available" */ +#endif diff --git a/third_party/infozip/zip/match.S b/third_party/infozip/zip/match.S new file mode 100644 index 000000000..eb8f7355a --- /dev/null +++ b/third_party/infozip/zip/match.S @@ -0,0 +1,407 @@ +/* + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2004-May-22 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, both of these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ + +/* + * match.s by Jean-loup Gailly. Translated to 32 bit code by Kai Uwe Rommel. + * The 68020 version has been written by Francesco Potorti` + * with adaptations by Carsten Steger , + * Andreas Schwab and + * Kristoffer Eriksson + */ + +/* This file is NOT used in conjunction with zlib. */ +#ifndef USE_ZLIB + +/* Preprocess with -DNO_UNDERLINE if your C compiler does not prefix + * external symbols with an underline character '_'. + */ +#if defined(NO_UNDERLINE) || defined(__ELF__) +# define _prev prev +# define _window window +# define _match_start match_start +# define _prev_length prev_length +# define _good_match good_match +# define _nice_match nice_match +# define _strstart strstart +# define _max_chain_length max_chain_length + +# define _match_init match_init +# define _longest_match longest_match +#endif + +#ifdef DYN_ALLOC + error: DYN_ALLOC not yet supported in match.s +#endif + +/* Use 16-bytes alignment if your assembler supports it. Warning: gas + * uses a log(x) parameter (.align 4 means 16-bytes alignment). On SVR4 + * the parameter is a number of bytes. + */ +#ifndef ALIGNMENT +# define ALIGNMENT 4 +#endif + + +#ifndef WSIZE +# define WSIZE 32768 +#endif +#define MIN_MATCH 3 +#define MAX_MATCH 258 +#define MIN_LOOKAHEAD (MAX_MATCH + MIN_MATCH + 1) +#define MAX_DIST (WSIZE - MIN_LOOKAHEAD) + +#if defined(i386) || defined(_I386) || defined(_i386) || defined(__i386) + +/* This version is for 386 Unix or OS/2 in 32 bit mode. + * Warning: it uses the AT&T syntax: mov source,dest + * This file is only optional. If you want to force the C version, + * add -DNO_ASM to CFLAGS in Makefile and set OBJA to an empty string. + * If you have reduced WSIZE in (g)zip.h, then make sure this is + * assembled with an equivalent -DWSIZE=. + * This version assumes static allocation of the arrays (-DDYN_ALLOC not used). + */ + + .file "match.S" + + .globl _match_init + .globl _longest_match + + .text + +_match_init: + ret + +/*----------------------------------------------------------------------- + * Set match_start to the longest match starting at the given string and + * return its length. Matches shorter or equal to prev_length are discarded, + * in which case the result is equal to prev_length and match_start is + * garbage. + * IN assertions: cur_match is the head of the hash chain for the current + * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 + */ + + .align ALIGNMENT + +_longest_match: /* int longest_match(cur_match) */ + +#define cur_match 20(%esp) + /* return address */ /* esp+16 */ + push %ebp /* esp+12 */ + push %edi /* esp+8 */ + push %esi /* esp+4 */ + push %ebx /* esp */ + +/* + * match equ esi + * scan equ edi + * chain_length equ ebp + * best_len equ ebx + * limit equ edx + */ + mov cur_match,%esi + mov _strstart,%edx + mov _max_chain_length,%ebp /* chain_length = max_chain_length */ + mov %edx,%edi + sub $(MAX_DIST),%edx /* limit = strstart-MAX_DIST */ + cld /* string ops increment si and di */ + jae limit_ok + sub %edx,%edx /* limit = NIL */ +limit_ok: + add $2+_window,%edi /* edi = offset(window+strstart+2) */ + mov _prev_length,%ebx /* best_len = prev_length */ + movw -2(%edi),%cx /* cx = scan[0..1] */ + movw -3(%ebx,%edi),%ax /* ax = scan[best_len-1..best_len] */ + cmp _good_match,%ebx /* do we have a good match already? */ + jb do_scan + shr $2,%ebp /* chain_length >>= 2 */ + jmp do_scan + + .align ALIGNMENT +long_loop: +/* at this point, edi == scan+2, esi == cur_match */ + movw -3(%ebx,%edi),%ax /* ax = scan[best_len-1..best_len] */ + movw -2(%edi),%cx /* cx = scan[0..1] */ +short_loop: +/* + * at this point, di == scan+2, si == cur_match, + * ax = scan[best_len-1..best_len] and cx = scan[0..1] + */ + and $(WSIZE-1), %esi + dec %ebp /* --chain_length */ + movw _prev(,%esi,2),%si /* cur_match = prev[cur_match] */ + /* top word of esi is still 0 */ + jz the_end + cmp %edx,%esi /* cur_match <= limit ? */ + jbe the_end +do_scan: + cmpw _window-1(%ebx,%esi),%ax/* check match at best_len-1 */ + jne short_loop + cmpw _window(%esi),%cx /* check min_match_length match */ + jne short_loop + + add $2+_window,%esi /* si = match */ + mov $((MAX_MATCH>>1)-1),%ecx/* scan for at most MAX_MATCH bytes */ + mov %edi,%eax /* ax = scan+2 */ + repe; cmpsw /* loop until mismatch */ + je maxmatch /* match of length MAX_MATCH? */ +mismatch: + movb -2(%edi),%cl /* mismatch on first or second byte? */ + xchg %edi,%eax /* edi = scan+2, eax = end of scan */ + subb -2(%esi),%cl /* cl = 0 if first bytes equal */ + sub %edi,%eax /* eax = len */ + sub $2+_window,%esi /* esi = cur_match + len */ + sub %eax,%esi /* esi = cur_match */ + subb $1,%cl /* set carry if cl == 0 (cannot use DEC) */ + adc $0,%eax /* eax = carry ? len+1 : len */ + cmp %ebx,%eax /* len > best_len ? */ + jle long_loop + mov %esi,_match_start /* match_start = cur_match */ + mov %eax,%ebx /* ebx = best_len = len */ +#ifdef FULL_SEARCH + cmp $(MAX_MATCH),%eax /* len >= MAX_MATCH ? */ +#else + cmp _nice_match,%eax /* len >= nice_match ? */ +#endif + jl long_loop +the_end: + mov %ebx,%eax /* result = eax = best_len */ + pop %ebx + pop %esi + pop %edi + pop %ebp + ret + .align ALIGNMENT +maxmatch: + cmpsb + jmp mismatch + +#else /* !(i386 || _I386 || _i386 || __i386) */ + +/* ======================== 680x0 version ================================= */ + +#if defined(m68k)||defined(mc68k)||defined(__mc68000__)||defined(__MC68000__) +# ifndef mc68000 +# define mc68000 +# endif +#endif + +#if defined(__mc68020__) || defined(__MC68020__) || defined(sysV68) +# ifndef mc68020 +# define mc68020 +# endif +#endif + +#if defined(mc68020) || defined(mc68000) + +#if (defined(mc68020) || defined(NeXT)) && !defined(UNALIGNED_OK) +# define UNALIGNED_OK +#endif + +#ifdef sysV68 /* Try Motorola Delta style */ + +# define GLOBAL(symbol) global symbol +# define TEXT text +# define FILE(filename) file filename +# define invert_maybe(src,dst) dst,src +# define imm(data) &data +# define reg(register) %register + +# define addl add.l +# define addql addq.l +# define blos blo.b +# define bhis bhi.b +# define bras bra.b +# define clrl clr.l +# define cmpmb cmpm.b +# define cmpw cmp.w +# define cmpl cmp.l +# define lslw lsl.w +# define lsrl lsr.l +# define movel move.l +# define movew move.w +# define moveb move.b +# define moveml movem.l +# define subl sub.l +# define subw sub.w +# define subql subq.l + +# define IndBase(bd,An) (bd,An) +# define IndBaseNdxl(bd,An,Xn) (bd,An,Xn.l) +# define IndBaseNdxw(bd,An,Xn) (bd,An,Xn.w) +# define predec(An) -(An) +# define postinc(An) (An)+ + +#else /* default style (Sun 3, NeXT, Amiga, Atari) */ + +# define GLOBAL(symbol) .globl symbol +# define TEXT .text +# define FILE(filename) .even +# define invert_maybe(src,dst) src,dst +# if defined(sun) || defined(mc68k) +# define imm(data) #data +# else +# define imm(data) \#data +# endif +# define reg(register) register + +# define blos bcss +# if defined(sun) || defined(mc68k) +# define movel movl +# define movew movw +# define moveb movb +# endif +# define IndBase(bd,An) An@(bd) +# define IndBaseNdxl(bd,An,Xn) An@(bd,Xn:l) +# define IndBaseNdxw(bd,An,Xn) An@(bd,Xn:w) +# define predec(An) An@- +# define postinc(An) An@+ + +#endif /* styles */ + +#define Best_Len reg(d0) /* unsigned */ +#define Cur_Match reg(d1) /* Ipos */ +#define Loop_Counter reg(d2) /* int */ +#define Scan_Start reg(d3) /* unsigned short */ +#define Scan_End reg(d4) /* unsigned short */ +#define Limit reg(d5) /* IPos */ +#define Chain_Length reg(d6) /* unsigned */ +#define Scan_Test reg(d7) +#define Scan reg(a0) /* *uch */ +#define Match reg(a1) /* *uch */ +#define Prev_Address reg(a2) /* *Pos */ +#define Scan_Ini reg(a3) /* *uch */ +#define Match_Ini reg(a4) /* *uch */ +#define Stack_Pointer reg(sp) + + GLOBAL (_match_init) + GLOBAL (_longest_match) + + TEXT + + FILE ("match.S") + +_match_init: + rts + +/*----------------------------------------------------------------------- + * Set match_start to the longest match starting at the given string and + * return its length. Matches shorter or equal to prev_length are discarded, + * in which case the result is equal to prev_length and match_start is + * garbage. + * IN assertions: cur_match is the head of the hash chain for the current + * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 + */ + +/* int longest_match (cur_match) */ + +#ifdef UNALIGNED_OK +# define pushreg 15928 /* d2-d6/a2-a4 */ +# define popreg 7292 +#else +# define pushreg 16184 /* d2-d7/a2-a4 */ +# define popreg 7420 +#endif + +_longest_match: + movel IndBase(4,Stack_Pointer),Cur_Match + moveml imm(pushreg),predec(Stack_Pointer) + movel _max_chain_length,Chain_Length + movel _prev_length,Best_Len + movel imm(_prev),Prev_Address + movel imm(_window+MIN_MATCH),Match_Ini + movel _strstart,Limit + movel Match_Ini,Scan_Ini + addl Limit,Scan_Ini + subw imm(MAX_DIST),Limit + bhis L__limit_ok + clrl Limit +L__limit_ok: + cmpl invert_maybe(_good_match,Best_Len) + blos L__length_ok + lsrl imm(2),Chain_Length +L__length_ok: + subql imm(1),Chain_Length +#ifdef UNALIGNED_OK + movew IndBase(-MIN_MATCH,Scan_Ini),Scan_Start + movew IndBaseNdxw(-MIN_MATCH-1,Scan_Ini,Best_Len),Scan_End +#else + moveb IndBase(-MIN_MATCH,Scan_Ini),Scan_Start + lslw imm(8),Scan_Start + moveb IndBase(-MIN_MATCH+1,Scan_Ini),Scan_Start + moveb IndBaseNdxw(-MIN_MATCH-1,Scan_Ini,Best_Len),Scan_End + lslw imm(8),Scan_End + moveb IndBaseNdxw(-MIN_MATCH,Scan_Ini,Best_Len),Scan_End +#endif + bras L__do_scan + +L__long_loop: +#ifdef UNALIGNED_OK + movew IndBaseNdxw(-MIN_MATCH-1,Scan_Ini,Best_Len),Scan_End +#else + moveb IndBaseNdxw(-MIN_MATCH-1,Scan_Ini,Best_Len),Scan_End + lslw imm(8),Scan_End + moveb IndBaseNdxw(-MIN_MATCH,Scan_Ini,Best_Len),Scan_End +#endif + +L__short_loop: + lslw imm(1),Cur_Match + movew IndBaseNdxl(0,Prev_Address,Cur_Match),Cur_Match + cmpw invert_maybe(Limit,Cur_Match) + dbls Chain_Length,L__do_scan + bras L__return + +L__do_scan: + movel Match_Ini,Match + addl Cur_Match,Match +#ifdef UNALIGNED_OK + cmpw invert_maybe(IndBaseNdxw(-MIN_MATCH-1,Match,Best_Len),Scan_End) + bne L__short_loop + cmpw invert_maybe(IndBase(-MIN_MATCH,Match),Scan_Start) + bne L__short_loop +#else + moveb IndBaseNdxw(-MIN_MATCH-1,Match,Best_Len),Scan_Test + lslw imm(8),Scan_Test + moveb IndBaseNdxw(-MIN_MATCH,Match,Best_Len),Scan_Test + cmpw invert_maybe(Scan_Test,Scan_End) + bne L__short_loop + moveb IndBase(-MIN_MATCH,Match),Scan_Test + lslw imm(8),Scan_Test + moveb IndBase(-MIN_MATCH+1,Match),Scan_Test + cmpw invert_maybe(Scan_Test,Scan_Start) + bne L__short_loop +#endif + + movew imm((MAX_MATCH-MIN_MATCH+1)-1),Loop_Counter + movel Scan_Ini,Scan +L__scan_loop: + cmpmb postinc(Match),postinc(Scan) + dbne Loop_Counter,L__scan_loop + + subl Scan_Ini,Scan + addql imm(MIN_MATCH-1),Scan + cmpl invert_maybe(Best_Len,Scan) + bls L__short_loop + movel Scan,Best_Len + movel Cur_Match,_match_start +#ifdef FULL_SEARCH + cmpl invert_maybe(imm(MAX_MATCH),Best_Len) +#else + cmpl invert_maybe(_nice_match,Best_Len) +#endif + blos L__long_loop +L__return: + moveml postinc(Stack_Pointer),imm(popreg) + rts + +#else + error: this asm version is for 386 or 680x0 only +#endif /* mc68000 || mc68020 */ +#endif /* i386 || _I386 || _i386 || __i386 */ + +#endif /* !USE_ZLIB */ diff --git a/third_party/infozip/zip/revision.h b/third_party/infozip/zip/revision.h new file mode 100644 index 000000000..0a9d64ab5 --- /dev/null +++ b/third_party/infozip/zip/revision.h @@ -0,0 +1,140 @@ +/* clang-format off */ +/* + revision.h - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * revision.h by Mark Adler. + */ + +#ifndef __revision_h +#define __revision_h 1 + +/* For api version checking */ +#define Z_MAJORVER 3 +#define Z_MINORVER 0 +#define Z_PATCHLEVEL 0 +#define Z_BETALEVEL "i BETA" + +#define VERSION "3.0" +#define REVDATE "July 5th 2008" + +#define DW_MAJORVER Z_MAJORVER +#define DW_MINORVER Z_MINORVER +#define DW_PATCHLEVEL Z_PATCHLEVEL + +#ifndef IZ_COMPANY_NAME /* might be already defined... */ +# define IZ_COMPANY_NAME "Info-ZIP" +#endif + +#if !defined(WINDLL) && !defined(IZ_VERSION_SYMBOLS_ONLY) +/* Copyright notice for binary executables--this notice only applies to + * those (zip, zipcloak, zipsplit, and zipnote), not to this file + * (revision.h). + */ + +#ifndef DEFCPYRT +/* copyright[] et.al. get defined only once ! */ +/* keep array sizes in sync with number of text */ +/* lines in the array definitions below !! */ +extern ZCONST char *copyright[1]; +extern ZCONST char * far swlicense[50]; +extern ZCONST char * far versinfolines[7]; +extern ZCONST char * far cryptnote[7]; + +#else /* DEFCPYRT */ + +ZCONST char *copyright[] = { +"Copyright (c) 1990-2008 Info-ZIP - Type '%s \"-L\"' for software license." +/* XXX still necessary ???? */ +#ifdef AZTEC_C +, /* extremely lame compiler bug workaround */ +#endif +}; + +ZCONST char * far versinfolines[] = { +"This is %s %s (%s), by Info-ZIP.", +"Currently maintained by E. Gordon. Please send bug reports to", +"the authors using the web page at www.info-zip.org; see README for details.", +"", +"Latest sources and executables are at ftp://ftp.info-zip.org/pub/infozip,", +"as of above date; see http://www.info-zip.org/ for other sites.", +"" +}; + +/* new notice - 4 March 2007 */ +ZCONST char * far cryptnote[] = { +"Encryption notice:", +"\tThe encryption code of this program is not copyrighted and is", +"\tput in the public domain. It was originally written in Europe", +"\tand, to the best of our knowledge, can be freely distributed", +"\tin both source and object forms from any country, including", +"\tthe USA under License Exception TSU of the U.S. Export", +"\tAdministration Regulations (section 740.13(e)) of 6 June 2002." +}; + +ZCONST char * far swlicense[] = { +"Copyright (c) 1990-2008 Info-ZIP. All rights reserved.", +"", +"For the purposes of this copyright and license, \"Info-ZIP\" is defined as", +"the following set of individuals:", +"", +" Mark Adler, John Bush, Karl Davis, Harald Denker, Jean-Michel Dubois,", +" Jean-loup Gailly, Hunter Goatley, Ed Gordon, Ian Gorman, Chris Herborth,", +" Dirk Haase, Greg Hartwig, Robert Heath, Jonathan Hudson, Paul Kienitz,", +" David Kirschbaum, Johnny Lee, Onno van der Linden, Igor Mandrichenko,", +" Steve P. Miller, Sergio Monesi, Keith Owens, George Petrov, Greg Roelofs,", +" Kai Uwe Rommel, Steve Salisbury, Dave Smith, Steven M. Schweda,", +" Christian Spieler, Cosmin Truta, Antoine Verheijen, Paul von Behren,", +" Rich Wales, Mike White", +"", +"This software is provided \"as is,\" without warranty of any kind, express", +"or implied. In no event shall Info-ZIP or its contributors be held liable", +"for any direct, indirect, incidental, special or consequential damages", +"arising out of the use of or inability to use this software.", +"", +"Permission is granted to anyone to use this software for any purpose,", +"including commercial applications, and to alter it and redistribute it", +"freely, subject to the above disclaimer and the following restrictions:", +"", +" 1. Redistributions of source code (in whole or in part) must retain", +" the above copyright notice, definition, disclaimer, and this list", +" of conditions.", +"", +" 2. Redistributions in binary form (compiled executables and libraries)", +" must reproduce the above copyright notice, definition, disclaimer,", +" and this list of conditions in documentation and/or other materials", +" provided with the distribution. The sole exception to this condition", +" is redistribution of a standard UnZipSFX binary (including SFXWiz) as", +" part of a self-extracting archive; that is permitted without inclusion", +" of this license, as long as the normal SFX banner has not been removed", +" from the binary or disabled.", +"", +" 3. Altered versions--including, but not limited to, ports to new operating", +" systems, existing ports with new graphical interfaces, versions with", +" modified or added functionality, and dynamic, shared, or static library", +" versions not from Info-ZIP--must be plainly marked as such and must not", +" be misrepresented as being the original source or, if binaries,", +" compiled from the original source. Such altered versions also must not", +" be misrepresented as being Info-ZIP releases--including, but not", +" limited to, labeling of the altered versions with the names \"Info-ZIP\"", +" (or any variation thereof, including, but not limited to, different", +" capitalizations), \"Pocket UnZip,\" \"WiZ\" or \"MacZip\" without the", +" explicit permission of Info-ZIP. Such altered versions are further", +" prohibited from misrepresentative use of the Zip-Bugs or Info-ZIP", +" e-mail addresses or the Info-ZIP URL(s), such as to imply Info-ZIP", +" will provide support for the altered versions.", +"", +" 4. Info-ZIP retains the right to use the names \"Info-ZIP,\" \"Zip,\" \"UnZip,\"", +" \"UnZipSFX,\" \"WiZ,\" \"Pocket UnZip,\" \"Pocket Zip,\" and \"MacZip\" for its", +" own source and binary releases." +}; +#endif /* DEFCPYRT */ +#endif /* !WINDLL && !IZ_VERSION_SYMBOLS_ONLY */ +#endif /* !__revision_h */ diff --git a/third_party/infozip/zip/tailor.h b/third_party/infozip/zip/tailor.h new file mode 100644 index 000000000..8fb2f5174 --- /dev/null +++ b/third_party/infozip/zip/tailor.h @@ -0,0 +1,892 @@ +/* clang-format off */ +/* + tailor.h - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ + +/* Some compiler distributions for Win32/i386 systems try to emulate + * a Unix (POSIX-compatible) environment. + */ +#if (defined(WIN32) && defined(UNIX)) + /* Zip does not support merging both ports in a single executable. */ +# if (defined(FORCE_WIN32_OVER_UNIX) && defined(FORCE_UNIX_OVER_WIN32)) + /* conflicting choice requests -> we prefer the Win32 environment */ +# undef FORCE_UNIX_OVER_WIN32 +# endif +# ifdef FORCE_WIN32_OVER_UNIX + /* native Win32 support was explicitely requested... */ +# undef UNIX +# else + /* use the POSIX (Unix) emulation features by default... */ +# undef WIN32 +# endif +#endif + + +/* UNICODE */ +#ifdef NO_UNICODE_SUPPORT +# ifdef UNICODE_SUPPORT +# undef UNICODE_SUPPORT +# endif +#endif + + +#ifdef AMIGA +#include "amiga/osdep.h" +#endif + +#ifdef AOSVS +#include "aosvs/osdep.h" +#endif + +#ifdef ATARI +#include "atari/osdep.h" +#endif + +#ifdef __ATHEOS__ +#include "atheos/osdep.h" +#endif + +#ifdef __BEOS__ +#include "beos/osdep.h" +#endif + +#ifdef DOS +#include "msdos/osdep.h" +#endif + +#ifdef __human68k__ +#include "human68k/osdep.h" +#endif + +#if ((defined(__MWERKS__) && defined(macintosh)) || defined(MACOS)) +#include "macos/osdep.h" +#endif + +#ifdef NLM +#include "novell/osdep.h" +#endif + +#ifdef OS2 +#include "os2/osdep.h" +#endif + +#ifdef __riscos +#include "acorn/osdep.h" +#endif + +#ifdef QDOS +#include "qdos/osdep.h" +#endif + +#ifdef __TANDEM +#include "tandem.h" +#include "tanzip.h" +#endif + +#ifdef UNIX +#include "third_party/infozip/zip/unix/osdep.h" +#endif + +#if defined(__COMPILER_KCC__) || defined(TOPS20) +#include "tops20/osdep.h" +#endif + +#if defined(VMS) || defined(__VMS) +#include "vms/osdep.h" +#endif + +#if defined(__VM__) || defined(VM_CMS) || defined(MVS) +#include "cmsmvs.h" +#endif + +#ifdef WIN32 +#include "win32/osdep.h" +#endif + +#ifdef THEOS +#include "theos/osdep.h" +#endif + + +/* generic LARGE_FILE_SUPPORT defines + These get used if not defined above. + 7/21/2004 EG +*/ +/* If a port hasn't defined ZOFF_T_FORMAT_SIZE_PREFIX + then probably need to define all of these. */ +#ifndef ZOFF_T_FORMAT_SIZE_PREFIX + +# ifdef LARGE_FILE_SUPPORT + /* Probably passed in from command line instead of in above + includes if get here. Assume large file support and hope. 8/14/04 EG */ + + /* Set the Large File Summit (LFS) defines to turn on large file support + in case it helps. */ + +# define _LARGEFILE_SOURCE /* some OSes need this for fseeko */ +# define _LARGEFILE64_SOURCE +# define _FILE_OFFSET_BITS 64 /* select default interface as 64 bit */ +# define _LARGE_FILES /* some OSes need this for 64-bit off_t */ + + typedef off_t zoff_t; + typedef unsigned long long uzoff_t; /* unsigned zoff_t (12/29/04 EG) */ + + /* go with common prefix */ +# define ZOFF_T_FORMAT_SIZE_PREFIX "ll" + +# else + /* Default type for offsets and file sizes was ulg but reports + of using ulg to create files from 2 GB to 4 GB suggest + it doesn't work well. Now just switch to Zip64 or not + support over 2 GB. 7/24/04 EG */ + /* Now use uzoff_t for unsigned things. 12/29/04 EG */ + typedef long zoff_t; + typedef unsigned long uzoff_t; + +# define ZOFF_T_FORMAT_SIZE_PREFIX "l" + +# endif + + typedef struct stat z_stat; + + /* flag that we are defaulting */ +# define USING_DEFAULT_LARGE_FILE_SUPPORT +#endif + + +#if (defined(USE_ZLIB) && defined(ASM_CRC)) +# undef ASM_CRC +#endif + +#if (defined(USE_ZLIB) && defined(ASMV)) +# undef ASMV +#endif + +/* When "void" is an alias for "int", prototypes cannot be used. */ +#if (defined(NO_VOID) && !defined(NO_PROTO)) +# define NO_PROTO +#endif + +/* Used to remove arguments in function prototypes for non-ANSI C */ +#ifndef NO_PROTO +# define OF(a) a +# define OFT(a) a +#else /* NO_PROTO */ +# define OF(a) () +# define OFT(a) +#endif /* ?NO_PROTO */ + +/* If the compiler can't handle const define ZCONST in osdep.h */ +/* Define const itself in case the system include files are bonkers */ +#ifndef ZCONST +# ifdef NO_CONST +# define ZCONST +# define const +# else +# define ZCONST const +# endif +#endif + +/* + * Some compiler environments may require additional attributes attached + * to declarations of runtime libary functions (e.g. to prepare for + * linking against a "shared dll" version of the RTL). Here, we provide + * the "empty" default for these attributes. + */ +#ifndef IZ_IMP +# define IZ_IMP +#endif + +/* + * case mapping functions. case_map is used to ignore case in comparisons, + * to_up is used to force upper case even on Unix (for dosify option). + */ +#ifdef USE_CASE_MAP +# define case_map(c) upper[(c) & 0xff] +# define to_up(c) upper[(c) & 0xff] +#else +# define case_map(c) (c) +# define to_up(c) ((c) >= 'a' && (c) <= 'z' ? (c)-'a'+'A' : (c)) +#endif /* USE_CASE_MAP */ + +/* Define void, zvoid, and extent (size_t) */ +#include "libc/stdio/stdio.h" + +#ifndef NO_STDDEF_H +// # include +#endif /* !NO_STDDEF_H */ + +#ifndef NO_STDLIB_H +# include "libc/mem/mem.h" +#endif /* !NO_STDLIB_H */ + +#ifndef NO_UNISTD_H +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/sysv/consts/fileno.h" +#include "libc/sysv/consts/o.h" +#endif /* !NO_UNISTD_H */ + +#ifndef NO_FCNTL_H +#include "libc/calls/calls.h" +#include "libc/sysv/consts/at.h" +#include "libc/sysv/consts/f.h" +#endif /* !NO_FNCTL_H */ + +#ifndef NO_STRING_H +# include "libc/str/str.h" +#else +# include "libc/str/str.h" +#endif /* NO_STRING_H */ + +#ifdef NO_VOID +# define void int + typedef char zvoid; +#else /* !NO_VOID */ +# ifdef NO_TYPEDEF_VOID +# define zvoid void +# else + typedef void zvoid; +# endif +#endif /* ?NO_VOID */ + +#ifdef NO_STRRCHR +# define strrchr rindex +#endif + +#ifdef NO_STRCHR +# define strchr index +#endif + +/* + * A couple of forward declarations that are needed on systems that do + * not supply C runtime library prototypes. + */ +#ifdef NO_PROTO +IZ_IMP char *strcpy(); +IZ_IMP char *strcat(); +IZ_IMP char *strrchr(); +/* XXX use !defined(ZMEM) && !defined(__hpux__) ? */ +#if !defined(ZMEM) && defined(NO_STRING_H) +IZ_IMP char *memset(); +IZ_IMP char *memcpy(); +#endif /* !ZMEM && NO_STRING_H */ + +/* XXX use !defined(__hpux__) ? */ +#ifdef NO_STDLIB_H +IZ_IMP char *calloc(); +IZ_IMP char *malloc(); +IZ_IMP char *getenv(); +IZ_IMP long atol(); +#endif /* NO_STDLIB_H */ + +#ifndef NO_MKTEMP +IZ_IMP char *mktemp(); +#endif /* !NO_MKTEMP */ + +#endif /* NO_PROTO */ + +/* + * SEEK_* macros, should be defined in stdio.h + */ +/* Define fseek() commands */ +#ifndef SEEK_SET +# define SEEK_SET 0 +#endif /* !SEEK_SET */ + +#ifndef SEEK_CUR +# define SEEK_CUR 1 +#endif /* !SEEK_CUR */ + +#ifndef FALSE +# define FALSE 0 +#endif + +#ifndef TRUE +# define TRUE 1 +#endif + +#ifdef NO_SIZE_T + typedef unsigned int extent; + /* define size_t 3/17/05 EG */ + typedef unsigned int size_t; +#else + typedef size_t extent; +#endif + +#ifdef NO_TIME_T + typedef long time_t; +#endif + +/* DBCS support for Info-ZIP's zip (mainly for japanese (-: ) + * by Yoshioka Tsuneo (QWF00133@nifty.ne.jp,tsuneo-y@is.aist-nara.ac.jp) + * This code is public domain! Date: 1998/12/20 + */ + +/* 2007-07-29 SMS. + * Include here if it will be needed later for Unicode. + * Otherwise, SETLOCALE may be defined here, and then defined again + * (differently) when is read later. + */ +#ifdef UNICODE_SUPPORT +# if defined( UNIX) || defined( VMS) + # include "libc/unicode/locale.h" +# endif /* defined( UNIX) || defined( VMS) */ +# include "libc/str/str.h" +#endif /* def UNICODE_SUPPORT */ + +#ifdef _MBCS +# include + + /* Multi Byte Character Set */ + extern char *___tmp_ptr; + unsigned char *zmbschr OF((ZCONST unsigned char *, unsigned int)); + unsigned char *zmbsrchr OF((ZCONST unsigned char *, unsigned int)); +# define CLEN(ptr) mblen((ZCONST char *)ptr, MB_CUR_MAX) +# define PREINCSTR(ptr) (ptr += CLEN(ptr)) +# define POSTINCSTR(ptr) (___tmp_ptr=(char *)ptr,ptr += CLEN(ptr),___tmp_ptr) + int lastchar OF((ZCONST char *ptr)); +# define MBSCHR(str,c) (char *)zmbschr((ZCONST unsigned char *)(str), c) +# define MBSRCHR(str,c) (char *)zmbsrchr((ZCONST unsigned char *)(str), (c)) +# ifndef SETLOCALE +# define SETLOCALE(category, locale) setlocale(category, locale) +# endif /* ndef SETLOCALE */ +#else /* !_MBCS */ +# define CLEN(ptr) 1 +# define PREINCSTR(ptr) (++(ptr)) +# define POSTINCSTR(ptr) ((ptr)++) +# define lastchar(ptr) ((*(ptr)=='\0') ? '\0' : ptr[strlen(ptr)-1]) +# define MBSCHR(str, c) strchr(str, c) +# define MBSRCHR(str, c) strrchr(str, c) +# ifndef SETLOCALE +# define SETLOCALE(category, locale) +# endif /* ndef SETLOCALE */ +#endif /* ?_MBCS */ +#define INCSTR(ptr) PREINCSTR(ptr) + + +/* System independent replacement for "struct utimbuf", which is missing + * in many older OS environments. + */ +typedef struct ztimbuf { + time_t actime; /* new access time */ + time_t modtime; /* new modification time */ +} ztimbuf; + +/* This macro round a time_t value to the OS specific resolution */ +#ifndef ROUNDED_TIME +# define ROUNDED_TIME(time) (time) +#endif + +/* Some systems define S_IFLNK but do not support symbolic links */ +#if defined (S_IFLNK) && defined(NO_SYMLINKS) +# undef S_IFLNK +#endif + +#ifndef Z_UINT4_DEFINED +# if !defined(NO_LIMITS_H) +# if (defined(UINT_MAX) && (UINT_MAX == 0xffffffffUL)) + typedef unsigned int z_uint4; +# define Z_UINT4_DEFINED +# else +# if (defined(ULONG_MAX) && (ULONG_MAX == 0xffffffffUL)) + typedef unsigned long z_uint4; +# define Z_UINT4_DEFINED +# else +# if (defined(USHRT_MAX) && (USHRT_MAX == 0xffffffffUL)) + typedef unsigned short z_uint4; +# define Z_UINT4_DEFINED +# endif +# endif +# endif +# endif /* !defined(NO_LIMITS_H) */ +#endif /* ndef Z_UINT4_DEFINED */ +#ifndef Z_UINT4_DEFINED + typedef ulg z_uint4; +# define Z_UINT4_DEFINED +#endif + +#ifndef FOPR /* fallback default definitions for FOPR, FOPM, FOPW: */ +# define FOPR "r" +# define FOPM "r+" +# define FOPW "w" +#endif /* fallback definition */ + +#ifndef FOPW_TMP /* fallback default for opening writable temp files */ +# define FOPW_TMP FOPW +#endif + +/* Open the old zip file in exclusive mode if possible (to avoid adding + * zip file to itself). + */ +#ifdef OS2 +# define FOPR_EX FOPM +#else +# define FOPR_EX FOPR +#endif + + +/* MSDOS file or directory attributes */ +#define MSDOS_HIDDEN_ATTR 0x02 +#define MSDOS_DIR_ATTR 0x10 + + +/* Define this symbol if your target allows access to unaligned data. + * This is not mandatory, just a speed optimization. The compressed + * output is strictly identical. + */ +#if (defined(MSDOS) && !defined(WIN32)) || defined(i386) +# define UNALIGNED_OK +#endif +#if defined(mc68020) || defined(vax) +# define UNALIGNED_OK +#endif + +#if (defined(SMALL_MEM) && !defined(CBSZ)) +# define CBSZ 2048 /* buffer size for copying files */ +# define ZBSZ 2048 /* buffer size for temporary zip file */ +#endif + +#if (defined(MEDIUM_MEM) && !defined(CBSZ)) +# define CBSZ 8192 +# define ZBSZ 8192 +#endif + +#ifndef CBSZ +# define CBSZ 16384 +# define ZBSZ 16384 +#endif + +#ifndef SBSZ +# define SBSZ CBSZ /* copy buf size for STORED entries, see zipup() */ +#endif + +#ifndef MEMORY16 +# ifdef __WATCOMC__ +# undef huge +# undef far +# undef near +# endif +# ifdef THEOS +# undef far +# undef near +# endif +# if (!defined(__IBMC__) || !defined(OS2)) +# ifndef huge +# define huge +# endif +# ifndef far +# define far +# endif +# ifndef near +# define near +# endif +# endif +# define nearmalloc malloc +# define nearfree free +# define farmalloc malloc +# define farfree free +#endif /* !MEMORY16 */ + +#ifndef Far +# define Far far +#endif + +/* MMAP and BIG_MEM cannot be used together -> let MMAP take precedence */ +#if (defined(MMAP) && defined(BIG_MEM)) +# undef BIG_MEM +#endif + +#if (defined(BIG_MEM) || defined(MMAP)) && !defined(DYN_ALLOC) +# define DYN_ALLOC +#endif + + +/* LARGE_FILE_SUPPORT + * + * Types are in osdep.h for each port + * + * LARGE_FILE_SUPPORT and ZIP64_SUPPORT are automatically + * set in osdep.h (for some ports) based on the port and compiler. + * + * Function prototypes are below as OF is defined earlier in this file + * but after osdep.h is included. In the future ANSI prototype + * support may be required and the OF define may then go away allowing + * the function defines to be in the port osdep.h. + * + * E. Gordon 9/21/2003 + * Updated 7/24/04 EG + */ +#ifdef LARGE_FILE_SUPPORT + /* 64-bit Large File Support */ + + /* Arguments for all functions are assumed to match the actual + arguments of the various port calls. As such only the + function names are mapped below. */ + +/* ---------------------------- */ +# ifdef UNIX + + /* Assume 64-bit file environment is defined. The below should all + be set to their 64-bit versions automatically. Neat. 7/20/2004 EG */ + + /* 64-bit stat functions */ +# define zstat stat +# define zfstat fstat +# define zlstat lstat + +# if defined(__alpha) && defined(__osf__) /* support for osf4.0f */ + /* 64-bit fseek */ +# define zfseeko fseek + + /* 64-bit ftell */ +# define zftello ftell + +# else + /* 64-bit fseeko */ +# define zfseeko fseeko + + /* 64-bit ftello */ +# define zftello ftello +# endif /* __alpha && __osf__ */ + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif /* UNIX */ + +/* ---------------------------- */ +# ifdef VMS + + /* 64-bit stat functions */ +# define zstat stat +# define zfstat fstat +# define zlstat lstat + + /* 64-bit fseeko */ +# define zfseeko fseeko + + /* 64-bit ftello */ +# define zftello ftello + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif /* def VMS */ + +/* ---------------------------- */ +# ifdef WIN32 + +# if defined(__MINGW32__) + /* GNU C, linked against "msvcrt.dll" */ + + /* 64-bit stat functions */ +# define zstat _stati64 +# ifdef UNICODE_SUPPORT +# define zwfstat _fstati64 +# define zwstat _wstati64 +# define zw_stat struct _stati64 +# endif +# define zfstat _fstati64 +# define zlstat lstat + + /* 64-bit fseeko */ + /* function in win32.c */ + int zfseeko OF((FILE *, zoff_t, int)); + + /* 64-bit ftello */ + /* function in win32.c */ + zoff_t zftello OF((FILE *)); + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif + +# if defined(__CYGWIN__) + /* GNU C, CygWin with its own POSIX compatible runtime library */ + + /* 64-bit stat functions */ +# define zstat stat +# define zfstat fstat +# define zlstat lstat + + /* 64-bit fseeko */ +# define zfseeko fseeko + + /* 64-bit ftello */ +# define zftello ftello + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif + +# ifdef __WATCOMC__ + /* WATCOM C */ + + /* 64-bit stat functions */ +# define zstat _stati64 +# ifdef UNICODE_SUPPORT +# define zwfstat _fstati64 +# define zwstat _wstati64 +# define zw_stat struct _stati64 +# endif +# define zfstat _fstati64 +# define zlstat lstat + + /* 64-bit fseeko */ + /* function in win32.c */ + int zfseeko OF((FILE *, zoff_t, int)); + + /* 64-bit ftello */ + /* function in win32.c */ + zoff_t zftello OF((FILE *)); + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif + +# ifdef _MSC_VER + /* MS C and VC */ + + /* 64-bit stat functions */ +# define zstat _stati64 +# ifdef UNICODE_SUPPORT +# define zwfstat _fstati64 +# define zwstat _wstati64 +# define zw_stat struct _stati64 +# endif +# define zfstat _fstati64 +# define zlstat lstat + + /* 64-bit fseeko */ + /* function in win32.c */ + int zfseeko OF((FILE *, zoff_t, int)); + + /* 64-bit ftello */ + /* function in win32.c */ + zoff_t zftello OF((FILE *)); + + /* 64-bit fopen */ +# define zfopen fopen +# define zfdopen fdopen + +# endif + +# ifdef __IBMC__ + /* IBM C */ + + /* 64-bit stat functions */ + + /* 64-bit fseeko */ + /* function in win32.c */ + int zfseeko OF((FILE *, zoff_t, int)); + + /* 64-bit ftello */ + /* function in win32.c */ + zoff_t zftello OF((FILE *)); + + /* 64-bit fopen */ + +# endif + +# endif /* WIN32 */ + +#else + /* No Large File Support or default for 64-bit environment */ + +# define zstat stat +# define zfstat fstat +# define zlstat lstat +# define zfseeko fseek +# define zftello ftell +# define zfopen fopen +# define zfdopen fdopen +# ifdef UNICODE_SUPPORT +# define zwfstat _fstat +# define zwstat _wstat +# define zw_stat struct _stat +# endif + +#endif + +#ifdef LARGE_FILE_SUPPORT /* E. Gordon 9/12/2003 */ + +# ifndef SSTAT +# define SSTAT zstat +# ifdef UNICODE_SUPPORT +# define SSTATW zwstat +# endif +# endif +# ifdef S_IFLNK +# define LSTAT zlstat +# define LSSTAT(n, s) (linkput ? zlstat((n), (s)) : SSTAT((n), (s))) +# else +# define LSTAT SSTAT +# define LSSTAT SSTAT +# ifdef UNICODE_SUPPORT +# define LSSTATW SSTATW +# endif +# endif + +#else /* no LARGE_FILE_SUPPORT */ + +# ifndef SSTAT +# define SSTAT stat +# endif +# ifdef S_IFLNK +# define LSTAT lstat +# define LSSTAT(n, s) (linkput ? lstat((n), (s)) : SSTAT((n), (s))) +# else +# define LSTAT SSTAT +# define LSSTAT SSTAT +# ifdef UNICODE_SUPPORT +# define LSSTATW SSTATW +# endif +# endif + +#endif + + +/*---------------------------------------------------------------------*/ + + +/* 2004-12-01 SMS. + * Added fancy zofft() macros, et c. + */ + +/* Default fzofft() format selection. + * Modified 2004-12-27 EG + */ + +#ifndef FZOFFT_FMT +# define FZOFFT_FMT ZOFF_T_FORMAT_SIZE_PREFIX /* printf for zoff_t values */ + +# ifdef LARGE_FILE_SUPPORT +# define FZOFFT_HEX_WID_VALUE "16" /* width of 64-bit hex values */ +# else +# define FZOFFT_HEX_WID_VALUE "8" /* digits in 32-bit hex values */ +# endif + +#endif /* ndef FZOFFT_FMT */ + +#define FZOFFT_HEX_WID ((char *) -1) +#define FZOFFT_HEX_DOT_WID ((char *) -2) + + + + +/* The following default definition of the second input for the crypthead() + * random seed computation can be used on most systems (all those that + * supply a UNIX compatible getpid() function). + */ +#ifdef ZCRYPT_INTERNAL +# ifndef ZCR_SEED2 +# define ZCR_SEED2 (unsigned) getpid() /* use PID as seed pattern */ +# endif +#endif /* ZCRYPT_INTERNAL */ + +/* The following OS codes are defined in pkzip appnote.txt */ +#ifdef AMIGA +# define OS_CODE 0x100 +#endif +#ifdef VMS +# define OS_CODE 0x200 +#endif +/* unix 3 */ +#ifdef VM_CMS +# define OS_CODE 0x400 +#endif +#ifdef ATARI +# define OS_CODE 0x500 +#endif +#ifdef OS2 +# define OS_CODE 0x600 +#endif +#ifdef MACOS +# define OS_CODE 0x700 +#endif +/* z system 8 */ +/* cp/m 9 */ +#ifdef TOPS20 +# define OS_CODE 0xa00 +#endif +#ifdef WIN32 +# define OS_CODE 0xb00 +#endif +#ifdef QDOS +# define OS_CODE 0xc00 +#endif +#ifdef RISCOS +# define OS_CODE 0xd00 +#endif +#ifdef VFAT +# define OS_CODE 0xe00 +#endif +#ifdef MVS +# define OS_CODE 0xf00 +#endif +#ifdef __BEOS__ +# define OS_CODE 0x1000 +#endif +#ifdef TANDEM +# define OS_CODE 0x1100 +#endif +#ifdef THEOS +# define OS_CODE 0x1200 +#endif +/* Yes, there is a gap here. */ +#ifdef __ATHEOS__ +# define OS_CODE 0x1E00 +#endif + +#define NUM_HOSTS 31 +/* Number of operating systems. Should be updated when new ports are made */ + +#if defined(DOS) && !defined(OS_CODE) +# define OS_CODE 0x000 +#endif + +#ifndef OS_CODE +# define OS_CODE 0x300 /* assume Unix */ +#endif + +/* can't use "return 0" from main() on VMS */ +#ifndef EXIT +# define EXIT exit +#endif +#ifndef RETURN +# define RETURN return +#endif + +#ifndef ZIPERR +# define ZIPERR ziperr +#endif + +#if (defined(USE_ZLIB) && defined(MY_ZCALLOC)) + /* special zcalloc function is not needed when linked against zlib */ +# undef MY_ZCALLOC +#endif + +#if (!defined(USE_ZLIB) && !defined(MY_ZCALLOC)) + /* Any system without a special calloc function */ +# define zcalloc(items,size) \ + (zvoid far *)calloc((unsigned)(items), (unsigned)(size)) +# define zcfree free +#endif /* !USE_ZLIB && !MY_ZCALLOC */ + +/* end of tailor.h */ diff --git a/third_party/infozip/zip/timezone.c b/third_party/infozip/zip/timezone.c new file mode 100644 index 000000000..54966dc98 --- /dev/null +++ b/third_party/infozip/zip/timezone.c @@ -0,0 +1,816 @@ +/* clang-format off */ +/* + timezone.c - Zip 3 + + Copyright (c) 1990-2004 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2003-May-08 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* Replacement time library functions, based on platform independent public + * domain timezone code from ftp://elsie.nci.nih.gov/pub, with mktime and + * mkgmtime from our own mktime.c in Zip. + * + * Contains: tzset() + * __tzset() + * gmtime() + * localtime() + * mktime() + * mkgmtime() + * GetPlatformLocalTimezone() [different versions] + */ + +/* HISTORY/CHANGES + * 17 Jun 00, Paul Kienitz, added the PD-based tzset(), localtime(), and so on + * to amiga/filedate.c, replacing GNU-based functions which had + * replaced time_lib.c, both having been rejected for licensing + * reasons. Support for timezone files and leap seconds was removed. + * + * 23 Aug 00, Paul Kienitz, split into separate timezone.c file, made platform + * independent, copied in mktime() and mkgmtime() from Zip, renamed + * locale_TZ as GetPlatformLocalTimezone(), for use as a generic + * hook by other platforms. + */ + +#ifndef __timezone_c +#define __timezone_c + + +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/timezone.h" +#include "libc/str/str.h" +#include "libc/errno.h" + +#ifdef IZTZ_DEFINESTDGLOBALS +long timezone = 0; +int daylight = 0; +char *tzname[2]; +#endif + +#ifndef IZTZ_GETLOCALETZINFO +# define IZTZ_GETLOCALETZINFO(ptzstruct, pgenrulefunct) (FALSE) +#endif + +int real_timezone_is_set = FALSE; /* set by tzset() */ + + +#define TZDEFRULESTRING ",M4.1.0,M10.5.0" +#define TZDEFAULT "EST5EDT" + +#define SECSPERMIN 60 +#define MINSPERHOUR 60 +#define HOURSPERDAY 24 +#define DAYSPERWEEK 7 +#define DAYSPERNYEAR 365 +#define DAYSPERLYEAR 366 +#define SECSPERHOUR (SECSPERMIN * MINSPERHOUR) +#define SECSPERDAY ((long) SECSPERHOUR * HOURSPERDAY) +#define MONSPERYEAR 12 + +#define EPOCH_WDAY 4 /* Jan 1, 1970 was thursday */ +#define EPOCH_YEAR 1970 +#define TM_YEAR_BASE 1900 +#define FIRST_GOOD_YEAR ((time_t) -1 < (time_t) 1 ? EPOCH_YEAR-68 : EPOCH_YEAR) +#define LAST_GOOD_YEAR (EPOCH_YEAR + ((time_t) -1 < (time_t) 1 ? 67 : 135)) + +#define YDAYS(month, year) yr_days[leap(year)][month] + +/* Nonzero if `y' is a leap year, else zero. */ +#define leap(y) (((y) % 4 == 0 && (y) % 100 != 0) || (y) % 400 == 0) + +/* Number of leap years from EPOCH_YEAR to `y' (not including `y' itself). */ +#define _P4 ((EPOCH_YEAR / 4) * 4 + 1) +#define _P100 ((EPOCH_YEAR / 100) * 100 + 1) +#define _P400 ((EPOCH_YEAR / 400) * 400 + 1) +#define nleap(y) (((y) - _P4) / 4 - ((y) - _P100) / 100 + ((y) - _P400) / 400) + +/* Length of month `m' (0 .. 11) */ +#define monthlen(m, y) (yr_days[0][(m)+1] - yr_days[0][m] + \ + ((m) == 1 && leap(y))) + +/* internal module-level constants */ +#ifndef IZ_MKTIME_ONLY +static ZCONST char gmt[] = "GMT"; +static ZCONST int mon_lengths[2][MONSPERYEAR] = { + { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, + { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } +}; +#endif /* !IZ_MKTIME_ONLY */ +static ZCONST int yr_days[2][MONSPERYEAR+1] = { + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } +}; +#ifndef IZ_MKTIME_ONLY +static ZCONST int year_lengths[2] = { + DAYSPERNYEAR, DAYSPERLYEAR +}; + +/* internal variables */ +static struct state statism; + + +/* prototypes of static functions */ +static time_t transtime OF((ZCONST time_t janfirst, ZCONST int year, + ZCONST struct rule * ZCONST rulep, + ZCONST long offset)); +static void generate_transitions OF((register struct state * ZCONST sp, + ZCONST struct rule * ZCONST start, + ZCONST struct rule * ZCONST end)); +static ZCONST char *getzname OF((ZCONST char *strp)); +static ZCONST char *getnum OF((ZCONST char *strp, int * ZCONST nump, + ZCONST int min, ZCONST int max)); +static ZCONST char *getsecs OF((ZCONST char *strp, long * ZCONST secsp)); +static ZCONST char *getoffset OF((ZCONST char *strp, long * ZCONST offsetp)); +static ZCONST char *getrule OF((ZCONST char *strp, struct rule * ZCONST rulep)); +static int Parse_TZ OF((ZCONST char *name, register struct state * ZCONST sp)); + + +static time_t transtime(janfirst, year, rulep, offset) + ZCONST time_t janfirst; + ZCONST int year; + ZCONST struct rule * ZCONST rulep; + ZCONST long offset; +{ + register int leapyear; + register time_t value; + register int i; + int d, m1, yy0, yy1, yy2, dow; + + value = 0; + leapyear = leap(year); + switch (rulep->r_type) { + + case JULIAN_DAY: + /* + ** Jn - Julian day, 1 == January 1, 60 == March 1 even in leap + ** years. + ** In non-leap years, or if the day number is 59 or less, just + ** add SECSPERDAY times the day number-1 to the time of + ** January 1, midnight, to get the day. + */ + value = janfirst + (rulep->r_day - 1) * SECSPERDAY; + if (leapyear && rulep->r_day >= 60) + value += SECSPERDAY; + break; + + case DAY_OF_YEAR: + /* + ** n - day of year. + ** Just add SECSPERDAY times the day number to the time of + ** January 1, midnight, to get the day. + */ + value = janfirst + rulep->r_day * SECSPERDAY; + break; + + case MONTH_NTH_DAY_OF_WEEK: + /* + ** Mm.n.d - nth "dth day" of month m. + */ + value = janfirst; +/* + for (i = 0; i < rulep->r_mon - 1; ++i) + value += mon_lengths[leapyear][i] * SECSPERDAY; +*/ + value += yr_days[leapyear][rulep->r_mon - 1] * SECSPERDAY; + + /* + ** Use Zeller's Congruence to get day-of-week of first day of + ** month. + */ + m1 = (rulep->r_mon + 9) % 12 + 1; + yy0 = (rulep->r_mon <= 2) ? (year - 1) : year; + yy1 = yy0 / 100; + yy2 = yy0 % 100; + dow = ((26 * m1 - 2) / 10 + + 1 + yy2 + yy2 / 4 + yy1 / 4 - 2 * yy1) % 7; + if (dow < 0) + dow += DAYSPERWEEK; + + /* + ** "dow" is the day-of-week of the first day of the month. Get + ** the day-of-month (zero-origin) of the first "dow" day of the + ** month. + */ + d = rulep->r_day - dow; + if (d < 0) + d += DAYSPERWEEK; + for (i = 1; i < rulep->r_week; ++i) { + if (d + DAYSPERWEEK >= mon_lengths[leapyear][rulep->r_mon - 1]) + break; + d += DAYSPERWEEK; + } + + /* + ** "d" is the day-of-month (zero-origin) of the day we want. + */ + value += d * SECSPERDAY; + break; + } + + /* + ** "value" is the Epoch-relative time of 00:00:00 UTC on the day in + ** question. To get the Epoch-relative time of the specified local + ** time on that day, add the transition time and the current offset + ** from UTC. + */ + return value + rulep->r_time + offset; +} + +static void generate_transitions(sp, start, end) + register struct state * ZCONST sp; + ZCONST struct rule * ZCONST start; + ZCONST struct rule * ZCONST end; +{ + register int year; + register time_t janfirst; + time_t starttime; + time_t endtime; + long stdoffset = -sp->ttis[0].tt_gmtoff; + long dstoffset = -sp->ttis[1].tt_gmtoff; + register time_t * atp; + register unsigned char * typep; + + /* + ** Two transitions per year, from EPOCH_YEAR to LAST_GOOD_YEAR. + */ + sp->timecnt = 2 * (LAST_GOOD_YEAR - EPOCH_YEAR + 1); + atp = sp->ats; + typep = sp->types; + janfirst = 0; + for (year = EPOCH_YEAR; year <= LAST_GOOD_YEAR; ++year) { + starttime = transtime(janfirst, year, start, stdoffset); + endtime = transtime(janfirst, year, end, dstoffset); + if (starttime > endtime) { + *atp++ = endtime; + *typep++ = 0; /* DST ends */ + *atp++ = starttime; + *typep++ = 1; /* DST begins */ + } else { + *atp++ = starttime; + *typep++ = 1; /* DST begins */ + *atp++ = endtime; + *typep++ = 0; /* DST ends */ + } + janfirst += year_lengths[leap(year)] * SECSPERDAY; + } +} + +static ZCONST char *getzname(strp) + ZCONST char *strp; +{ + register char c; + + while ((c = *strp) != '\0' && !isdigit(c) && c != ',' && c != '-' && + c != '+') + ++strp; + return strp; +} + +static ZCONST char *getnum(strp, nump, min, max) + ZCONST char *strp; + int * ZCONST nump; + ZCONST int min; + ZCONST int max; +{ + register char c; + register int num; + + if (strp == NULL || !isdigit(c = *strp)) + return NULL; + num = 0; + do { + num = num * 10 + (c - '0'); + if (num > max) + return NULL; /* illegal value */ + c = *++strp; + } while (isdigit(c)); + if (num < min) + return NULL; /* illegal value */ + *nump = num; + return strp; +} + +static ZCONST char *getsecs(strp, secsp) + ZCONST char *strp; + long * ZCONST secsp; +{ + int num; + + /* + ** `HOURSPERDAY * DAYSPERWEEK - 1' allows quasi-Posix rules like + ** "M10.4.6/26", which does not conform to Posix, + ** but which specifies the equivalent of + ** ``02:00 on the first Sunday on or after 23 Oct''. + */ + strp = getnum(strp, &num, 0, HOURSPERDAY * DAYSPERWEEK - 1); + if (strp == NULL) + return NULL; + *secsp = num * (long) SECSPERHOUR; + if (*strp == ':') { + ++strp; + strp = getnum(strp, &num, 0, MINSPERHOUR - 1); + if (strp == NULL) + return NULL; + *secsp += num * SECSPERMIN; + if (*strp == ':') { + ++strp; + /* `SECSPERMIN' allows for leap seconds. */ + strp = getnum(strp, &num, 0, SECSPERMIN); + if (strp == NULL) + return NULL; + *secsp += num; + } + } + return strp; +} + +static ZCONST char *getoffset(strp, offsetp) + ZCONST char *strp; + long * ZCONST offsetp; +{ + register int neg = 0; + + if (*strp == '-') { + neg = 1; + ++strp; + } else if (*strp == '+') + ++strp; + strp = getsecs(strp, offsetp); + if (strp == NULL) + return NULL; /* illegal time */ + if (neg) + *offsetp = -*offsetp; + return strp; +} + +static ZCONST char *getrule(strp, rulep) + ZCONST char *strp; + struct rule * ZCONST rulep; +{ + if (*strp == 'J') { + /* + ** Julian day. + */ + rulep->r_type = JULIAN_DAY; + ++strp; + strp = getnum(strp, &rulep->r_day, 1, DAYSPERNYEAR); + } else if (*strp == 'M') { + /* + ** Month, week, day. + */ + rulep->r_type = MONTH_NTH_DAY_OF_WEEK; + ++strp; + strp = getnum(strp, &rulep->r_mon, 1, MONSPERYEAR); + if (strp == NULL) + return NULL; + if (*strp++ != '.') + return NULL; + strp = getnum(strp, &rulep->r_week, 1, 5); + if (strp == NULL) + return NULL; + if (*strp++ != '.') + return NULL; + strp = getnum(strp, &rulep->r_day, 0, DAYSPERWEEK - 1); + } else if (isdigit(*strp)) { + /* + ** Day of year. + */ + rulep->r_type = DAY_OF_YEAR; + strp = getnum(strp, &rulep->r_day, 0, DAYSPERLYEAR - 1); + } else return NULL; /* invalid format */ + if (strp == NULL) + return NULL; + if (*strp == '/') { + /* + ** Time specified. + */ + ++strp; + strp = getsecs(strp, &rulep->r_time); + } else + rulep->r_time = 2 * SECSPERHOUR; /* default = 2:00:00 */ + return strp; +} + +static int Parse_TZ(name, sp) + ZCONST char *name; + register struct state * ZCONST sp; +{ + ZCONST char * stdname; + ZCONST char * dstname; + size_t stdlen; + size_t dstlen; + long stdoffset; + long dstoffset; + register char * cp; + + dstname = NULL; + stdname = name; + name = getzname(name); + stdlen = name - stdname; + if (stdlen < 3) + return -1; + if (*name == '\0') + return -1; + name = getoffset(name, &stdoffset); + if (name == NULL) + return -1; + if (*name != '\0') { + dstname = name; + name = getzname(name); + dstlen = name - dstname; /* length of DST zone name */ + if (dstlen < 3) + return -1; + if (*name != '\0' && *name != ',' && *name != ';') { + name = getoffset(name, &dstoffset); + if (name == NULL) + return -1; + } else + dstoffset = stdoffset - SECSPERHOUR; + if (*name == '\0') + name = TZDEFRULESTRING; + if (*name == ',' || *name == ';') { + struct rule start; + struct rule end; + + ++name; + if ((name = getrule(name, &start)) == NULL) + return -1; + if (*name++ != ',') + return -1; + if ((name = getrule(name, &end)) == NULL) + return -1; + if (*name != '\0') + return -1; + sp->typecnt = 2; /* standard time and DST */ + sp->ttis[0].tt_gmtoff = -stdoffset; + sp->ttis[0].tt_isdst = 0; + sp->ttis[0].tt_abbrind = 0; + sp->ttis[1].tt_gmtoff = -dstoffset; + sp->ttis[1].tt_isdst = 1; + sp->ttis[1].tt_abbrind = stdlen + 1; + generate_transitions(sp, &start, &end); + } + } else { + dstlen = 0; + sp->typecnt = 1; /* only standard time */ + sp->timecnt = 0; + sp->ttis[0].tt_gmtoff = -stdoffset; + sp->ttis[0].tt_isdst = 0; + sp->ttis[0].tt_abbrind = 0; + } + sp->charcnt = stdlen + 1; + if (dstlen != 0) + sp->charcnt += dstlen + 1; + if ((size_t) sp->charcnt > sizeof(sp->chars)) + return -1; + cp = sp->chars; + (void) strncpy(cp, stdname, stdlen); + cp += stdlen; + *cp++ = '\0'; + if (dstlen != 0) { + (void) strncpy(cp, dstname, dstlen); + *(cp + dstlen) = '\0'; + } + return 0; +} + +void tzset() +{ + char *TZstring; + int dstfirst; + static char *old_TZstring = NULL; + + TZstring = getenv("TZ"); /* read TZ envvar */ + if (old_TZstring && TZstring && !strcmp(old_TZstring, TZstring)) + /* do not repeatedly parse an unchanged TZ specification */ + return; + if ((TZstring && TZstring[0] && Parse_TZ(TZstring, &statism) == 0) + || IZTZ_GETLOCALETZINFO(&statism, generate_transitions) + || Parse_TZ(gmt, &statism) == 0) { + daylight = statism.typecnt > 1; + dstfirst = daylight && statism.ttis[0].tt_isdst && !statism.ttis[1].tt_isdst; + timezone = -statism.ttis[dstfirst].tt_gmtoff; + tzname[0] = statism.chars + statism.ttis[dstfirst].tt_abbrind; + tzname[1] = statism.chars + statism.ttis[!dstfirst].tt_abbrind; + real_timezone_is_set = TRUE; + if (TZstring) { + if (old_TZstring) + old_TZstring = realloc(old_TZstring, strlen(TZstring) + 1); + else + old_TZstring = malloc(strlen(TZstring) + 1); + if (old_TZstring) + strcpy(old_TZstring, TZstring); + } + } else { + timezone = 0; /* default is GMT0 which means no offsets */ + daylight = 0; /* from local system time */ + real_timezone_is_set = FALSE; + if (old_TZstring) { + free(old_TZstring); + old_TZstring = NULL; + } + } +#ifdef IZTZ_SETLOCALTZINFO + /* Some SAS/C library functions, e.g. stat(), call library */ + /* __tzset() themselves. So envvar TZ *must* exist in order to */ + /* to get the right offset from GMT. XXX TRY HARD to fix this! */ + set_TZ(timezone, daylight); +#endif /* IZTZ_SETLOCALTZINFO */ +} + +/* XXX Does this also help SAS/C library work? */ +void __tzset() +{ + if (!real_timezone_is_set) tzset(); +} + +static struct tm _tmbuf; + +struct tm *gmtime(when) + ZCONST time_t *when; +{ + long days = *when / SECSPERDAY; + long secs = *when % SECSPERDAY; + int isleap; + + memset(&_tmbuf, 0, sizeof(_tmbuf)); /* get any nonstandard fields */ + _tmbuf.tm_wday = (days + EPOCH_WDAY) % 7; + _tmbuf.tm_year = EPOCH_YEAR - TM_YEAR_BASE; + isleap = leap(_tmbuf.tm_year + TM_YEAR_BASE); + while (days >= year_lengths[isleap]) { + days -= year_lengths[isleap]; + _tmbuf.tm_year++; + isleap = leap(_tmbuf.tm_year + TM_YEAR_BASE); + } + _tmbuf.tm_mon = 0; + _tmbuf.tm_yday = days; + while (days >= mon_lengths[isleap][_tmbuf.tm_mon]) + days -= mon_lengths[isleap][_tmbuf.tm_mon++]; + _tmbuf.tm_mday = days + 1; + _tmbuf.tm_isdst = 0; + _tmbuf.tm_sec = secs % SECSPERMIN; + _tmbuf.tm_min = (secs / SECSPERMIN) % SECSPERMIN; + _tmbuf.tm_hour = secs / SECSPERHOUR; + return &_tmbuf; +} + +struct tm *localtime(when) + ZCONST time_t *when; +{ + time_t localwhen = *when; + int timetype; + struct tm *ret; + + __tzset(); + if (statism.timecnt == 0 || localwhen < statism.ats[0]) + timetype = statism.ttis[0].tt_isdst && statism.typecnt > 1 && + !statism.ttis[1].tt_isdst; + else { + for (timetype = 1; timetype < statism.timecnt; ++timetype) + if (localwhen < statism.ats[timetype]) + break; + timetype = statism.types[timetype - 1]; + } + localwhen += statism.ttis[timetype].tt_gmtoff; + ret = gmtime(&localwhen); + ret->tm_isdst = statism.ttis[timetype].tt_isdst; + return ret; +} + +#ifdef NEED__ISINDST +int _isindst(tb) + struct tm *tb; +{ + time_t localt; /* time_t equivalent of given tm struct */ + time_t univt; /* assumed UTC value of given time */ + long tzoffset_adj; /* timezone-adjustment `remainder' */ + int bailout_cnt; /* counter of tries for tz correction */ + int timetype; + + __tzset(); + + /* when DST is unsupported in current timezone, DST is always off */ + if (statism.typecnt <= 1) return FALSE; + + localt = mkgmtime(tb); + if (localt == (time_t)-1) + /* specified time is out-of-range, default to FALSE */ + return FALSE; + + univt = localt - statism.ttis[0].tt_gmtoff; + bailout_cnt = 3; + do { + if (statism.timecnt == 0 || univt < statism.ats[0]) + timetype = statism.ttis[0].tt_isdst && statism.typecnt > 1 && + !statism.ttis[1].tt_isdst; + else { + for (timetype = 1; timetype < statism.timecnt; ++timetype) + if (univt < statism.ats[timetype]) + break; + timetype = statism.types[timetype - 1]; + } + if ((tzoffset_adj = localt - univt - statism.ttis[timetype].tt_gmtoff) + == 0L) + break; + univt += tzoffset_adj; + } while (--bailout_cnt > 0); + + /* return TRUE when DST is active at given time */ + return (statism.ttis[timetype].tt_isdst); +} +#endif /* NEED__ISINDST */ +#endif /* !IZ_MKTIME_ONLY */ + +/* Return the equivalent in seconds past 12:00:00 a.m. Jan 1, 1970 GMT + of the local time and date in the exploded time structure `tm', + adjust out of range fields in `tm' and set `tm->tm_yday', `tm->tm_wday'. + If `tm->tm_isdst < 0' was passed to mktime(), the correct setting of + tm_isdst is determined and returned. Otherwise, mktime() assumes this + field as valid; its information is used when converting local time + to UTC. + Return -1 if time in `tm' cannot be represented as time_t value. */ + +time_t mktime(tm) + struct tm *tm; +{ + struct tm *ltm; /* Local time. */ + time_t loctime; /* The time_t value of local time. */ + time_t then; /* The time to return. */ + long tzoffset_adj; /* timezone-adjustment `remainder' */ + int bailout_cnt; /* counter of tries for tz correction */ + int save_isdst; /* Copy of the tm->isdst input value */ + + save_isdst = tm->tm_isdst; + loctime = mkgmtime(tm); + if (loctime == -1) { + tm->tm_isdst = save_isdst; + return (time_t)-1; + } + + /* Correct for the timezone and any daylight savings time. + The correction is verified and repeated when not correct, to + take into account the rare case that a change to or from daylight + savings time occurs between when it is the time in `tm' locally + and when it is that time in Greenwich. After the second correction, + the "timezone & daylight" offset should be correct in all cases. To + be sure, we allow a third try, but then the loop is stopped. */ + bailout_cnt = 3; + then = loctime; + do { + ltm = localtime(&then); + if (ltm == (struct tm *)NULL || + (tzoffset_adj = loctime - mkgmtime(ltm)) == 0L) + break; + then += tzoffset_adj; + } while (--bailout_cnt > 0); + + if (ltm == (struct tm *)NULL || tzoffset_adj != 0L) { + /* Signal failure if timezone adjustment did not converge. */ + tm->tm_isdst = save_isdst; + return (time_t)-1; + } + + if (save_isdst >= 0) { + if (ltm->tm_isdst && !save_isdst) + { + if (then + 3600 < then) + then = (time_t)-1; + else + then += 3600; + } + else if (!ltm->tm_isdst && save_isdst) + { + if (then - 3600 > then) + then = (time_t)-1; + else + then -= 3600; + } + ltm->tm_isdst = save_isdst; + } + + if (tm != ltm) /* `tm' may already point to localtime's internal storage */ + *tm = *ltm; + + return then; +} + + +#ifndef NO_TIME_T_MAX + /* Provide default values for the upper limit of the time_t range. + These are the result of the decomposition into a `struct tm' for + the time value 0xFFFFFFFEL ( = (time_t)-2 ). + Note: `(time_t)-1' is reserved for "invalid time"! */ +# ifndef TM_YEAR_MAX +# define TM_YEAR_MAX 2106 +# endif +# ifndef TM_MON_MAX +# define TM_MON_MAX 1 /* February */ +# endif +# ifndef TM_MDAY_MAX +# define TM_MDAY_MAX 7 +# endif +# ifndef TM_HOUR_MAX +# define TM_HOUR_MAX 6 +# endif +# ifndef TM_MIN_MAX +# define TM_MIN_MAX 28 +# endif +# ifndef TM_SEC_MAX +# define TM_SEC_MAX 14 +# endif +#endif /* NO_TIME_T_MAX */ + +/* Adjusts out-of-range values for `tm' field `tm_member'. */ +#define ADJUST_TM(tm_member, tm_carry, modulus) \ + if ((tm_member) < 0) { \ + tm_carry -= (1 - ((tm_member)+1) / (modulus)); \ + tm_member = (modulus-1) + (((tm_member)+1) % (modulus)); \ + } else if ((tm_member) >= (modulus)) { \ + tm_carry += (tm_member) / (modulus); \ + tm_member = (tm_member) % (modulus); \ + } + +/* Return the equivalent in seconds past 12:00:00 a.m. Jan 1, 1970 GMT + of the Greenwich Mean time and date in the exploded time structure `tm'. + This function does always put back normalized values into the `tm' struct, + parameter, including the calculated numbers for `tm->tm_yday', + `tm->tm_wday', and `tm->tm_isdst'. + Returns -1 if the time in the `tm' parameter cannot be represented + as valid `time_t' number. */ + +time_t mkgmtime(tm) + struct tm *tm; +{ + int years, months, days, hours, minutes, seconds; + + years = tm->tm_year + TM_YEAR_BASE; /* year - 1900 -> year */ + months = tm->tm_mon; /* 0..11 */ + days = tm->tm_mday - 1; /* 1..31 -> 0..30 */ + hours = tm->tm_hour; /* 0..23 */ + minutes = tm->tm_min; /* 0..59 */ + seconds = tm->tm_sec; /* 0..61 in ANSI C. */ + + ADJUST_TM(seconds, minutes, 60) + ADJUST_TM(minutes, hours, 60) + ADJUST_TM(hours, days, 24) + ADJUST_TM(months, years, 12) + if (days < 0) + do { + if (--months < 0) { + --years; + months = 11; + } + days += monthlen(months, years); + } while (days < 0); + else + while (days >= monthlen(months, years)) { + days -= monthlen(months, years); + if (++months >= 12) { + ++years; + months = 0; + } + } + + /* Restore adjusted values in tm structure */ + tm->tm_year = years - TM_YEAR_BASE; + tm->tm_mon = months; + tm->tm_mday = days + 1; + tm->tm_hour = hours; + tm->tm_min = minutes; + tm->tm_sec = seconds; + + /* Set `days' to the number of days into the year. */ + days += YDAYS(months, years); + tm->tm_yday = days; + + /* Now calculate `days' to the number of days since Jan 1, 1970. */ + days = (unsigned)days + 365 * (unsigned)(years - EPOCH_YEAR) + + (unsigned)(nleap (years)); + tm->tm_wday = ((unsigned)days + EPOCH_WDAY) % 7; + tm->tm_isdst = 0; + + if (years < EPOCH_YEAR) + return (time_t)-1; + +#if (defined(TM_YEAR_MAX) && defined(TM_MON_MAX) && defined(TM_MDAY_MAX)) +#if (defined(TM_HOUR_MAX) && defined(TM_MIN_MAX) && defined(TM_SEC_MAX)) + if (years > TM_YEAR_MAX || + (years == TM_YEAR_MAX && + (tm->tm_yday > (YDAYS(TM_MON_MAX, TM_YEAR_MAX) + (TM_MDAY_MAX - 1)) || + (tm->tm_yday == (YDAYS(TM_MON_MAX, TM_YEAR_MAX) + (TM_MDAY_MAX - 1)) && + (hours > TM_HOUR_MAX || + (hours == TM_HOUR_MAX && + (minutes > TM_MIN_MAX || + (minutes == TM_MIN_MAX && seconds > TM_SEC_MAX) ))))))) + return (time_t)-1; +#endif +#endif + + return (time_t)(SECSPERDAY * (unsigned long)(unsigned)days + + SECSPERHOUR * (unsigned long)hours + + (unsigned long)(SECSPERMIN * minutes + seconds)); +} + +#endif /* __timezone_c */ diff --git a/third_party/infozip/zip/timezone.h b/third_party/infozip/zip/timezone.h new file mode 100644 index 000000000..076c7f05f --- /dev/null +++ b/third_party/infozip/zip/timezone.h @@ -0,0 +1,84 @@ +/* clang-format off */ +/* + timezone.h - Zip 3 + + Copyright (c) 1990-2004 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2003-May-08 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +#ifndef __timezone_h +#define __timezone_h + +#ifndef IZ_MKTIME_ONLY + +/* limits for our timezone info data: + * we support only basic standard and daylight time, with max 2 transitions + * per year, but for the maximum range of years a 32-bit second counter + * can cover (these are 136 years plus a bit more than one month) + */ +#define TZ_MAX_TIMES 272 /* (=2*(LastGoodYr + 1 - FirstGoodYr) */ +#define TZ_MAX_TYPES 2 /* We only support basic standard and daylight */ +#ifdef WIN32 /* Win32 tzinfo supplies at max (2 * 32) chars of tz names */ +#define TZ_MAX_CHARS 64 /* Maximum number of abbreviation characters */ +#else +#define TZ_MAX_CHARS 50 /* Maximum number of abbreviation characters */ +#endif + +/* supported types of transition rules */ +#define JULIAN_DAY 0 /* Jn - Julian day */ +#define DAY_OF_YEAR 1 /* n - day of year */ +#define MONTH_NTH_DAY_OF_WEEK 2 /* Mm.n.d - month, week, day of week */ + + +struct ttinfo { + long tt_gmtoff; /* UTC offset in seconds */ + int tt_isdst; /* used to set tm_isdst */ + int tt_abbrind; /* abbreviation list index */ +}; + +struct state { + int timecnt; + int typecnt; + int charcnt; + time_t ats[TZ_MAX_TIMES]; + unsigned char types[TZ_MAX_TIMES]; + struct ttinfo ttis[TZ_MAX_TYPES]; + char chars[TZ_MAX_CHARS]; +}; + +struct rule { + int r_type; /* type of rule--JULIAN_DAY etc */ + int r_day; /* day number of rule */ + int r_week; /* week number of rule */ + int r_mon; /* month number of rule */ + long r_time; /* transition time of rule */ +}; + +extern int real_timezone_is_set; /* set by tzset() */ + + +/* prototypes of functions not in time.h */ + +void __tzset OF((void)); + +#ifdef NEED__ISINDST +int _isindst OF((struct tm *tb)); +#endif + +/* callback function to be supplied by the program that uses this library */ +int GetPlatformLocalTimezone OF((register struct state * ZCONST sp, + void (*fill_tzstate_from_rules)(struct state * ZCONST sp_res, + ZCONST struct rule * ZCONST start, + ZCONST struct rule * ZCONST end))); +#ifdef IZTZ_SETLOCALTZINFO +void set_TZ OF((long time_zone, int day_light)); +#endif + +#endif /* !IZ_MKTIME_ONLY */ + +time_t mkgmtime OF((struct tm *tm)); + +#endif diff --git a/third_party/infozip/zip/trees.c b/third_party/infozip/zip/trees.c new file mode 100644 index 000000000..f9df6b09e --- /dev/null +++ b/third_party/infozip/zip/trees.c @@ -0,0 +1,1475 @@ +/* clang-format off */ +/* + trees.h - Zip 3 + + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * trees.c by Jean-loup Gailly + * + * This is a new version of im_ctree.c originally written by Richard B. Wales + * for the defunct implosion method. + * The low level bit string handling routines from bits.c (originally + * im_bits.c written by Richard B. Wales) have been merged into this version + * of trees.c. + * + * PURPOSE + * + * Encode various sets of source values using variable-length + * binary code trees. + * Output the resulting variable-length bit strings. + * Compression can be done to a file or to memory. + * + * DISCUSSION + * + * The PKZIP "deflation" process uses several Huffman trees. The more + * common source values are represented by shorter bit sequences. + * + * Each code tree is stored in the ZIP file in a compressed form + * which is itself a Huffman encoding of the lengths of + * all the code strings (in ascending order by source values). + * The actual code strings are reconstructed from the lengths in + * the UNZIP process, as described in the "application note" + * (APPNOTE.TXT) distributed as part of PKWARE's PKZIP program. + * + * The PKZIP "deflate" file format interprets compressed file data + * as a sequence of bits. Multi-bit strings in the file may cross + * byte boundaries without restriction. + * The first bit of each byte is the low-order bit. + * + * The routines in this file allow a variable-length bit value to + * be output right-to-left (useful for literal values). For + * left-to-right output (useful for code strings from the tree routines), + * the bits must have been reversed first with bi_reverse(). + * + * For in-memory compression, the compressed bit stream goes directly + * into the requested output buffer. The buffer is limited to 64K on + * 16 bit machines; flushing of the output buffer during compression + * process is not supported. + * The input data is read in blocks by the (*read_buf)() function. + * + * For more details about input to and output from the deflation routines, + * see the actual input functions for (*read_buf)(), flush_outbuf(), and + * the filecompress() resp. memcompress() wrapper functions which handle + * the I/O setup. + * + * REFERENCES + * + * Lynch, Thomas J. + * Data Compression: Techniques and Applications, pp. 53-55. + * Lifetime Learning Publications, 1985. ISBN 0-534-03418-7. + * + * Storer, James A. + * Data Compression: Methods and Theory, pp. 49-50. + * Computer Science Press, 1988. ISBN 0-7167-8156-5. + * + * Sedgewick, R. + * Algorithms, p290. + * Addison-Wesley, 1983. ISBN 0-201-06672-6. + * + * INTERFACE + * + * void ct_init (ush *attr, int *method) + * Allocate the match buffer, initialize the various tables and save + * the location of the internal file attribute (ascii/binary) and + * method (DEFLATE/STORE) + * + * void ct_tally (int dist, int lc); + * Save the match info and tally the frequency counts. + * + * uzoff_t flush_block (char *buf, ulg stored_len, int eof) + * Determine the best encoding for the current block: dynamic trees, + * static trees or store, and output the encoded block to the zip + * file. Returns the total compressed length for the file so far. + * + * void bi_init (char *tgt_buf, unsigned tgt_size, int flsh_allowed) + * Initialize the bit string routines. + * + * Most of the bit string output functions are only used internally + * in this source file, they are normally declared as "local" routines: + * + * local void send_bits (int value, int length) + * Write out a bit string, taking the source bits right to + * left. + * + * local unsigned bi_reverse (unsigned code, int len) + * Reverse the bits of a bit string, taking the source bits left to + * right and emitting them right to left. + * + * local void bi_windup (void) + * Write out any remaining bits in an incomplete byte. + * + * local void copy_block(char *buf, unsigned len, int header) + * Copy a stored block to the zip file, storing first the length and + * its one's complement if requested. + * + * All output that exceeds the bitstring output buffer size (as initialized + * by bi_init() is fed through an externally provided transfer routine + * which flushes the bitstring output buffer on request and resets the + * buffer fill counter: + * + * extern void flush_outbuf(char *o_buf, unsigned *o_idx); + * + */ +#define __TREES_C + +/* Put zip.h first as when using 64-bit file environment in unix ctype.h + defines off_t and then while other files are using an 8-byte off_t this + file gets a 4-byte off_t. Once zip.h sets the large file defines can + then include ctype.h and get 8-byte off_t. 8/14/04 EG */ +#include "third_party/infozip/zip/zip.h" +#include "libc/str/str.h" + +#ifndef USE_ZLIB + +/* =========================================================================== + * Constants + */ + +#define MAX_BITS 15 +/* All codes must not exceed MAX_BITS bits */ + +#define MAX_BL_BITS 7 +/* Bit length codes must not exceed MAX_BL_BITS bits */ + +#define LENGTH_CODES 29 +/* number of length codes, not counting the special END_BLOCK code */ + +#define LITERALS 256 +/* number of literal bytes 0..255 */ + +#define END_BLOCK 256 +/* end of block literal code */ + +#define L_CODES (LITERALS+1+LENGTH_CODES) +/* number of Literal or Length codes, including the END_BLOCK code */ + +#define D_CODES 30 +/* number of distance codes */ + +#define BL_CODES 19 +/* number of codes used to transfer the bit lengths */ + + +local int near extra_lbits[LENGTH_CODES] /* extra bits for each length code */ + = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0}; + +local int near extra_dbits[D_CODES] /* extra bits for each distance code */ + = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +local int near extra_blbits[BL_CODES]/* extra bits for each bit length code */ + = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7}; + +#define STORED_BLOCK 0 +#define STATIC_TREES 1 +#define DYN_TREES 2 +/* The three kinds of block type */ + +#ifndef LIT_BUFSIZE +# ifdef SMALL_MEM +# define LIT_BUFSIZE 0x2000 +# else +# ifdef MEDIUM_MEM +# define LIT_BUFSIZE 0x4000 +# else +# define LIT_BUFSIZE 0x8000 +# endif +# endif +#endif +#define DIST_BUFSIZE LIT_BUFSIZE +/* Sizes of match buffers for literals/lengths and distances. There are + * 4 reasons for limiting LIT_BUFSIZE to 64K: + * - frequencies can be kept in 16 bit counters + * - if compression is not successful for the first block, all input data is + * still in the window so we can still emit a stored block even when input + * comes from standard input. (This can also be done for all blocks if + * LIT_BUFSIZE is not greater than 32K.) + * - if compression is not successful for a file smaller than 64K, we can + * even emit a stored file instead of a stored block (saving 5 bytes). + * - creating new Huffman trees less frequently may not provide fast + * adaptation to changes in the input data statistics. (Take for + * example a binary file with poorly compressible code followed by + * a highly compressible string table.) Smaller buffer sizes give + * fast adaptation but have of course the overhead of transmitting trees + * more frequently. + * - I can't count above 4 + * The current code is general and allows DIST_BUFSIZE < LIT_BUFSIZE (to save + * memory at the expense of compression). Some optimizations would be possible + * if we rely on DIST_BUFSIZE == LIT_BUFSIZE. + */ + +#define REP_3_6 16 +/* repeat previous bit length 3-6 times (2 bits of repeat count) */ + +#define REPZ_3_10 17 +/* repeat a zero length 3-10 times (3 bits of repeat count) */ + +#define REPZ_11_138 18 +/* repeat a zero length 11-138 times (7 bits of repeat count) */ + +/* =========================================================================== + * Local data + */ + +/* Data structure describing a single value and its code string. */ +typedef struct ct_data { + union { + ush freq; /* frequency count */ + ush code; /* bit string */ + } fc; + union { + ush dad; /* father node in Huffman tree */ + ush len; /* length of bit string */ + } dl; +} ct_data; + +#define Freq fc.freq +#define Code fc.code +#define Dad dl.dad +#define Len dl.len + +#define HEAP_SIZE (2*L_CODES+1) +/* maximum heap size */ + +local ct_data near dyn_ltree[HEAP_SIZE]; /* literal and length tree */ +local ct_data near dyn_dtree[2*D_CODES+1]; /* distance tree */ + +local ct_data near static_ltree[L_CODES+2]; +/* The static literal tree. Since the bit lengths are imposed, there is no + * need for the L_CODES extra codes used during heap construction. However + * The codes 286 and 287 are needed to build a canonical tree (see ct_init + * below). + */ + +local ct_data near static_dtree[D_CODES]; +/* The static distance tree. (Actually a trivial tree since all codes use + * 5 bits.) + */ + +local ct_data near bl_tree[2*BL_CODES+1]; +/* Huffman tree for the bit lengths */ + +typedef struct tree_desc { + ct_data near *dyn_tree; /* the dynamic tree */ + ct_data near *static_tree; /* corresponding static tree or NULL */ + int near *extra_bits; /* extra bits for each code or NULL */ + int extra_base; /* base index for extra_bits */ + int elems; /* max number of elements in the tree */ + int max_length; /* max bit length for the codes */ + int max_code; /* largest code with non zero frequency */ +} tree_desc; + +local tree_desc near l_desc = +{dyn_ltree, static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS, 0}; + +local tree_desc near d_desc = +{dyn_dtree, static_dtree, extra_dbits, 0, D_CODES, MAX_BITS, 0}; + +local tree_desc near bl_desc = +{bl_tree, NULL, extra_blbits, 0, BL_CODES, MAX_BL_BITS, 0}; + + +local ush near bl_count[MAX_BITS+1]; +/* number of codes at each bit length for an optimal tree */ + +local uch near bl_order[BL_CODES] + = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15}; +/* The lengths of the bit length codes are sent in order of decreasing + * probability, to avoid transmitting the lengths for unused bit length codes. + */ + +local int near heap[2*L_CODES+1]; /* heap used to build the Huffman trees */ +local int heap_len; /* number of elements in the heap */ +local int heap_max; /* element of largest frequency */ +/* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used. + * The same heap array is used to build all trees. + */ + +local uch near depth[2*L_CODES+1]; +/* Depth of each subtree used as tie breaker for trees of equal frequency */ + +local uch length_code[MAX_MATCH-MIN_MATCH+1]; +/* length code for each normalized match length (0 == MIN_MATCH) */ + +local uch dist_code[512]; +/* distance codes. The first 256 values correspond to the distances + * 3 .. 258, the last 256 values correspond to the top 8 bits of + * the 15 bit distances. + */ + +local int near base_length[LENGTH_CODES]; +/* First normalized length for each code (0 = MIN_MATCH) */ + +local int near base_dist[D_CODES]; +/* First normalized distance for each code (0 = distance of 1) */ + +#ifndef DYN_ALLOC + local uch far l_buf[LIT_BUFSIZE]; /* buffer for literals/lengths */ + local ush far d_buf[DIST_BUFSIZE]; /* buffer for distances */ +#else + local uch far *l_buf; + local ush far *d_buf; +#endif + +local uch near flag_buf[(LIT_BUFSIZE/8)]; +/* flag_buf is a bit array distinguishing literals from lengths in + * l_buf, and thus indicating the presence or absence of a distance. + */ + +local unsigned last_lit; /* running index in l_buf */ +local unsigned last_dist; /* running index in d_buf */ +local unsigned last_flags; /* running index in flag_buf */ +local uch flags; /* current flags not yet saved in flag_buf */ +local uch flag_bit; /* current bit used in flags */ +/* bits are filled in flags starting at bit 0 (least significant). + * Note: these flags are overkill in the current code since we don't + * take advantage of DIST_BUFSIZE == LIT_BUFSIZE. + */ + +local ulg opt_len; /* bit length of current block with optimal trees */ +local ulg static_len; /* bit length of current block with static trees */ + +/* zip64 support 08/29/2003 R.Nausedat */ +/* now all file sizes and offsets are zoff_t 7/24/04 EG */ +local uzoff_t cmpr_bytelen; /* total byte length of compressed file */ +local ulg cmpr_len_bits; /* number of bits past 'cmpr_bytelen' */ + +#ifdef DEBUG +local uzoff_t input_len; /* total byte length of input file */ +/* input_len is for debugging only since we can get it by other means. */ +#endif + +local ush *file_type; /* pointer to UNKNOWN, BINARY or ASCII */ +local int *file_method; /* pointer to DEFLATE or STORE */ + +/* =========================================================================== + * Local data used by the "bit string" routines. + */ + +local int flush_flg; + +#if (!defined(ASMV) || !defined(RISCOS)) +local unsigned bi_buf; +#else +unsigned bi_buf; +#endif +/* Output buffer. bits are inserted starting at the bottom (least significant + * bits). The width of bi_buf must be at least 16 bits. + */ + +#define Buf_size (8 * 2*sizeof(char)) +/* Number of bits used within bi_buf. (bi_buf may be implemented on + * more than 16 bits on some systems.) + */ + +#if (!defined(ASMV) || !defined(RISCOS)) +local int bi_valid; +#else +int bi_valid; +#endif +/* Number of valid bits in bi_buf. All bits above the last valid bit + * are always zero. + */ + +#if (!defined(ASMV) || !defined(RISCOS)) +local char *out_buf; +#else +char *out_buf; +#endif +/* Current output buffer. */ + +#if (!defined(ASMV) || !defined(RISCOS)) +local unsigned out_offset; +#else +unsigned out_offset; +#endif +/* Current offset in output buffer. + * On 16 bit machines, the buffer is limited to 64K. + */ + +#if !defined(ASMV) || !defined(RISCOS) +local unsigned out_size; +#else +unsigned out_size; +#endif +/* Size of current output buffer */ + +/* Output a 16 bit value to the bit stream, lower (oldest) byte first */ +#define PUTSHORT(w) \ +{ if (out_offset >= out_size-1) \ + flush_outbuf(out_buf, &out_offset); \ + out_buf[out_offset++] = (char) ((w) & 0xff); \ + out_buf[out_offset++] = (char) ((ush)(w) >> 8); \ +} + +#define PUTBYTE(b) \ +{ if (out_offset >= out_size) \ + flush_outbuf(out_buf, &out_offset); \ + out_buf[out_offset++] = (char) (b); \ +} + +#ifdef DEBUG +local uzoff_t bits_sent; /* bit length of the compressed data */ +extern uzoff_t isize; /* byte length of input file */ +#endif + +extern long block_start; /* window offset of current block */ +extern unsigned near strstart; /* window offset of current string */ + + +/* =========================================================================== + * Local (static) routines in this file. + */ + +local void init_block OF((void)); +local void pqdownheap OF((ct_data near *tree, int k)); +local void gen_bitlen OF((tree_desc near *desc)); +local void gen_codes OF((ct_data near *tree, int max_code)); +local void build_tree OF((tree_desc near *desc)); +local void scan_tree OF((ct_data near *tree, int max_code)); +local void send_tree OF((ct_data near *tree, int max_code)); +local int build_bl_tree OF((void)); +local void send_all_trees OF((int lcodes, int dcodes, int blcodes)); +local void compress_block OF((ct_data near *ltree, ct_data near *dtree)); +local void set_file_type OF((void)); +#if (!defined(ASMV) || !defined(RISCOS)) +local void send_bits OF((int value, int length)); +local unsigned bi_reverse OF((unsigned code, int len)); +#endif +local void bi_windup OF((void)); +local void copy_block OF((char *buf, unsigned len, int header)); + + +#ifndef DEBUG +# define send_code(c, tree) send_bits(tree[c].Code, tree[c].Len) + /* Send a code of the given tree. c and tree must not have side effects */ + +#else /* DEBUG */ +# define send_code(c, tree) \ + { if (verbose>1) fprintf(mesg,"\ncd %3d ",(c)); \ + send_bits(tree[c].Code, tree[c].Len); } +#endif + +#define d_code(dist) \ + ((dist) < 256 ? dist_code[dist] : dist_code[256+((dist)>>7)]) +/* Mapping from a distance to a distance code. dist is the distance - 1 and + * must not have side effects. dist_code[256] and dist_code[257] are never + * used. + */ + +#define Max(a,b) (a >= b ? a : b) +/* the arguments must not have side effects */ + +/* =========================================================================== + * Allocate the match buffer, initialize the various tables and save the + * location of the internal file attribute (ascii/binary) and method + * (DEFLATE/STORE). + */ +void ct_init(attr, method) + ush *attr; /* pointer to internal file attribute */ + int *method; /* pointer to compression method */ +{ + int n; /* iterates over tree elements */ + int bits; /* bit counter */ + int length; /* length value */ + int code; /* code value */ + int dist; /* distance index */ + + file_type = attr; + file_method = method; + cmpr_len_bits = 0L; + cmpr_bytelen = (uzoff_t)0; +#ifdef DEBUG + input_len = (uzoff_t)0; +#endif + + if (static_dtree[0].Len != 0) return; /* ct_init already called */ + +#ifdef DYN_ALLOC + d_buf = (ush far *) zcalloc(DIST_BUFSIZE, sizeof(ush)); + l_buf = (uch far *) zcalloc(LIT_BUFSIZE/2, 2); + /* Avoid using the value 64K on 16 bit machines */ + if (l_buf == NULL || d_buf == NULL) + ziperr(ZE_MEM, "ct_init: out of memory"); +#endif + + /* Initialize the mapping length (0..255) -> length code (0..28) */ + length = 0; + for (code = 0; code < LENGTH_CODES-1; code++) { + base_length[code] = length; + for (n = 0; n < (1< dist code (0..29) */ + dist = 0; + for (code = 0 ; code < 16; code++) { + base_dist[code] = dist; + for (n = 0; n < (1<>= 7; /* from now on, all distances are divided by 128 */ + for ( ; code < D_CODES; code++) { + base_dist[code] = dist << 7; + for (n = 0; n < (1<<(extra_dbits[code]-7)); n++) { + dist_code[256 + dist++] = (uch)code; + } + } + Assert(dist == 256, "ct_init: 256+dist != 512"); + + /* Construct the codes of the static literal tree */ + for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; + n = 0; + while (n <= 143) static_ltree[n++].Len = 8, bl_count[8]++; + while (n <= 255) static_ltree[n++].Len = 9, bl_count[9]++; + while (n <= 279) static_ltree[n++].Len = 7, bl_count[7]++; + while (n <= 287) static_ltree[n++].Len = 8, bl_count[8]++; + /* Codes 286 and 287 do not exist, but we must include them in the + * tree construction to get a canonical Huffman tree (longest code + * all ones) + */ + gen_codes((ct_data near *)static_ltree, L_CODES+1); + + /* The static distance tree is trivial: */ + for (n = 0; n < D_CODES; n++) { + static_dtree[n].Len = 5; + static_dtree[n].Code = (ush)bi_reverse(n, 5); + } + + /* Initialize the first block of the first file: */ + init_block(); +} + +/* =========================================================================== + * Initialize a new block. + */ +local void init_block() +{ + int n; /* iterates over tree elements */ + + /* Initialize the trees. */ + for (n = 0; n < L_CODES; n++) dyn_ltree[n].Freq = 0; + for (n = 0; n < D_CODES; n++) dyn_dtree[n].Freq = 0; + for (n = 0; n < BL_CODES; n++) bl_tree[n].Freq = 0; + + dyn_ltree[END_BLOCK].Freq = 1; + opt_len = static_len = 0L; + last_lit = last_dist = last_flags = 0; + flags = 0; flag_bit = 1; +} + +#define SMALLEST 1 +/* Index within the heap array of least frequent node in the Huffman tree */ + + +/* =========================================================================== + * Remove the smallest element from the heap and recreate the heap with + * one less element. Updates heap and heap_len. + */ +#define pqremove(tree, top) \ +{\ + top = heap[SMALLEST]; \ + heap[SMALLEST] = heap[heap_len--]; \ + pqdownheap(tree, SMALLEST); \ +} + +/* =========================================================================== + * Compares to subtrees, using the tree depth as tie breaker when + * the subtrees have equal frequency. This minimizes the worst case length. + */ +#define smaller(tree, n, m) \ + (tree[n].Freq < tree[m].Freq || \ + (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m])) + +/* =========================================================================== + * Restore the heap property by moving down the tree starting at node k, + * exchanging a node with the smallest of its two sons if necessary, stopping + * when the heap property is re-established (each father smaller than its + * two sons). + */ +local void pqdownheap(tree, k) + ct_data near *tree; /* the tree to restore */ + int k; /* node to move down */ +{ + int v = heap[k]; + int j = k << 1; /* left son of k */ + int htemp; /* required because of bug in SASC compiler */ + + while (j <= heap_len) { + /* Set j to the smallest of the two sons: */ + if (j < heap_len && smaller(tree, heap[j+1], heap[j])) j++; + + /* Exit if v is smaller than both sons */ + htemp = heap[j]; + if (smaller(tree, v, htemp)) break; + + /* Exchange v with the smallest son */ + heap[k] = htemp; + k = j; + + /* And continue down the tree, setting j to the left son of k */ + j <<= 1; + } + heap[k] = v; +} + +/* =========================================================================== + * Compute the optimal bit lengths for a tree and update the total bit length + * for the current block. + * IN assertion: the fields freq and dad are set, heap[heap_max] and + * above are the tree nodes sorted by increasing frequency. + * OUT assertions: the field len is set to the optimal bit length, the + * array bl_count contains the frequencies for each bit length. + * The length opt_len is updated; static_len is also updated if stree is + * not null. + */ +local void gen_bitlen(desc) + tree_desc near *desc; /* the tree descriptor */ +{ + ct_data near *tree = desc->dyn_tree; + int near *extra = desc->extra_bits; + int base = desc->extra_base; + int max_code = desc->max_code; + int max_length = desc->max_length; + ct_data near *stree = desc->static_tree; + int h; /* heap index */ + int n, m; /* iterate over the tree elements */ + int bits; /* bit length */ + int xbits; /* extra bits */ + ush f; /* frequency */ + int overflow = 0; /* number of elements with bit length too large */ + + for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; + + /* In a first pass, compute the optimal bit lengths (which may + * overflow in the case of the bit length tree). + */ + tree[heap[heap_max]].Len = 0; /* root of the heap */ + + for (h = heap_max+1; h < HEAP_SIZE; h++) { + n = heap[h]; + bits = tree[tree[n].Dad].Len + 1; + if (bits > max_length) bits = max_length, overflow++; + tree[n].Len = (ush)bits; + /* We overwrite tree[n].Dad which is no longer needed */ + + if (n > max_code) continue; /* not a leaf node */ + + bl_count[bits]++; + xbits = 0; + if (n >= base) xbits = extra[n-base]; + f = tree[n].Freq; + opt_len += (ulg)f * (bits + xbits); + if (stree) static_len += (ulg)f * (stree[n].Len + xbits); + } + if (overflow == 0) return; + + Trace((stderr,"\nbit length overflow\n")); + /* This happens for example on obj2 and pic of the Calgary corpus */ + + /* Find the first bit length which could increase: */ + do { + bits = max_length-1; + while (bl_count[bits] == 0) bits--; + bl_count[bits]--; /* move one leaf down the tree */ + bl_count[bits+1] += (ush)2; /* move one overflow item as its brother */ + bl_count[max_length]--; + /* The brother of the overflow item also moves one step up, + * but this does not affect bl_count[max_length] + */ + overflow -= 2; + } while (overflow > 0); + + /* Now recompute all bit lengths, scanning in increasing frequency. + * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all + * lengths instead of fixing only the wrong ones. This idea is taken + * from 'ar' written by Haruhiko Okumura.) + */ + for (bits = max_length; bits != 0; bits--) { + n = bl_count[bits]; + while (n != 0) { + m = heap[--h]; + if (m > max_code) continue; + if (tree[m].Len != (ush)bits) { + Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + opt_len += ((long)bits-(long)tree[m].Len)*(long)tree[m].Freq; + tree[m].Len = (ush)bits; + } + n--; + } + } +} + +/* =========================================================================== + * Generate the codes for a given tree and bit counts (which need not be + * optimal). + * IN assertion: the array bl_count contains the bit length statistics for + * the given tree and the field len is set for all tree elements. + * OUT assertion: the field code is set for all tree elements of non + * zero code length. + */ +local void gen_codes (tree, max_code) + ct_data near *tree; /* the tree to decorate */ + int max_code; /* largest code with non zero frequency */ +{ + ush next_code[MAX_BITS+1]; /* next code value for each bit length */ + ush code = 0; /* running code value */ + int bits; /* bit index */ + int n; /* code index */ + + /* The distribution counts are first used to generate the code values + * without bit reversal. + */ + for (bits = 1; bits <= MAX_BITS; bits++) { + next_code[bits] = code = (ush)((code + bl_count[bits-1]) << 1); + } + /* Check that the bit counts in bl_count are consistent. The last code + * must be all ones. + */ + Assert(code + bl_count[MAX_BITS]-1 == (1<< ((ush) MAX_BITS)) - 1, + "inconsistent bit counts"); + Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); + + for (n = 0; n <= max_code; n++) { + int len = tree[n].Len; + if (len == 0) continue; + /* Now reverse the bits */ + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); + + Tracec(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", + n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); + } +} + +/* =========================================================================== + * Construct one Huffman tree and assigns the code bit strings and lengths. + * Update the total bit length for the current block. + * IN assertion: the field freq is set for all tree elements. + * OUT assertions: the fields len and code are set to the optimal bit length + * and corresponding code. The length opt_len is updated; static_len is + * also updated if stree is not null. The field max_code is set. + */ +local void build_tree(desc) + tree_desc near *desc; /* the tree descriptor */ +{ + ct_data near *tree = desc->dyn_tree; + ct_data near *stree = desc->static_tree; + int elems = desc->elems; + int n, m; /* iterate over heap elements */ + int max_code = -1; /* largest code with non zero frequency */ + int node = elems; /* next internal node of the tree */ + + /* Construct the initial heap, with least frequent element in + * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1]. + * heap[0] is not used. + */ + heap_len = 0, heap_max = HEAP_SIZE; + + for (n = 0; n < elems; n++) { + if (tree[n].Freq != 0) { + heap[++heap_len] = max_code = n; + depth[n] = 0; + } else { + tree[n].Len = 0; + } + } + + /* The pkzip format requires that at least one distance code exists, + * and that at least one bit should be sent even if there is only one + * possible code. So to avoid special checks later on we force at least + * two codes of non zero frequency. + */ + while (heap_len < 2) { + int new = heap[++heap_len] = (max_code < 2 ? ++max_code : 0); + tree[new].Freq = 1; + depth[new] = 0; + opt_len--; if (stree) static_len -= stree[new].Len; + /* new is 0 or 1 so it does not have extra bits */ + } + desc->max_code = max_code; + + /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree, + * establish sub-heaps of increasing lengths: + */ + for (n = heap_len/2; n >= 1; n--) pqdownheap(tree, n); + + /* Construct the Huffman tree by repeatedly combining the least two + * frequent nodes. + */ + do { + pqremove(tree, n); /* n = node of least frequency */ + m = heap[SMALLEST]; /* m = node of next least frequency */ + + heap[--heap_max] = n; /* keep the nodes sorted by frequency */ + heap[--heap_max] = m; + + /* Create a new node father of n and m */ + tree[node].Freq = (ush)(tree[n].Freq + tree[m].Freq); + depth[node] = (uch) (Max(depth[n], depth[m]) + 1); + tree[n].Dad = tree[m].Dad = (ush)node; +#ifdef DUMP_BL_TREE + if (tree == bl_tree) { + fprintf(mesg,"\nnode %d(%d), sons %d(%d) %d(%d)", + node, tree[node].Freq, n, tree[n].Freq, m, tree[m].Freq); + } +#endif + /* and insert the new node in the heap */ + heap[SMALLEST] = node++; + pqdownheap(tree, SMALLEST); + + } while (heap_len >= 2); + + heap[--heap_max] = heap[SMALLEST]; + + /* At this point, the fields freq and dad are set. We can now + * generate the bit lengths. + */ + gen_bitlen((tree_desc near *)desc); + + /* The field len is now set, we can generate the bit codes */ + gen_codes ((ct_data near *)tree, max_code); +} + +/* =========================================================================== + * Scan a literal or distance tree to determine the frequencies of the codes + * in the bit length tree. Updates opt_len to take into account the repeat + * counts. (The contribution of the bit length codes will be added later + * during the construction of bl_tree.) + */ +local void scan_tree (tree, max_code) + ct_data near *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + if (nextlen == 0) max_count = 138, min_count = 3; + tree[max_code+1].Len = (ush)-1; /* guard */ + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + bl_tree[curlen].Freq += (ush)count; + } else if (curlen != 0) { + if (curlen != prevlen) bl_tree[curlen].Freq++; + bl_tree[REP_3_6].Freq++; + } else if (count <= 10) { + bl_tree[REPZ_3_10].Freq++; + } else { + bl_tree[REPZ_11_138].Freq++; + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Send a literal or distance tree in compressed form, using the codes in + * bl_tree. + */ +local void send_tree (tree, max_code) + ct_data near *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + /* tree[max_code+1].Len = -1; */ /* guard already set */ + if (nextlen == 0) max_count = 138, min_count = 3; + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + do { send_code(curlen, bl_tree); } while (--count != 0); + + } else if (curlen != 0) { + if (curlen != prevlen) { + send_code(curlen, bl_tree); count--; + } + Assert(count >= 3 && count <= 6, " 3_6?"); + send_code(REP_3_6, bl_tree); send_bits(count-3, 2); + + } else if (count <= 10) { + send_code(REPZ_3_10, bl_tree); send_bits(count-3, 3); + + } else { + send_code(REPZ_11_138, bl_tree); send_bits(count-11, 7); + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Construct the Huffman tree for the bit lengths and return the index in + * bl_order of the last bit length code to send. + */ +local int build_bl_tree() +{ + int max_blindex; /* index of last bit length code of non zero freq */ + + /* Determine the bit length frequencies for literal and distance trees */ + scan_tree((ct_data near *)dyn_ltree, l_desc.max_code); + scan_tree((ct_data near *)dyn_dtree, d_desc.max_code); + + /* Build the bit length tree: */ + build_tree((tree_desc near *)(&bl_desc)); + /* opt_len now includes the length of the tree representations, except + * the lengths of the bit lengths codes and the 5+5+4 bits for the counts. + */ + + /* Determine the number of bit length codes to send. The pkzip format + * requires that at least 4 bit length codes be sent. (appnote.txt says + * 3 but the actual value used is 4.) + */ + for (max_blindex = BL_CODES-1; max_blindex >= 3; max_blindex--) { + if (bl_tree[bl_order[max_blindex]].Len != 0) break; + } + /* Update opt_len to include the bit length tree and counts */ + opt_len += 3*(max_blindex+1) + 5+5+4; + Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", opt_len, static_len)); + + return max_blindex; +} + +/* =========================================================================== + * Send the header for a block using dynamic Huffman trees: the counts, the + * lengths of the bit length codes, the literal tree and the distance tree. + * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. + */ +local void send_all_trees(lcodes, dcodes, blcodes) + int lcodes, dcodes, blcodes; /* number of codes for each tree */ +{ + int rank; /* index in bl_order */ + + Assert(lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes"); + Assert(lcodes <= L_CODES && dcodes <= D_CODES && blcodes <= BL_CODES, + "too many codes"); + Tracev((stderr, "\nbl counts: ")); + send_bits(lcodes-257, 5); + /* not +255 as stated in appnote.txt 1.93a or -256 in 2.04c */ + send_bits(dcodes-1, 5); + send_bits(blcodes-4, 4); /* not -3 as stated in appnote.txt */ + for (rank = 0; rank < blcodes; rank++) { + Tracev((stderr, "\nbl code %2d ", bl_order[rank])); + send_bits(bl_tree[bl_order[rank]].Len, 3); + } + Tracev((stderr, "\nbl tree: sent %s", + zip_fuzofft(bits_sent, NULL, NULL))); + + send_tree((ct_data near *)dyn_ltree, lcodes-1); /* send the literal tree */ + Tracev((stderr, "\nlit tree: sent %s", + zip_fuzofft(bits_sent, NULL, NULL))); + + send_tree((ct_data near *)dyn_dtree, dcodes-1); /* send the distance tree */ + Tracev((stderr, "\ndist tree: sent %ld", + zip_fuzofft(bits_sent, NULL, NULL))); +} + +/* =========================================================================== + * Determine the best encoding for the current block: dynamic trees, static + * trees or store, and output the encoded block to the zip file. This function + * returns the total compressed length (in bytes) for the file so far. + */ +/* zip64 support 08/29/2003 R.Nausedat */ +uzoff_t flush_block(buf, stored_len, eof) + char *buf; /* input block, or NULL if too old */ + ulg stored_len; /* length of input block */ + int eof; /* true if this is the last block for a file */ +{ + ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ + int max_blindex; /* index of last bit length code of non zero freq */ + + flag_buf[last_flags] = flags; /* Save the flags for the last 8 items */ + + /* Check if the file is ascii or binary */ + if (*file_type == (ush)UNKNOWN) set_file_type(); + + /* Construct the literal and distance trees */ + build_tree((tree_desc near *)(&l_desc)); + Tracev((stderr, "\nlit data: dyn %ld, stat %ld", opt_len, static_len)); + + build_tree((tree_desc near *)(&d_desc)); + Tracev((stderr, "\ndist data: dyn %ld, stat %ld", opt_len, static_len)); + /* At this point, opt_len and static_len are the total bit lengths of + * the compressed block data, excluding the tree representations. + */ + + /* Build the bit length tree for the above two trees, and get the index + * in bl_order of the last bit length code to send. + */ + max_blindex = build_bl_tree(); + + /* Determine the best encoding. Compute first the block length in bytes */ + opt_lenb = (opt_len+3+7)>>3; + static_lenb = (static_len+3+7)>>3; +#ifdef DEBUG + input_len += stored_len; /* for debugging only */ +#endif + + Trace((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u dist %u ", + opt_lenb, opt_len, static_lenb, static_len, stored_len, + last_lit, last_dist)); + + if (static_lenb <= opt_lenb) opt_lenb = static_lenb; + +#ifndef PGP /* PGP can't handle stored blocks */ + /* If compression failed and this is the first and last block, + * the whole file is transformed into a stored file: + */ +#ifdef FORCE_METHOD + if (level == 1 && eof && file_method != NULL && + cmpr_bytelen == (uzoff_t)0 && cmpr_len_bits == 0L + ) { /* force stored file */ +#else + if (stored_len <= opt_lenb && eof && file_method != NULL && + cmpr_bytelen == (uzoff_t)0 && cmpr_len_bits == 0L && + seekable() && !use_descriptors) { +#endif + /* Since LIT_BUFSIZE <= 2*WSIZE, the input data must be there: */ + if (buf == NULL) error ("block vanished"); + + copy_block(buf, (unsigned)stored_len, 0); /* without header */ + cmpr_bytelen = stored_len; + *file_method = STORE; + } else +#endif /* PGP */ + +#ifdef FORCE_METHOD + if (level <= 2 && buf != (char*)NULL) { /* force stored block */ +#else + if (stored_len+4 <= opt_lenb && buf != (char*)NULL) { + /* 4: two words for the lengths */ +#endif + /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE. + * Otherwise we can't have processed more than WSIZE input bytes since + * the last block flush, because compression would have been + * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to + * transform a block into a stored block. + */ + send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */ + cmpr_bytelen += ((cmpr_len_bits + 3 + 7) >> 3) + stored_len + 4; + cmpr_len_bits = 0L; + + copy_block(buf, (unsigned)stored_len, 1); /* with header */ + +#ifdef FORCE_METHOD + } else if (level == 3) { /* force static trees */ +#else + } else if (static_lenb == opt_lenb) { +#endif + send_bits((STATIC_TREES<<1)+eof, 3); + compress_block((ct_data near *)static_ltree, (ct_data near *)static_dtree); + cmpr_len_bits += 3 + static_len; + cmpr_bytelen += cmpr_len_bits >> 3; + cmpr_len_bits &= 7L; + } else { + send_bits((DYN_TREES<<1)+eof, 3); + send_all_trees(l_desc.max_code+1, d_desc.max_code+1, max_blindex+1); + compress_block((ct_data near *)dyn_ltree, (ct_data near *)dyn_dtree); + cmpr_len_bits += 3 + opt_len; + cmpr_bytelen += cmpr_len_bits >> 3; + cmpr_len_bits &= 7L; + } + Assert(((cmpr_bytelen << 3) + cmpr_len_bits) == bits_sent, + "bad compressed size"); + init_block(); + + if (eof) { +#if defined(PGP) && !defined(MMAP) + /* Wipe out sensitive data for pgp */ +# ifdef DYN_ALLOC + extern uch *window; +# else + extern uch window[]; +# endif + memset(window, 0, (unsigned)(2*WSIZE-1)); /* -1 needed if WSIZE=32K */ +#else /* !PGP */ + Assert(input_len == isize, "bad input size"); +#endif + bi_windup(); + cmpr_len_bits += 7; /* align on byte boundary */ + } + Tracev((stderr,"\ncomprlen %s(%s) ", + zip_fuzofft( cmpr_bytelen + (cmpr_len_bits>>3), NULL, NULL), + zip_fuzofft( (cmpr_bytelen << 3) + cmpr_len_bits - 7*eof, NULL, NULL))); + Trace((stderr, "\n")); + + return cmpr_bytelen + (cmpr_len_bits >> 3); +} + +/* =========================================================================== + * Save the match info and tally the frequency counts. Return true if + * the current block must be flushed. + */ +int ct_tally (dist, lc) + int dist; /* distance of matched string */ + int lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ +{ + l_buf[last_lit++] = (uch)lc; + if (dist == 0) { + /* lc is the unmatched char */ + dyn_ltree[lc].Freq++; + } else { + /* Here, lc is the match length - MIN_MATCH */ + dist--; /* dist = match distance - 1 */ + Assert((ush)dist < (ush)MAX_DIST && + (ush)lc <= (ush)(MAX_MATCH-MIN_MATCH) && + (ush)d_code(dist) < (ush)D_CODES, "ct_tally: bad match"); + + dyn_ltree[length_code[lc]+LITERALS+1].Freq++; + dyn_dtree[d_code(dist)].Freq++; + + d_buf[last_dist++] = (ush)dist; + flags |= flag_bit; + } + flag_bit <<= 1; + + /* Output the flags if they fill a byte: */ + if ((last_lit & 7) == 0) { + flag_buf[last_flags++] = flags; + flags = 0, flag_bit = 1; + } + /* Try to guess if it is profitable to stop the current block here */ + if (level > 2 && (last_lit & 0xfff) == 0) { + /* Compute an upper bound for the compressed length */ + ulg out_length = (ulg)last_lit*8L; + ulg in_length = (ulg)strstart-block_start; + int dcode; + for (dcode = 0; dcode < D_CODES; dcode++) { + out_length += (ulg)dyn_dtree[dcode].Freq*(5L+extra_dbits[dcode]); + } + out_length >>= 3; + Trace((stderr,"\nlast_lit %u, last_dist %u, in %ld, out ~%ld(%ld%%) ", + last_lit, last_dist, in_length, out_length, + 100L - out_length*100L/in_length)); + if (last_dist < last_lit/2 && out_length < in_length/2) return 1; + } + return (last_lit == LIT_BUFSIZE-1 || last_dist == DIST_BUFSIZE); + /* We avoid equality with LIT_BUFSIZE because of wraparound at 64K + * on 16 bit machines and because stored blocks are restricted to + * 64K-1 bytes. + */ +} + +/* =========================================================================== + * Send the block data compressed using the given Huffman trees + */ +local void compress_block(ltree, dtree) + ct_data near *ltree; /* literal tree */ + ct_data near *dtree; /* distance tree */ +{ + unsigned dist; /* distance of matched string */ + int lc; /* match length or unmatched char (if dist == 0) */ + unsigned lx = 0; /* running index in l_buf */ + unsigned dx = 0; /* running index in d_buf */ + unsigned fx = 0; /* running index in flag_buf */ + uch flag = 0; /* current flags */ + unsigned code; /* the code to send */ + int extra; /* number of extra bits to send */ + + if (last_lit != 0) do { + if ((lx & 7) == 0) flag = flag_buf[fx++]; + lc = l_buf[lx++]; + if ((flag & 1) == 0) { + send_code(lc, ltree); /* send a literal byte */ + Tracecv(isgraph(lc), (stderr," '%c' ", lc)); + } else { + /* Here, lc is the match length - MIN_MATCH */ + code = length_code[lc]; + send_code(code+LITERALS+1, ltree); /* send the length code */ + extra = extra_lbits[code]; + if (extra != 0) { + lc -= base_length[code]; + send_bits(lc, extra); /* send the extra length bits */ + } + dist = d_buf[dx++]; + /* Here, dist is the match distance - 1 */ + code = d_code(dist); + Assert(code < D_CODES, "bad d_code"); + + send_code(code, dtree); /* send the distance code */ + extra = extra_dbits[code]; + if (extra != 0) { + dist -= base_dist[code]; + send_bits(dist, extra); /* send the extra distance bits */ + } + } /* literal or match pair ? */ + flag >>= 1; + } while (lx < last_lit); + + send_code(END_BLOCK, ltree); +} + +/* =========================================================================== + * Set the file type to TEXT (ASCII) or BINARY, using following algorithm: + * - TEXT, either ASCII or an ASCII-compatible extension such as ISO-8859, + * UTF-8, etc., when the following two conditions are satisfied: + * a) There are no non-portable control characters belonging to the + * "black list" (0..6, 14..25, 28..31). + * b) There is at least one printable character belonging to the + * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). + * - BINARY otherwise. + * + * Note that the following partially-portable control characters form a + * "gray list" that is ignored in this detection algorithm: + * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). + * + * Also note that, unlike in the previous 20% binary detection algorithm, + * any control characters in the black list will set the file type to + * BINARY. If a text file contains a single accidental black character, + * the file will be flagged as BINARY in the archive. + * + * IN assertion: the fields freq of dyn_ltree are set. + */ +local void set_file_type() +{ + /* bit-mask of black-listed bytes + * bit is set if byte is black-listed + * set bits 0..6, 14..25, and 28..31 + * 0xf3ffc07f = binary 11110011111111111100000001111111 + */ + unsigned long mask = 0xf3ffc07fL; + int n; + + /* Check for non-textual ("black-listed") bytes. */ + for (n = 0; n <= 31; n++, mask >>= 1) + if ((mask & 1) && (dyn_ltree[n].Freq != 0)) + { + *file_type = BINARY; + return; + } + + /* Check for textual ("white-listed") bytes. */ + *file_type = ASCII; + if (dyn_ltree[9].Freq != 0 || dyn_ltree[10].Freq != 0 + || dyn_ltree[13].Freq != 0) + return; + for (n = 32; n < LITERALS; n++) + if (dyn_ltree[n].Freq != 0) + return; + + /* This deflate stream is either empty, or + * it has tolerated ("gray-listed") bytes only. + */ + *file_type = BINARY; +} + + +/* =========================================================================== + * Initialize the bit string routines. + */ +void bi_init (tgt_buf, tgt_size, flsh_allowed) + char *tgt_buf; + unsigned tgt_size; + int flsh_allowed; +{ + out_buf = tgt_buf; + out_size = tgt_size; + out_offset = 0; + flush_flg = flsh_allowed; + + bi_buf = 0; + bi_valid = 0; +#ifdef DEBUG + bits_sent = (uzoff_t)0; +#endif +} + +#if (!defined(ASMV) || !defined(RISCOS)) +/* =========================================================================== + * Send a value on a given number of bits. + * IN assertion: length <= 16 and value fits in length bits. + */ +local void send_bits(value, length) + int value; /* value to send */ + int length; /* number of bits */ +{ +#ifdef DEBUG + Tracevv((stderr," l %2d v %4x ", length, value)); + Assert(length > 0 && length <= 15, "invalid length"); + bits_sent += (uzoff_t)length; +#endif + /* If not enough room in bi_buf, use (bi_valid) bits from bi_buf and + * (Buf_size - bi_valid) bits from value to flush the filled bi_buf, + * then fill in the rest of (value), leaving (length - (Buf_size-bi_valid)) + * unused bits in bi_buf. + */ + bi_buf |= (value << bi_valid); + bi_valid += length; + if (bi_valid > (int)Buf_size) { + PUTSHORT(bi_buf); + bi_valid -= Buf_size; + bi_buf = (unsigned)value >> (length - bi_valid); + } +} + +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +local unsigned bi_reverse(code, len) + unsigned code; /* the value to invert */ + int len; /* its bit length */ +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} +#endif /* !ASMV || !RISCOS */ + +/* =========================================================================== + * Write out any remaining bits in an incomplete byte. + */ +local void bi_windup() +{ + if (bi_valid > 8) { + PUTSHORT(bi_buf); + } else if (bi_valid > 0) { + PUTBYTE(bi_buf); + } + if (flush_flg) { + flush_outbuf(out_buf, &out_offset); + } + bi_buf = 0; + bi_valid = 0; +#ifdef DEBUG + bits_sent = (bits_sent+7) & ~7; +#endif +} + +/* =========================================================================== + * Copy a stored block to the zip file, storing first the length and its + * one's complement if requested. + * + * Buffer Overwrite fix + * + * A buffer flush has been added to fix a bug when encrypting deflated files + * with embedded "copied blocks". When encrypting, the flush_out() routine + * modifies its data buffer because encryption is done "in-place" in + * zfwrite(), whereas without encryption, the flush_out() data buffer is + * left unaltered. This can be a problem as noted below by the submitter. + * + * "But an exception comes when a block of stored data (data that could not + * be compressed) is being encrypted. In this case, the data that is passed + * to zfwrite (and is therefore encrypted-in-place) is actually a block of + * data from within the sliding input window that is being managed by + * deflate.c. + * + * "Since part of the sliding input window has now been overwritten by + * encrypted (and essentially random) data, deflate.c's search for previous + * text that matches the current text will usually fail but on rare + * occasions will find a match with something in the encrypted data. This + * incorrect match then causes incorrect information to be placed in the + * ZIP file." + * + * The problem results in the zip file having bad data and so a bad CRC. + * This does not happen often and to recreate the problem a large file + * with non-compressable data is needed so that deflate chooses to store the + * data. A test file of 400 MB seems large enough to recreate the problem + * using a command such as + * zip -1 -e crcerror.zip testfile.dat + * maybe half the time. + * + * This problem has been fixed by copying the data into the deflate output + * buffer before calling flush_outbuf(), when encryption is enabled. + * + * Thanks to the nice people at WinZip for identifying the problem and + * passing it on. Also see Changes. + * + * 2006-03-06 EG, CS + */ +local void copy_block(block, len, header) + char *block; /* the input data */ + unsigned len; /* its length */ + int header; /* true if block header must be written */ +{ + bi_windup(); /* align on byte boundary */ + + if (header) { + PUTSHORT((ush)len); + PUTSHORT((ush)~len); +#ifdef DEBUG + bits_sent += 2*16; +#endif + } + if (flush_flg) { + flush_outbuf(out_buf, &out_offset); + if (key != (char *)NULL) { /* key is the global password pointer */ + /* Encryption modifies the data in the output buffer. But the + * copied input data must remain intact for further deflate + * string matching lookups. Therefore, the input data is + * copied into the compression output buffer for flushing + * to the compressed/encrypted output stream. + */ + while(len > 0) { + out_offset = (len < out_size ? len : out_size); + memcpy(out_buf, block, out_offset); + block += out_offset; + len -= out_offset; + flush_outbuf(out_buf, &out_offset); + } + } else { + /* Without encryption, the output routines do not touch the + * written data, so there is no need for an additional copy + * operation. + */ + out_offset = len; + flush_outbuf(block, &out_offset); + } + } else if (out_offset + len > out_size) { + error("output buffer too small for in-memory compression"); + } else { + memcpy(out_buf + out_offset, block, len); + out_offset += len; + } +#ifdef DEBUG + bits_sent += (ulg)len<<3; +#endif +} + +#endif /* !USE_ZLIB */ diff --git a/third_party/infozip/zip/ttyio.c b/third_party/infozip/zip/ttyio.c new file mode 100644 index 000000000..830bac9d5 --- /dev/null +++ b/third_party/infozip/zip/ttyio.c @@ -0,0 +1,706 @@ +/* clang-format off */ +/* + ttyio.c - Zip 3 + + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/*--------------------------------------------------------------------------- + + ttyio.c + + This file contains routines for doing console input/output, including code + for non-echoing input. It is used by the encryption/decryption code but + does not contain any restricted code itself. This file is shared between + Info-ZIP's Zip and UnZip. + + Contains: echo() (VMS only) + Echon() (Unix only) + Echoff() (Unix only) + screensize() (Unix only) + zgetch() (Unix, VMS, and non-Unix/VMS versions) + getp() ("PC," Unix/Atari/Be, VMS/VMCMS/MVS) + + ---------------------------------------------------------------------------*/ + +#define __TTYIO_C /* identifies this source module */ + +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/crypt.h" + +#if (CRYPT || (defined(UNZIP) && !defined(FUNZIP))) +/* Non-echo console/keyboard input is needed for (en/de)cryption's password + * entry, and for UnZip(SFX)'s MORE and Pause features. + * (The corresponding #endif is found at the end of this module.) + */ + +#include "third_party/infozip/zip/ttyio.h" + +#ifndef PUTC +# define PUTC putc +#endif + +#ifdef ZIP +# ifdef GLOBAL /* used in Amiga system headers, maybe others too */ +# undef GLOBAL +# endif +# define GLOBAL(g) g +#else +# define GLOBAL(g) G.g +#endif + +#if (defined(__ATHEOS__) || defined(__BEOS__)) /* why yes, we do */ +# define HAVE_TERMIOS_H +#endif + +#ifdef _POSIX_VERSION +# ifndef USE_POSIX_TERMIOS +# define USE_POSIX_TERMIOS /* use POSIX style termio (termios) */ +# endif +# ifndef HAVE_TERMIOS_H +# define HAVE_TERMIOS_H /* POSIX termios.h */ +# endif +#endif /* _POSIX_VERSION */ + +#ifdef UNZIP /* Zip handles this with the unix/configure script */ +# ifndef _POSIX_VERSION +# if (defined(SYSV) || defined(CRAY)) && !defined(__MINT__) +# ifndef USE_SYSV_TERMIO +# define USE_SYSV_TERMIO +# endif +# ifdef COHERENT +# ifndef HAVE_TERMIO_H +# define HAVE_TERMIO_H +# endif +# ifdef HAVE_SYS_TERMIO_H +# undef HAVE_SYS_TERMIO_H +# endif +# else /* !COHERENT */ +# ifdef HAVE_TERMIO_H +# undef HAVE_TERMIO_H +# endif +# ifndef HAVE_SYS_TERMIO_H +# define HAVE_SYS_TERMIO_H +# endif +# endif /* ?COHERENT */ +# endif /* (SYSV || CRAY) && !__MINT__ */ +# endif /* !_POSIX_VERSION */ +# if !(defined(BSD4_4) || defined(SYSV) || defined(__convexc__)) +# ifndef NO_FCNTL_H +# define NO_FCNTL_H +# endif +# endif /* !(BSD4_4 || SYSV || __convexc__) */ +#endif /* UNZIP */ + +#ifdef HAVE_TERMIOS_H +# ifndef USE_POSIX_TERMIOS +# define USE_POSIX_TERMIOS +# endif +#endif + +#if (defined(HAVE_TERMIO_H) || defined(HAVE_SYS_TERMIO_H)) +# ifndef USE_SYSV_TERMIO +# define USE_SYSV_TERMIO +# endif +#endif + +#if (defined(UNZIP) && !defined(FUNZIP) && defined(UNIX) && defined(MORE)) +# include +# define GOT_IOCTL_H + /* int ioctl OF((int, int, zvoid *)); GRR: may need for some systems */ +#endif + +#ifndef HAVE_WORKING_GETCH + /* include system support for switching of console echo */ +# ifdef VMS +# include +# include +# include +# include +# include +# else /* !VMS */ +# ifdef HAVE_TERMIOS_H +#include "libc/calls/termios.h" +#include "libc/sysv/consts/termios.h" +# define sgttyb termios +# define sg_flags c_lflag +# define GTTY(f, s) tcgetattr(f, (zvoid *) s) +# define STTY(f, s) tcsetattr(f, TCSAFLUSH, (zvoid *) s) +# else /* !HAVE_TERMIOS_H */ +# ifdef USE_SYSV_TERMIO /* Amdahl, Cray, all SysV? */ +# ifdef HAVE_TERMIO_H +# include +# endif +# ifdef HAVE_SYS_TERMIO_H +# include +# endif +# ifdef NEED_PTEM +# include +# include +# endif +# define sgttyb termio +# define sg_flags c_lflag +# define GTTY(f,s) ioctl(f,TCGETA,(zvoid *)s) +# define STTY(f,s) ioctl(f,TCSETAW,(zvoid *)s) +# else /* !USE_SYSV_TERMIO */ +# ifndef CMS_MVS +# if (!defined(MINIX) && !defined(GOT_IOCTL_H)) +# include +# endif +# include +# define GTTY gtty +# define STTY stty +# ifdef UNZIP + /* + * XXX : Are these declarations needed at all ???? + */ + /* + * GRR: let's find out... Hmmm, appears not... + int gtty OF((int, struct sgttyb *)); + int stty OF((int, struct sgttyb *)); + */ +# endif +# endif /* !CMS_MVS */ +# endif /* ?USE_SYSV_TERMIO */ +# endif /* ?HAVE_TERMIOS_H */ +# ifndef NO_FCNTL_H +# ifndef UNZIP +#include "libc/calls/calls.h" +#include "libc/sysv/consts/at.h" +#include "libc/sysv/consts/f.h" +# endif +# else + char *ttyname OF((int)); +# endif +# endif /* ?VMS */ +#endif /* !HAVE_WORKING_GETCH */ + + + +#ifndef HAVE_WORKING_GETCH +#ifdef VMS + +static struct dsc$descriptor_s DevDesc = + {11, DSC$K_DTYPE_T, DSC$K_CLASS_S, "SYS$COMMAND"}; + /* {dsc$w_length, dsc$b_dtype, dsc$b_class, dsc$a_pointer}; */ + +/* + * Turn keyboard echoing on or off (VMS). Loosely based on VMSmunch.c + * and hence on Joe Meadows' file.c code. + */ +int echo(opt) + int opt; +{ + /* + * For VMS v5.x: + * IO$_SENSEMODE/SETMODE info: Programming, Vol. 7A, System Programming, + * I/O User's: Part I, sec. 8.4.1.1, 8.4.3, 8.4.5, 8.6 + * sys$assign(), sys$qio() info: Programming, Vol. 4B, System Services, + * System Services Reference Manual, pp. sys-23, sys-379 + * fixed-length descriptor info: Programming, Vol. 3, System Services, + * Intro to System Routines, sec. 2.9.2 + * Greg Roelofs, 15 Aug 91 + */ + + short DevChan, iosb[4]; + long status; + unsigned long ttmode[2]; /* space for 8 bytes */ + + + /* assign a channel to standard input */ + status = sys$assign(&DevDesc, &DevChan, 0, 0); + if (!(status & 1)) + return status; + + /* use sys$qio and the IO$_SENSEMODE function to determine the current + * tty status (for password reading, could use IO$_READVBLK function + * instead, but echo on/off will be more general) + */ + status = sys$qiow(0, DevChan, IO$_SENSEMODE, &iosb, 0, 0, + ttmode, 8, 0, 0, 0, 0); + if (!(status & 1)) + return status; + status = iosb[0]; + if (!(status & 1)) + return status; + + /* modify mode buffer to be either NOECHO or ECHO + * (depending on function argument opt) + */ + if (opt == 0) /* off */ + ttmode[1] |= TT$M_NOECHO; /* set NOECHO bit */ + else + ttmode[1] &= ~((unsigned long) TT$M_NOECHO); /* clear NOECHO bit */ + + /* use the IO$_SETMODE function to change the tty status */ + status = sys$qiow(0, DevChan, IO$_SETMODE, &iosb, 0, 0, + ttmode, 8, 0, 0, 0, 0); + if (!(status & 1)) + return status; + status = iosb[0]; + if (!(status & 1)) + return status; + + /* deassign the sys$input channel by way of clean-up */ + status = sys$dassgn(DevChan); + if (!(status & 1)) + return status; + + return SS$_NORMAL; /* we be happy */ + +} /* end function echo() */ + + +/* + * Read a single character from keyboard in non-echoing mode (VMS). + * (returns EOF in case of errors) + */ +int tt_getch() +{ + short DevChan, iosb[4]; + long status; + char kbbuf[16]; /* input buffer with - some - excess length */ + + /* assign a channel to standard input */ + status = sys$assign(&DevDesc, &DevChan, 0, 0); + if (!(status & 1)) + return EOF; + + /* read a single character from SYS$COMMAND (no-echo) and + * wait for completion + */ + status = sys$qiow(0,DevChan, + IO$_READVBLK|IO$M_NOECHO|IO$M_NOFILTR, + &iosb, 0, 0, + &kbbuf, 1, 0, 0, 0, 0); + if ((status&1) == 1) + status = iosb[0]; + + /* deassign the sys$input channel by way of clean-up + * (for this step, we do not need to check the completion status) + */ + sys$dassgn(DevChan); + + /* return the first char read, or EOF in case the read request failed */ + return (int)(((status&1) == 1) ? (uch)kbbuf[0] : EOF); + +} /* end function tt_getch() */ + + +#else /* !VMS: basically Unix */ + + +/* For VM/CMS and MVS, non-echo terminal input is not (yet?) supported. */ +#ifndef CMS_MVS + +#ifdef ZIP /* moved to globals.h for UnZip */ + static int echofd=(-1); /* file descriptor whose echo is off */ +#endif + +/* + * Turn echo off for file descriptor f. Assumes that f is a tty device. + */ +void Echoff(__G__ f) + __GDEF + int f; /* file descriptor for which to turn echo off */ +{ + struct sgttyb sg; /* tty device structure */ + + GLOBAL(echofd) = f; + GTTY(f, &sg); /* get settings */ + sg.sg_flags &= ~ECHO; /* turn echo off */ + STTY(f, &sg); +} + +/* + * Turn echo back on for file descriptor echofd. + */ +void Echon(__G) + __GDEF +{ + struct sgttyb sg; /* tty device structure */ + + if (GLOBAL(echofd) != -1) { + GTTY(GLOBAL(echofd), &sg); /* get settings */ + sg.sg_flags |= ECHO; /* turn echo on */ + STTY(GLOBAL(echofd), &sg); + GLOBAL(echofd) = -1; + } +} + +#endif /* !CMS_MVS */ +#endif /* ?VMS */ + + +#if (defined(UNZIP) && !defined(FUNZIP)) + +#ifdef ATH_BEO_UNX +#ifdef MORE + +/* + * Get the number of lines on the output terminal. SCO Unix apparently + * defines TIOCGWINSZ but doesn't support it (!M_UNIX). + * + * GRR: will need to know width of terminal someday, too, to account for + * line-wrapping. + */ + +#if (defined(TIOCGWINSZ) && !defined(M_UNIX)) + +int screensize(tt_rows, tt_cols) + int *tt_rows; + int *tt_cols; +{ + struct winsize wsz; +#ifdef DEBUG_WINSZ + static int firsttime = TRUE; +#endif + + /* see termio(4) under, e.g., SunOS */ + if (ioctl(1, TIOCGWINSZ, &wsz) == 0) { +#ifdef DEBUG_WINSZ + if (firsttime) { + firsttime = FALSE; + fprintf(stderr, "ttyio.c screensize(): ws_row = %d\n", + wsz.ws_row); + fprintf(stderr, "ttyio.c screensize(): ws_col = %d\n", + wsz.ws_col); + } +#endif + /* number of rows */ + if (tt_rows != NULL) + *tt_rows = (int)((wsz.ws_row > 0) ? wsz.ws_row : 24); + /* number of columns */ + if (tt_cols != NULL) + *tt_cols = (int)((wsz.ws_col > 0) ? wsz.ws_col : 80); + return 0; /* signal success */ + } else { /* this happens when piping to more(1), for example */ +#ifdef DEBUG_WINSZ + if (firsttime) { + firsttime = FALSE; + fprintf(stderr, + "ttyio.c screensize(): ioctl(TIOCGWINSZ) failed\n")); + } +#endif + /* VT-100 assumed to be minimal hardware */ + if (tt_rows != NULL) + *tt_rows = 24; + if (tt_cols != NULL) + *tt_cols = 80; + return 1; /* signal failure */ + } +} + +#else /* !TIOCGWINSZ: service not available, fall back to semi-bogus method */ + +int screensize(tt_rows, tt_cols) + int *tt_rows; + int *tt_cols; +{ + char *envptr, *getenv(); + int n; + int errstat = 0; + + /* GRR: this is overly simplistic, but don't have access to stty/gtty + * system anymore + */ + if (tt_rows != NULL) { + envptr = getenv("LINES"); + if (envptr == (char *)NULL || (n = atoi(envptr)) < 5) { + /* VT-100 assumed to be minimal hardware */ + *tt_rows = 24; + errstat = 1; /* signal failure */ + } else { + *tt_rows = n; + } + } + if (tt_cols != NULL) { + envptr = getenv("COLUMNS"); + if (envptr == (char *)NULL || (n = atoi(envptr)) < 5) { + *tt_cols = 80; + errstat = 1; /* signal failure */ + } else { + *tt_cols = n; + } + } + return errstat; +} + +#endif /* ?(TIOCGWINSZ && !M_UNIX) */ +#endif /* MORE */ + + +/* + * Get a character from the given file descriptor without echo or newline. + */ +int zgetch(__G__ f) + __GDEF + int f; /* file descriptor from which to read */ +{ +#if (defined(USE_SYSV_TERMIO) || defined(USE_POSIX_TERMIOS)) + char oldmin, oldtim; +#endif + char c; + struct sgttyb sg; /* tty device structure */ + + GTTY(f, &sg); /* get settings */ +#if (defined(USE_SYSV_TERMIO) || defined(USE_POSIX_TERMIOS)) + oldmin = sg.c_cc[VMIN]; /* save old values */ + oldtim = sg.c_cc[VTIME]; + sg.c_cc[VMIN] = 1; /* need only one char to return read() */ + sg.c_cc[VTIME] = 0; /* no timeout */ + sg.sg_flags &= ~ICANON; /* canonical mode off */ +#else + sg.sg_flags |= CBREAK; /* cbreak mode on */ +#endif + sg.sg_flags &= ~ECHO; /* turn echo off, too */ + STTY(f, &sg); /* set cbreak mode */ + GLOBAL(echofd) = f; /* in case ^C hit (not perfect: still CBREAK) */ + + read(f, &c, 1); /* read our character */ + +#if (defined(USE_SYSV_TERMIO) || defined(USE_POSIX_TERMIOS)) + sg.c_cc[VMIN] = oldmin; /* restore old values */ + sg.c_cc[VTIME] = oldtim; + sg.sg_flags |= ICANON; /* canonical mode on */ +#else + sg.sg_flags &= ~CBREAK; /* cbreak mode off */ +#endif + sg.sg_flags |= ECHO; /* turn echo on */ + STTY(f, &sg); /* restore canonical mode */ + GLOBAL(echofd) = -1; + + return (int)(uch)c; +} + + +#else /* !ATH_BEO_UNX */ +#ifndef VMS /* VMS supplies its own variant of getch() */ + + +int zgetch(__G__ f) + __GDEF + int f; /* file descriptor from which to read (must be open already) */ +{ + char c, c2; + +/*--------------------------------------------------------------------------- + Get a character from the given file descriptor without echo; can't fake + CBREAK mode (i.e., newline required), but can get rid of all chars up to + and including newline. + ---------------------------------------------------------------------------*/ + + echoff(f); + read(f, &c, 1); + if (c != '\n') + do { + read(f, &c2, 1); /* throw away all other chars up thru newline */ + } while (c2 != '\n'); + echon(); + return (int)c; +} + +#endif /* !VMS */ +#endif /* ?ATH_BEO_UNX */ + +#endif /* UNZIP && !FUNZIP */ +#endif /* !HAVE_WORKING_GETCH */ + + +#if CRYPT /* getp() is only used with full encryption */ + +/* + * Simple compile-time check for source compatibility between + * zcrypt and ttyio: + */ +#if (!defined(CR_MAJORVER) || (CR_MAJORVER < 2) || (CR_MINORVER < 7)) + error: This Info-ZIP tool requires zcrypt 2.7 or later. +#endif + +/* + * Get a password of length n-1 or less into *p using the prompt *m. + * The entered password is not echoed. + */ + +#ifdef HAVE_WORKING_GETCH +/* + * For the AMIGA, getch() is defined as Agetch(), which is in + * amiga/filedate.c; SAS/C 6.x provides a getch(), but since Agetch() + * uses the infrastructure that is already in place in filedate.c, it is + * smaller. With this function, echoff() and echon() are not needed. + * + * For the MAC, a non-echo macgetch() function is defined in the MacOS + * specific sources which uses the event handling mechanism of the + * desktop window manager to get a character from the keyboard. + * + * For the other systems in this section, a non-echo getch() function + * is either contained the C runtime library (conio package), or getch() + * is defined as an alias for a similar system specific RTL function. + */ + +#ifndef WINDLL /* WINDLL does not support a console interface */ +#ifndef QDOS /* QDOS supplies a variant of this function */ + +/* This is the getp() function for all systems (with TTY type user interface) + * that supply a working `non-echo' getch() function for "raw" console input. + */ +char *getp(__G__ m, p, n) + __GDEF + ZCONST char *m; /* prompt for password */ + char *p; /* return value: line input */ + int n; /* bytes available in p[] */ +{ + char c; /* one-byte buffer for read() to use */ + int i; /* number of characters input */ + char *w; /* warning on retry */ + + /* get password */ + w = ""; + do { + fputs(w, stderr); /* warning if back again */ + fputs(m, stderr); /* display prompt and flush */ + fflush(stderr); + i = 0; + do { /* read line, keeping first n characters */ + if ((c = (char)getch()) == '\r') + c = '\n'; /* until user hits CR */ + if (c == 8 || c == 127) { + if (i > 0) i--; /* the `backspace' and `del' keys works */ + } + else if (i < n) + p[i++] = c; /* truncate past n */ + } while (c != '\n'); + PUTC('\n', stderr); fflush(stderr); + w = "(line too long--try again)\n"; + } while (p[i-1] != '\n'); + p[i-1] = 0; /* terminate at newline */ + + return p; /* return pointer to password */ + +} /* end function getp() */ + +#endif /* !QDOS */ +#endif /* !WINDLL */ + + +#else /* !HAVE_WORKING_GETCH */ + + +#if (defined(ATH_BEO_UNX) || defined(__MINT__)) + +#ifndef _PATH_TTY +# ifdef __MINT__ +# define _PATH_TTY ttyname(2) +# else +# define _PATH_TTY "/dev/tty" +# endif +#endif + +char *getp(__G__ m, p, n) + __GDEF + ZCONST char *m; /* prompt for password */ + char *p; /* return value: line input */ + int n; /* bytes available in p[] */ +{ + char c; /* one-byte buffer for read() to use */ + int i; /* number of characters input */ + char *w; /* warning on retry */ + int f; /* file descriptor for tty device */ + +#ifdef PASSWD_FROM_STDIN + /* Read from stdin. This is unsafe if the password is stored on disk. */ + f = 0; +#else + /* turn off echo on tty */ + + if ((f = open(_PATH_TTY, 0)) == -1) + return NULL; +#endif + /* get password */ + w = ""; + do { + fputs(w, stderr); /* warning if back again */ + fputs(m, stderr); /* prompt */ + fflush(stderr); + i = 0; + echoff(f); + do { /* read line, keeping n */ + read(f, &c, 1); + if (i < n) + p[i++] = c; + } while (c != '\n'); + echon(); + PUTC('\n', stderr); fflush(stderr); + w = "(line too long--try again)\n"; + } while (p[i-1] != '\n'); + p[i-1] = 0; /* terminate at newline */ + +#ifndef PASSWD_FROM_STDIN + close(f); +#endif + + return p; /* return pointer to password */ + +} /* end function getp() */ + +#endif /* ATH_BEO_UNX || __MINT__ */ + + + +#if (defined(VMS) || defined(CMS_MVS)) + +char *getp(__G__ m, p, n) + __GDEF + ZCONST char *m; /* prompt for password */ + char *p; /* return value: line input */ + int n; /* bytes available in p[] */ +{ + char c; /* one-byte buffer for read() to use */ + int i; /* number of characters input */ + char *w; /* warning on retry */ + FILE *f; /* file structure for SYS$COMMAND device */ + +#ifdef PASSWD_FROM_STDIN + f = stdin; +#else + if ((f = fopen(ctermid(NULL), "r")) == NULL) + return NULL; +#endif + + /* get password */ + fflush(stdout); + w = ""; + do { + if (*w) /* bug: VMS apparently adds \n to NULL fputs */ + fputs(w, stderr); /* warning if back again */ + fputs(m, stderr); /* prompt */ + fflush(stderr); + i = 0; + echoff(f); + do { /* read line, keeping n */ + if ((c = (char)getc(f)) == '\r') + c = '\n'; + if (i < n) + p[i++] = c; + } while (c != '\n'); + echon(); + PUTC('\n', stderr); fflush(stderr); + w = "(line too long--try again)\n"; + } while (p[i-1] != '\n'); + p[i-1] = 0; /* terminate at newline */ +#ifndef PASSWD_FROM_STDIN + fclose(f); +#endif + + return p; /* return pointer to password */ + +} /* end function getp() */ + +#endif /* VMS || CMS_MVS */ +#endif /* ?HAVE_WORKING_GETCH */ +#endif /* CRYPT */ +#endif /* CRYPT || (UNZIP && !FUNZIP) */ diff --git a/third_party/infozip/zip/ttyio.h b/third_party/infozip/zip/ttyio.h new file mode 100644 index 000000000..4a397292b --- /dev/null +++ b/third_party/infozip/zip/ttyio.h @@ -0,0 +1,230 @@ +/* clang-format off */ +/* + ttyio.h - Zip 3 + + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + ttyio.h + */ + +#ifndef __ttyio_h /* don't include more than once */ +#define __ttyio_h + +#ifndef __crypt_h +# include "third_party/infozip/zip/crypt.h" /* ensure that encryption header file has been seen */ +#endif + +#if (CRYPT || (defined(UNZIP) && !defined(FUNZIP))) +/* + * Non-echo keyboard/console input support is needed and enabled. + */ + +#ifndef __G /* UnZip only, for now (DLL stuff) */ +# define __G +# define __G__ +# define __GDEF +# define __GPRO void +# define __GPRO__ +#endif + +#ifndef ZCONST /* UnZip only (until have configure script like Zip) */ +# define ZCONST const +#endif + +#if (defined(MSDOS) || defined(OS2) || defined(WIN32)) +# ifndef DOS_OS2_W32 +# define DOS_OS2_W32 +# endif +#endif + +#if (defined(DOS_OS2_W32) || defined(__human68k__)) +# ifndef DOS_H68_OS2_W32 +# define DOS_H68_OS2_W32 +# endif +#endif + +#if (defined(DOS_OS2_W32) || defined(FLEXOS)) +# ifndef DOS_FLX_OS2_W32 +# define DOS_FLX_OS2_W32 +# endif +#endif + +#if (defined(DOS_H68_OS2_W32) || defined(FLEXOS)) +# ifndef DOS_FLX_H68_OS2_W32 +# define DOS_FLX_H68_OS2_W32 +# endif +#endif + +#if (defined(__ATHEOS__) || defined(__BEOS__) || defined(UNIX)) +# ifndef ATH_BEO_UNX +# define ATH_BEO_UNX +# endif +#endif + +#if (defined(VM_CMS) || defined(MVS)) +# ifndef CMS_MVS +# define CMS_MVS +# endif +#endif + + +/* Function prototypes */ + +/* The following systems supply a `non-echo' character input function "getch()" + * (or an alias) and do not need the echoff() / echon() function pair. + */ +#ifdef AMIGA +# define echoff(f) +# define echon() +# define getch() Agetch() +# define HAVE_WORKING_GETCH +#endif /* AMIGA */ + +#ifdef ATARI +# define echoff(f) +# define echon() +# include +# define getch() (Cnecin() & 0x000000ff) +# define HAVE_WORKING_GETCH +#endif + +#ifdef MACOS +# define echoff(f) +# define echon() +# define getch() macgetch() +# define HAVE_WORKING_GETCH +#endif + +#ifdef NLM +# define echoff(f) +# define echon() +# define HAVE_WORKING_GETCH +#endif + +#ifdef QDOS +# define echoff(f) +# define echon() +# define HAVE_WORKING_GETCH +#endif + +#ifdef RISCOS +# define echoff(f) +# define echon() +# define getch() SWI_OS_ReadC() +# define HAVE_WORKING_GETCH +#endif + +#ifdef DOS_H68_OS2_W32 +# define echoff(f) +# define echon() +# ifdef WIN32 +# ifndef getch +# define getch() getch_win32() +# endif +# else /* !WIN32 */ +# ifdef __EMX__ +# ifndef getch +# define getch() _read_kbd(0, 1, 0) +# endif +# else /* !__EMX__ */ +# ifdef __GO32__ +# include +# define getch() getkey() +# else /* !__GO32__ */ +# include +# endif /* ?__GO32__ */ +# endif /* ?__EMX__ */ +# endif /* ?WIN32 */ +# define HAVE_WORKING_GETCH +#endif /* DOS_H68_OS2_W32 */ + +#ifdef FLEXOS +# define echoff(f) +# define echon() +# define getch() getchar() /* not correct, but may not be on a console */ +# define HAVE_WORKING_GETCH +#endif + +/* For VM/CMS and MVS, we do not (yet) have any support to switch terminal + * input echo on and off. The following "fake" definitions allow inclusion + * of crypt support and UnZip's "pause prompting" features, but without + * any echo suppression. + */ +#ifdef CMS_MVS +# define echoff(f) +# define echon() +#endif + +#ifdef TANDEM +# define echoff(f) +# define echon() +# define getch() zgetch() /* defined in TANDEMC */ +# define HAVE_WORKING_GETCH +#endif + +/* The THEOS C runtime library supplies the function conmask() to toggle + * terminal input echo on (conmask("e")) and off (conmask("n")). But, + * since THEOS C RTL also contains a working non-echo getch() function, + * the echo toggles are not needed. + */ +#ifdef THEOS +# define echoff(f) +# define echon() +# define HAVE_WORKING_GETCH +#endif + +/* VMS has a single echo() function in ttyio.c to toggle terminal + * input echo on and off. + */ +#ifdef VMS +# define echoff(f) echo(0) +# define echon() echo(1) +# define getch() tt_getch() +# define FGETCH(f) tt_getch() + int echo OF((int)); + int tt_getch OF((void)); +#endif + +/* For all other systems, ttyio.c supplies the two functions Echoff() and + * Echon() for suppressing and (re)enabling console input echo. + */ +#ifndef echoff +# define echoff(f) Echoff(__G__ f) +# define echon() Echon(__G) + void Echoff OF((__GPRO__ int f)); + void Echon OF((__GPRO)); +#endif + +/* this stuff is used by MORE and also now by the ctrl-S code; fileio.c only */ +#if (defined(UNZIP) && !defined(FUNZIP)) +# ifdef HAVE_WORKING_GETCH +# define FGETCH(f) getch() +# endif +# ifndef FGETCH + /* default for all systems where no getch()-like function is available */ + int zgetch OF((__GPRO__ int f)); +# define FGETCH(f) zgetch(__G__ f) +# endif +#endif /* UNZIP && !FUNZIP */ + +#if (CRYPT && !defined(WINDLL)) + char *getp OF((__GPRO__ ZCONST char *m, char *p, int n)); +#endif + +#else /* !(CRYPT || (UNZIP && !FUNZIP)) */ + +/* + * No need for non-echo keyboard/console input; provide dummy definitions. + */ +#define echoff(f) +#define echon() + +#endif /* ?(CRYPT || (UNZIP && !FUNZIP)) */ + +#endif /* !__ttyio_h */ diff --git a/third_party/infozip/zip/unix/Makefile b/third_party/infozip/zip/unix/Makefile new file mode 100644 index 000000000..abd0c445d --- /dev/null +++ b/third_party/infozip/zip/unix/Makefile @@ -0,0 +1,329 @@ +# Makefile for Zip, ZipNote, ZipCloak and ZipSplit + +# what you can make ... +all: + @echo '' + @echo 'Make what? You must say what system to make Zip for--e.g.' + @echo '"make generic".' + @echo 'Choices: generic, generic_gcc, att6300nodir,' + @echo 'coherent, cray_v3, cygwin, lynx, minix, os390,' + @echo 'qnx, qnxnto, solaris, solaris_gcc' + @echo 'Try first "make -f unix/Makefile generic" as' + @echo 'it should autodetect and set the proper flags.' + @echo 'To make the manuals use "make zipsman" after Zip is made.' + @echo 'See the files INSTALL and zip.txt for more information.' + @echo '' + +list: all + +#MAKE = make -f unix/Makefile +MAKEF = -f unix/Makefile +SHELL = /bin/sh +LN = ln -s + +# (to use the GNU compiler, change cc to gcc in CC) +CC = cc +BIND = $(CC) +AS = $(CC) -c +CPP = /lib/cpp +E = + +# probably can change this to 'install' if you have it +INSTALL_PROGRAM = cp +# probably can change this to 'install -d' if you have it +# XXX NextStep 3.3 and Openstep 4.x don't know about -p ! +INSTALL_D = mkdir -p +CHMOD = chmod +BINFLAGS = 755 +MANFLAGS = 644 + +# target directories - where to install executables and man pages to +prefix = /usr/local +BINDIR = $(prefix)/bin +MANEXT=1 +MANDIR = $(prefix)/man/man$(MANEXT) +ZIPMANUAL = zip.txt +ZIPMANUALcloak = zipcloak.txt +ZIPMANUALnote = zipnote.txt +ZIPMANUALsplit = zipsplit.txt +ZIPMANUALs = zip.txt zipcloak.txt zipnote.txt zipsplit.txt +PKGDIR = IZzip +VERSION = Version 3.0 + +# Our bzip2 directory +IZ_OUR_BZIP2_DIR = bzip2 + +# flags +# CFLAGS flags for C compile +# LFLAGS1 flags after output file spec, before obj file list +# LFLAGS2 flags after obj file list (libraries, etc) +CFLAGS_NOOPT = -I. -DUNIX $(LOCAL_ZIP) +CFLAGS = -O2 $(CFLAGS_NOOPT) +LFLAGS1 = +LFLAGS2 = -s + +# object file lists +OBJZ = zip.o zipfile.o zipup.o fileio.o util.o globals.o crypt.o ttyio.o \ + unix.o crc32.o zbz2err.o +OBJI = deflate.o trees.o +OBJA = +OCRCU8 = +OCRCTB = crc32_.o +OBJU = zipfile_.o fileio_.o util_.o globals.o unix_.o $(OCRCU8) +OBJN = zipnote.o $(OBJU) +OBJC = zipcloak.o $(OBJU) $(OCRCTB) crypt_.o ttyio.o +OBJS = zipsplit.o $(OBJU) + +ZIP_H = zip.h ziperr.h tailor.h unix/osdep.h + +# suffix rules +.SUFFIXES: +.SUFFIXES: _.o .o .c .doc .1 +.c_.o: + $(CC) -c $(CFLAGS) -DUTIL -o $@ $< + +.c.o: + $(CC) -c $(CFLAGS) $< + +.1.doc: + nroff -man $< | col -bx | uniq > $@ + +# rules for zip, zipnote, zipcloak, zipsplit, and the Zip MANUALs. +$(OBJZ): $(ZIP_H) +$(OBJI): $(ZIP_H) +$(OBJN): $(ZIP_H) +$(OBJS): $(ZIP_H) +$(OBJC): $(ZIP_H) +zip.o zipup.o zipfile.o fileio.o crc32.o crypt.o: crc32.h +zipcloak.o zipfile_.o fileio_.o crc32_.o crypt_.o: crc32.h +zip.o zipup.o crypt.o ttyio.o zipcloak.o crypt_.o: crypt.h +zip.o zipup.o zipnote.o zipcloak.o zipsplit.o: revision.h +zip.o crypt.o ttyio.o zipcloak.o crypt_.o: ttyio.h +zipup.o: unix/zipup.h + +match.o: match.S + $(CPP) match.S > _match.s + $(AS) _match.s + mv _match.o match.o + rm -f _match.s + +crc_i386.o: crc_i386.S + $(CPP) crc_i386.S > _crc_i386.s + $(AS) _crc_i386.s + mv _crc_i386.o crc_i386.o + rm -f _crc_i386.s + +unix.o: unix/unix.c + $(CC) -c $(CFLAGS) unix/unix.c + +unix_.o: unix/unix.c + $(CC) -c $(CFLAGS) -DUTIL -o $@ unix/unix.c + +ZIPS = zip$E zipcloak$E zipnote$E zipsplit$E + +zips: $(ZIPS) +zipsman: $(ZIPS) $(ZIPMANUALs) + +zip$E: $(OBJZ) $(OBJI) $(OBJA) $(LIB_BZ) + $(BIND) -o zip$E $(LFLAGS1) $(OBJZ) $(OBJI) $(OBJA) $(LFLAGS2) +zipnote$E: $(OBJN) + $(BIND) -o zipnote$E $(LFLAGS1) $(OBJN) $(LFLAGS2) +zipcloak$E: $(OBJC) $(OCRCTB) + $(BIND) -o zipcloak$E $(LFLAGS1) $(OBJC) $(LFLAGS2) +zipsplit$E: $(OBJS) + $(BIND) -o zipsplit$E $(LFLAGS1) $(OBJS) $(LFLAGS2) + +$(ZIPMANUAL): man/zip.1 + nroff -man man/zip.1 | col -bx | uniq > $(ZIPMANUAL) + +$(ZIPMANUALcloak): man/zipcloak.1 + nroff -man man/zipcloak.1 | col -bx | uniq > $(ZIPMANUALcloak) + +$(ZIPMANUALnote): man/zipnote.1 + nroff -man man/zipnote.1 | col -bx | uniq > $(ZIPMANUALnote) + +$(ZIPMANUALsplit): man/zipsplit.1 + nroff -man man/zipsplit.1 | col -bx | uniq > $(ZIPMANUALsplit) + + +# bzip2 object library + +$(IZ_OUR_BZIP2_DIR)/libbz2.a : $(IZ_OUR_BZIP2_DIR)/Makefile + @echo "Building bzip2 object library..." + ( cd $(IZ_OUR_BZIP2_DIR); \ + $(MAKE) CC="$(CC_BZ)" CFLAGS="$(CFLAGS_BZ)" libbz2.a ) + @echo " bzip2 object library created." + + +# install +install: $(ZIPS) + -$(INSTALL_D) $(BINDIR) + $(INSTALL_PROGRAM) $(ZIPS) $(BINDIR) + -cd $(BINDIR); $(CHMOD) $(BINFLAGS) $(ZIPS) + -$(INSTALL_D) $(MANDIR) + $(INSTALL_PROGRAM) man/zip.1 $(MANDIR)/zip.$(MANEXT) + $(CHMOD) $(MANFLAGS) $(MANDIR)/zip.$(MANEXT) + $(INSTALL_PROGRAM) man/zipcloak.1 $(MANDIR)/zipcloak.$(MANEXT) + $(CHMOD) $(MANFLAGS) $(MANDIR)/zipcloak.$(MANEXT) + $(INSTALL_PROGRAM) man/zipnote.1 $(MANDIR)/zipnote.$(MANEXT) + $(CHMOD) $(MANFLAGS) $(MANDIR)/zipnote.$(MANEXT) + $(INSTALL_PROGRAM) man/zipsplit.1 $(MANDIR)/zipsplit.$(MANEXT) + $(CHMOD) $(MANFLAGS) $(MANDIR)/zipsplit.$(MANEXT) + +uninstall: + -cd $(BINDIR); rm -f $(ZIPS) + -cd $(MANDIR); rm -f \ + zip.$(MANEXT) zipcloak.$(MANEXT) zipnote.$(MANEXT) zipsplit.$(MANEXT) + + +flags: unix/configure + sh unix/configure "${CC}" "${CFLAGS_NOOPT}" "${IZ_BZIP2}" + +# These symbols, when #defined using -D have these effects on compilation: +# ZMEM - includes C language versions of memset(), memcpy(), +# and memcmp() (util.c). +# HAVE_DIRENT_H - use instead of +# NODIR - for 3B1, which has neither getdents() nor opendir(). +# HAVE_NDIR_H - use (unix/unix.c). +# HAVE_SYS_DIR_H - use +# HAVE_SYS_NDIR_H - use +# UTIL - select routines for utilities (note, cloak, split) +# NO_RMDIR - remove directories using a system("rmdir ...") call. +# NO_PROTO - cannot handle ANSI prototypes +# NO_CONST - cannot handle ANSI const +# NO_LARGE_FILE_SUPPORT - do not enable Large File support even if available. +# NO_ZIP64_SUPPORT - do not enable Zip64 archive support even if available. +# NO_UNICODE_SUPPORT - do not enable Unicode support even if available. +# NO_BZIP2_SUPPORT - do not compile in bzip2 code even if available. + +# Generic targets: + +generic: flags + eval $(MAKE) $(MAKEF) zips `cat flags` + +generic_gcc: + $(MAKE) $(MAKEF) generic CC=gcc CPP="gcc -E" + +# AT&T 6300 PLUS (don't know yet how to allocate 64K bytes): +att6300nodir: + $(MAKE) $(MAKEF) zips LFLAGS1="-Ml -s" \ + CFLAGS="-DUNIX -I. -O -Ml -DNO_RMDIR -DDYN_ALLOC -DMEDIUM_MEM \ +-DWSIZE=16384 -DNO_STDLIB_H -DNO_STDDEF_H -DNO_RENAME \ +-DNO_MKTIME -DNO_SIZE_T -DNO_VOID -DNO_PROTO -DNO_DIR \ +-DNO_CONST -DHAVE_TERMIO_H" \ + "LFLAGS2=" + +# Coherent (AS definition not needed for gcc) +coherent: + $(MAKE) $(MAKEF) zips CFLAGS="-DUNIX -I. -O -DDIRENT -DASMV" \ + AS="as -gx" OBJA=match.o + +# Cray Unicos 6.1, Standard C compiler 3.0 (all routines except trees.c +# may be compiled with vector3; internal compiler bug in 3.0.2.3 and +# earlier requires vector2 for trees.c) +cray_v3: + $(MAKE) $(MAKEF) zips CC="scc" \ + CFLAGS="-DUNIX -I. -O -h vector2 -h scalar3 -DHAVE_DIRENT_H" + +# Cygwin +cygwin: + $(MAKE) $(MAKEF) generic CC="gcc" CPP="gcc -E" EXE=".exe" + +# LynxOS +lynx: + $(MAKE) $(MAKEF) generic CC=gcc CPP="gcc -E" CFLAGS="$(CFLAGS) \ + -DNO_UNDERLINE -DLynx -DLYNX LFLAGS2="$LFLAGS2 -lc_p" + +# MINIX 1.5.10 with Bruce Evans 386 patches and gcc/GNU make +minix: + $(MAKE) $(MAKEF) zips CFLAGS="-DUNIX -I. -O -DDIRENT -DMINIX" CC=gcc + chmem =262144 zip + +# IBM OS/390 (formerly MVS) compiled under "OpenEdition" shell +# You can make the zip executable with IBM's make, but you will +# get errors dealing with the _.o targets for the other executables +# (zipcloak, etc). GNU make will build all the executables. +# If you have GNU make in your path as gmake, you can uncomment +# the following, but it shouldn't be needed: +#MAKE = gmake + +os390: + $(MAKE) $(MAKEF) zips CFLAGS="$(CF) -I. -DUNIX -DOS390 -DEBCDIC \ + -DSYSV -DNO_PARAM_H" LFLAGS2="" + +# QNX is "special" because out /bin/sh is ksh and it doesn't grok the +# configure script properly, generating a bad flags file. D'oh! [cjh] +# +# QNX/Neutrino is "special" because you don't have any native development +# tools yet. Set ARCH to "x86", "ppcbe", "ppcle", "mipsbe", or "mipsle" +# to produce x86, PowerPC (big- or little-endian) and MIPS (big- +# or little-endian) using gcc. [cjh] +qnx: + $(MAKE) $(MAKEF) zips LN=ln CC=cc CFLAGS="-DUNIX -I. -O \ + -DHAVE_DIRENT_H -DHAVE_TERMIOS_H -DNO_MKTEMP" + +qnxnto: + @if [ "$(ARCH)" = "" ] ; then \ + echo "You didn't set ARCH; I'll assume you meant ARCH=x86..." ; \ + echo "" ; \ + $(MAKE) $(MAKEF) zips LN=ln CC="qcc -Vgcc_ntox86" \ + CFLAGS="-g -DUNIX -I. -O -DHAVE_DIRENT_H -DHAVE_TERMIOS_H -DNO_MKTEMP" \ + LFLAGS2=-g ; \ + else \ + echo "Making zip for $(ARCH)..." ; \ + echo "" ; \ + $(MAKE) $(MAKEF) zips LN=ln CC="qcc -Vgcc_nto$(ARCH)" \ + CFLAGS="-g -DUNIX -I. -O -DHAVE_DIRENT_H -DHAVE_TERMIOS_H -DNO_MKTEMP" \ + LFLAGS2=-g ; \ + fi + +# Solaris: Generic, plus generation of installable package. +solaris: generic svr4package + +# Solaris with GCC: generic_gcc, plus generation of installable package +solaris_gcc: generic_gcc svr4package + +# Package generation interface (by JBush). Originally tested under Sun Solaris. +# Other SVr4s may be very similar, and could possibly use this. +# Note: Expects version info to be stored in VERSION macro variable. +# See "README" under ./unix/Packaging +svr4package: + @echo "Creating SVR4 package for Unix ..." + -@rm -rf ./$(PKGDIR) ./$(PKGDIR)_`uname -p`.pkg + -@sed -e "s/.VERSION./$(VERSION)/g" \ + -e "s/.PSTAMP./$(LOGNAME)_`date | tr ' ' '_'`/g" \ + -e "s/.ARCH./Solaris_`uname -rp | tr ' ' ','`/g" \ + ./unix/Packaging/pkginfo.in > ./unix/Packaging/pkginfo + -@sed -e "s/.ARCH./`uname -p`/g" \ + ./unix/Packaging/preinstall.in > ./unix/Packaging/preinstall + /usr/bin/pkgmk -d . -b . -r . -f ./unix/Packaging/prototype $(PKGDIR) + /usr/bin/pkgtrans -o -s . $(PKGDIR)_`uname -p`.pkg $(PKGDIR) + @echo " " + @echo "To install, copy $(PKGDIR)_`uname -p`.pkg to the target system, and" + @echo "issue the command (as root): pkgadd -d $(PKGDIR)_`uname -p`.pkg" + @echo " " + +# make a distribution +dist: $(ZIPMANUAL) + eval zip -r9 zip`sed -e '/VERSION/!d' -e 's/.*"\(.*\)".*/\1/' \ + -e 's/[.]//g' -e 's/ .*//g' -e q revision.h` * + +# clean up after making stuff and installing it +clean: + rm -f *.o $(ZIPS) flags + rm -rf $(PKGDIR) + +clean_bzip2 : + @if test -f "$(IZ_OUR_BZIP2_DIR)/Makefile"; then \ + ( cd $(IZ_OUR_BZIP2_DIR); make clean ); \ + else \ + if test -z "$(IZ_OUR_BZIP2_DIR)"; then \ + echo "No bzip2 directory (\"IZ_OUR_BZIP2_DIR\") specified."; \ + else \ + echo "No bzip2 make file found: $(IZ_OUR_BZIP2_DIR)/Makefile."; \ + fi; \ + fi + +clean_exe : + rm -f $(ZIPS) +# diff --git a/third_party/infozip/zip/unix/Packaging/README b/third_party/infozip/zip/unix/Packaging/README new file mode 100644 index 000000000..6b2a45172 --- /dev/null +++ b/third_party/infozip/zip/unix/Packaging/README @@ -0,0 +1,44 @@ +Solaris packaging +----------------- + +To generate a Solaris package for Info-ZIP zip utilities, +first see the top level INSTALL and README files. Do a +"make solaris", which will automatically build two Solaris +installable package files for the package, IZzip. + + IZzip -- Solaris installable package in directory format. + IZzip_$(arch).pkg -- Solaris installable package in "stream" format. + + Where: $(arch) := system architecture, currently i386, sparc, or ppc. + (use "uname -p" to determine) + +The ".pkg" file is a single file datastream that can be compressed +and/or ftp'd. This is the recommended form, because all required +files are resident in the archive, and it is easily distributed. + +To install, simply: + + 1) copy the package to the target system's /tmp directory. + 2) login or su to root + 3) pkgadd -d /tmp/IZzip_$(arch).pkg + 4) add /opt/Info-ZIP/IZzip/bin to PATH + 5) add /opt/Info-ZIP/IZzip/man to MANPATH + +This works for both SPARC and x86. + +Ongoing maintenance: + + Keep the files, "prototype" and "pkginfo.in" up to date. + Observe variable substitutions made by "Makefile". + See manpages for pkginfo(1), pkginfo(4), pkgmk(1), pkgproto(1) + +Variations: + + If you wish the base directory to be set to something other than + /opt/Info-ZIP, change the setting BASEDIR in pkginfo.in and + re-run the make. + + +-John Bush (John.Bush@East.Sun.COM) + July 20, 1996 + diff --git a/third_party/infozip/zip/unix/Packaging/pkginfo.in b/third_party/infozip/zip/unix/Packaging/pkginfo.in new file mode 100644 index 000000000..c31395d3d --- /dev/null +++ b/third_party/infozip/zip/unix/Packaging/pkginfo.in @@ -0,0 +1,13 @@ +PKG=IZzip +NAME=Info-ZIP Zip Utilities +CATEGORY=application +VENDOR=Info-ZIP +EMAIL=http://info-zip.org/zip-bug.html +HOTLINE=http://info-zip.org/zip-bug.html +DESC=Copyrighted FREEWARE. See README, WHERE, and docs in pkg's doc dir. +CLASSES=none +BASEDIR=/opt/Info-ZIP +#BASEDIR=/usr/local +VERSION=".VERSION." +PSTAMP=".PSTAMP." +ARCH=".ARCH." diff --git a/third_party/infozip/zip/unix/Packaging/postinstall b/third_party/infozip/zip/unix/Packaging/postinstall new file mode 100644 index 000000000..086ec26ea --- /dev/null +++ b/third_party/infozip/zip/unix/Packaging/postinstall @@ -0,0 +1,29 @@ +#!/bin/sh +# +# Info-ZIP Zip post-installation script. +# +# Last revised: 2007-09-29 SMS. Zip 3.0. +# +# Post installation script (simply inform installer about PATH etc) +# +echo '' +echo 'Installation is complete. Users should adjust their environment' +echo 'variables to include these directories:' +echo " PATH: ${BASEDIR}/${PKG}/bin" +echo " MANPATH: ${BASEDIR}/${PKG}/man" +echo '' +echo "Commands like the following may be added to a user's shell start-up" +echo 'file (.cshrc, .login, .profile, ...) to do this:' +echo '' +echo ' For a Bourne-like shell:' +echo " PATH=\"\${PATH}:${BASEDIR}/${PKG}/bin\"" +echo " MANPATH=\"\${MANPATH}:${BASEDIR}/${PKG}/man\"" +echo ' export PATH MANPATH' +echo '' +echo ' For a C shell:' +echo " setenv PATH \"\${PATH}:${BASEDIR}/${PKG}/bin\"" +echo " setenv MANPATH \"\${MANPATH}:${BASEDIR}/${PKG}/man\"" +echo '' +echo "See the files under ${BASEDIR}/${PKG}/doc for more information." +echo '' +exit 0 diff --git a/third_party/infozip/zip/unix/Packaging/preinstall.in b/third_party/infozip/zip/unix/Packaging/preinstall.in new file mode 100644 index 000000000..de1961b8b --- /dev/null +++ b/third_party/infozip/zip/unix/Packaging/preinstall.in @@ -0,0 +1,29 @@ +#!/bin/sh +# +# Info-ZIP Zip pre-installation script. +# +# Last revised: 2007-09-29 SMS. Zip 3.0. +# +# pkgadd should set a good PATH, but just in case, ... +PATH="/sbin:/usr/bin:${PATH}" +export PATH +echo '' +echo 'Please report problems to Info-ZIP using:' +echo '' +echo ' http://info-zip.org/zip-bug.html' +echo '' +arch=`uname -p` +if [ "arch_${arch}" != "arch_.ARCH." ]; then + echo "This product MUST be installed on a Solaris \".ARCH.\" system." + echo "This system appears to have \"${arch}\" architecture, not \".ARCH.\"." + echo "Please install the version for the \".ARCH.\" architecture." + echo 'Aborting installation...' + returncode=1 +else + echo "Installing on \".ARCH.\" architecture..." + returncode=0 +fi +echo '' +sleep 4 +exit ${returncode:-1} +# diff --git a/third_party/infozip/zip/unix/Packaging/prototype b/third_party/infozip/zip/unix/Packaging/prototype new file mode 100644 index 000000000..002eaf656 --- /dev/null +++ b/third_party/infozip/zip/unix/Packaging/prototype @@ -0,0 +1,29 @@ +d none $BASEDIR 0755 root bin +d none $PKG 0755 root bin +d none $PKG/doc 0755 root bin +f none $PKG/doc/BUGS=BUGS 0644 root bin +f none $PKG/doc/CHANGES=CHANGES 0644 root bin +f none $PKG/doc/INSTALL=INSTALL 0644 root bin +f none $PKG/doc/LICENSE=LICENSE 0644 root bin +f none $PKG/doc/README=README 0644 root bin +f none $PKG/doc/TODO=TODO 0644 root bin +f none $PKG/doc/USexport.msg=USexport.msg 0644 root bin +f none $PKG/doc/WHATSNEW=WHATSNEW 0644 root bin +f none $PKG/doc/WHERE=WHERE 0644 root bin +f none $PKG/doc/zip.txt=zip.txt 0644 root bin +d none $PKG/man 0755 root bin +d none $PKG/man/man1 0755 root bin +f none $PKG/man/man1/zip.1=man/zip.1 0644 root bin +f none $PKG/man/man1/zipcloak.1=man/zipcloak.1 0644 root bin +f none $PKG/man/man1/zipnote.1=man/zipnote.1 0644 root bin +f none $PKG/man/man1/zipsplit.1=man/zipsplit.1 0644 root bin +d none $PKG/bin 0755 root bin +f none $PKG/bin/zip=zip 0755 root bin +f none $PKG/bin/zipcloak=zipcloak 0755 root bin +f none $PKG/bin/zipnote=zipnote 0755 root bin +f none $PKG/bin/zipsplit=zipsplit 0755 root bin +i README +i pkginfo +i prototype +i preinstall +i postinstall diff --git a/third_party/infozip/zip/unix/README.OS390 b/third_party/infozip/zip/unix/README.OS390 new file mode 100644 index 000000000..6fef92b1c --- /dev/null +++ b/third_party/infozip/zip/unix/README.OS390 @@ -0,0 +1,85 @@ + +OS/390 is IBM's follow-on to MVS and includes a POSIX, XOPEN, +XPG4, build environment, a Unix-style filesystem (called HFS), and +a POSIX (Born) shell. This port uses this environment and is a fairly +straight-forward port of ZIP's Unix port - but uses the existing EBCDIC +code. This port does not work with non-HFS (traditional MVS) +filesystems. + +I believe all my changes are isolated with #ifdef's. + +Here's some text which might be useful for an OS390 README or +the manual. + +ZIP for OS390 HFS datasets +-------------------------- +Allows you to create ZIP archives from the OS/390 OpenEdition +command prompt. This port uses standard Unix-style I/O routines +and only works with HFS files. + +Usage +----- +By default, ZIP does not perform character-set translation, but has +options to make it easy to convert text files to be compatible with +other systems + zip zipfile list # add the files in 'list' to archive 'zipfile' + zip -a zipfile list # same as above, but translate files to ASCII + zip -al zipfile list # same as above, translate linefeeds to DOS style + zip -all zipfile list # same as '-a', translate linefeeds to UNIX style + +Build process +------------- +Assuming GNU make is available in your path and is called "gmake" (See +the notes on Makefile changes below) and a C compiler is available as +"cc", then type + gmake -f unix/Makefile MAKE=gmake os390 + +If GNU make is not available, the existing makefile can create zip, but will +error on the other executable (zipsplit, zipcloak, zipnote) if you type + make -f unix/Makefile os390 + +Overview of Changes +------------------- +The OS/390 port is treated as a variant of the Unix port. EBCDIC support +was already implemented for CMS/MVS-batch ports. The specific changes I +made are summarized below. + +unix/Makefile - zip uses a unusual _.o target which IBM's make can't handle. +Since the Makefile has a macro called MAKE that is used for a recursive +call to make, I changed the MACRO to call "gmake" - GNU's make - which +can handle the _.o target. If you don't have GNU make, you can +workaround by manually applying symlinks from whatever.c to whatever_.c. +Alternatively, the whatever_.o files could be explicitely added for os390. + +I added an os390 target with appropriate defines. + +zipup.c - added code (#ifdef OS390) to convert test to ASCII if -a flag +was set. + +zip.c - changed logic which always used DOS-style newlines when -a was +set to be consistent with other port (DOS newlines if -l option) + +zipfile.c - miscellaneous changes to force storing file names and +descriptions in ASCII in the zip directory. This makes zip files +portable across all platforms. This in turn meant names did not +need to be translated when displaying messages. + +zip.h - strtoasc was missing a closing parenthesis. + +ebcdic.h - changed translation table to be consistent with current IBM +recommendations - exact same changes to ebcdic.h as in my unzip port. + +tailor.h - define huge/far/near to be empty + +unix/unix.c - substantial changes to deal with mode flags. Under +the current XOPEN standards, some of the traditional unix file mode +bits need not be in fixed locations, but standard access macros must be +available to access the values. The old unix.c code just picked up these +values and saved them as-is where unzip interpreted them. Existing +Unix system provided the macros for XOPEN compliance, but left the flags +in their traditional locations. OS/390 has a brand new filesystem which +is XOPEN compliant without revealing the positions of these flags. +To create the bitmask in the same format unzip expects, the macros are +tested one-by-one to set the appropriate bits. This same logic should +work on any XOPEN system, but takes more instructions (I did test this +logic on Linux). diff --git a/third_party/infozip/zip/unix/configure b/third_party/infozip/zip/unix/configure new file mode 100644 index 000000000..73ba803ba --- /dev/null +++ b/third_party/infozip/zip/unix/configure @@ -0,0 +1,695 @@ +: +#!/bin/sh -x +# The above : is necessary on some buggy systems. + +# configure: Guess values for system-dependent variables +# Output the flag definitions to the file "flags". +# +# Parameters: $1 = $CC, $2 = $CFLAGS, $3 = $IZ_BZIP2 +# +# This file is typically called from Makefile rather than executed +# from the command line. +# +# To construct zip automatically using this file, type +# "make -f unix/Makefile generic". +# If this fails, then type "make list" to get a list of special targets. + +trap "rm -f conftest* core a.out; exit 1" 1 2 3 15 + +CC=${1-cc} +CFLAGS=${2-"-I. -DUNIX"} +LFLAGS1='' +LFLAGS2='' +LN="ln -s" + +CFLAGS_OPT='' + +# bzip2 +IZ_BZIP2=${3-} +CFLAGS_BZ='' + + +echo 'Check C compiler type (optimization options)' +# Sun C? +cat > conftest.c << _EOF_ +int main() +{ +#ifndef __SUNPRO_C + bad code +#endif + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null +if test $? -eq 0; then + CFLAGS_OPT='-xO3' + echo " Sun C ($CFLAGS_OPT)" +else + # Tru64 DEC/Compaq/HP C? + cat > conftest.c << _EOF_ +int main() +{ +#ifndef __DECC + bad code +#endif + return 0; +} +_EOF_ + $CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null + if test $? -eq 0; then + CFLAGS_OPT='-O3' + echo " DEC C ($CFLAGS_OPT)" + else + # HP-UX HP C? + cat > conftest.c << _EOF_ +int main() +{ +#ifdef __GNUC__ + bad code +#endif +#ifndef __hpux + bad code +#endif + return 0; +} +_EOF_ + $CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null + if test $? -eq 0; then + # HP-UX, not GCC. Lame bundled or real ANSI compiler? + CFLAGS_OPT_TRY="+O3 +Onolimit" + $CC $CFLAGS $CFLAGS_OPT_TRY -c conftest.c 2>&1 | \ + grep '(Bundled)' > /dev/null + if test $? -ne 0; then + CFLAGS_OPT="$CFLAGS_OPT_TRY" + echo " HP-UX ANSI C ($CFLAGS_OPT)" + else + echo ' HP-UX Bundled C (no opt)' + fi + else + # GNU C? + cat > conftest.c << _EOF_ +int main() +{ +#ifndef __GNUC__ + bad code +#endif + return 0; +} +_EOF_ + $CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null + if test $? -eq 0; then + CFLAGS_OPT='-O3' + echo " GNU C ($CFLAGS_OPT)" + # Special Mac OS X shared library "ld" option? + if test ` uname -s 2> /dev/null ` = 'Darwin'; then + lf='-Wl,-search_paths_first' + $CC $CFLAGS $lf conftest.c > /dev/null 2>/dev/null + if test $? -eq 0; then + LFLAGS2="${LFLAGS2} ${lf}" + fi + rm -f conftest + fi + else + CFLAGS_OPT='-O' + echo " Other-unknown C ($CFLAGS_OPT)" + fi + fi + fi +fi + +# optimization flags +if test -n "${CFLAGS_OPT}"; then + CFLAGS="${CFLAGS} ${CFLAGS_OPT}" + CFLAGS_BZ="${CFLAGS_BZ} ${CFLAGS_OPT}" +fi + + +# bzip2 + +echo "Check bzip2 support" +CC_BZ="${CC}" +LIB_BZ='' +if test -n "${IZ_BZIP2}"; then + echo " Check for bzip2 compiled library in IZ_BZIP2 (${IZ_BZIP2})" + if test -f "${IZ_BZIP2}/libbz2.a"; then +# +# A bzip2 library built with BZ_NO_STDIO should have an +# unresolved external, "bz_internal_error". The default, +# full-function library will not mention it. +# + nm ${IZ_BZIP2}/libbz2.a | grep bz_internal_error > /dev/null + if test $? -eq 0; then + echo " Found bzip2 BZ_NO_STDIO library, ${IZ_BZIP2}/libbz2.a" + else + echo " Found bzip2 library, ${IZ_BZIP2}/libbz2.a," + echo " but library not compiled with BZ_NO_STDIO" + echo " WARNING: We recommend using a bzip2 library compiled" + echo " with BZ_NO_STDIO defined for proper error handling" + echo " Please see the Zip installation instructions in bzip2/install.txt" + echo " Continuing anyway with standard bzip2 library..." + fi + if test -f "${IZ_BZIP2}/bzlib.h"; then + CFLAGS="${CFLAGS} -I${IZ_BZIP2} -DBZIP2_SUPPORT" + LFLAGS2="${LFLAGS2} -L${IZ_BZIP2} -lbz2" + echo "-- Found bzip2 library - linking in bzip2" + else + echo " ${IZ_BZIP2}/bzlib.h not found" + echo "-- Since IZ_BZIP2 defined, skipping OS and bzip2 dir checks - no bzip2" + fi + else + echo " ${IZ_BZIP2}/libbz2.a not found" + echo "-- Since IZ_BZIP2 defined, skipping OS and bzip2 checks - no bzip2" + fi +else + echo " Check for bzip2 in bzip2 directory" + IZ_BZIP2=bzip2 + if test -f "${IZ_BZIP2}/libbz2.a"; then + nm ${IZ_BZIP2}/libbz2.a | grep bz_internal_error > /dev/null + if test $? -eq 0; then + echo " Found bzip2 BZ_NO_STDIO library in bzip2 directory" + else + echo " Found bzip2 library in bzip2 directory," + echo " but not built with the BZ_NO_STDIO option" + echo " WARNING: We recommend using a bzip2 library compiled" + echo " with BZ_NO_STDIO defined for proper error handling" + echo " Please see the Zip installation instructions" + echo " Continuing anyway with standard bzip2 library..." + fi + fi + if test -f "bzip2/bzlib.h" -a -f "bzip2/libbz2.a"; then + CFLAGS="${CFLAGS} -I${IZ_BZIP2} -DBZIP2_SUPPORT" + LFLAGS2="${LFLAGS2} -Lbzip2 -lbz2" + echo "-- Found bzip2 library - linking in bzip2" + else + if test -f "bzip2/bzlib.c" -a -f "bzip2/bzlib.h"; then + echo "-- No library, but found bzip2 source in bzip2 directory" + echo "-- Will try to build bzip2 library from source and link in" +# +# Arrange to build a BZ_NO_STDIO bzip2 object library using the +# same compiler and optimization options as used for Zip, and +# to compile and link Zip with bzip2. +# + CFLAGS_BZ="${CFLAGS_BZ} -DBZ_NO_STDIO" + LIB_BZ="bzip2/libbz2.a" + CFLAGS="${CFLAGS} -Ibzip2 -DBZIP2_SUPPORT" + LFLAGS2="${LFLAGS2} -Lbzip2 -lbz2" + else + echo " Check if OS already has bzip2 library installed" + cat > conftest.c << _EOF_ +#include "bzlib.h" +int main() +{ + bz_stream strm; + BZ2_bzCompressEnd(&strm); + return 0; +} +_EOF_ + $CC $CFLAGS -o conftest conftest.c -lbz2 > /dev/null 2>/dev/null + if test $? -eq 0; then + echo "-- OS supports bzip2 - linking in bzip2" + CFLAGS="${CFLAGS} -DBZIP2_SUPPORT" + LFLAGS2="${LFLAGS2} -lbz2" + else + echo "-- Either bzlib.h or libbz2.a not found - no bzip2" + fi + fi + fi +fi + + +echo Check for the C preprocessor +# on SVR4, cc -E does not produce correct assembler files. Need /lib/cpp. +CPP="${CC} -E" +# solaris as(1) needs -P, maybe others as well ? +[ -f /usr/ccs/lib/cpp ] && CPP="/usr/ccs/lib/cpp -P" +[ -f /usr/lib/cpp ] && CPP=/usr/lib/cpp +[ -f /lib/cpp ] && CPP=/lib/cpp +[ -f /usr/bin/cpp ] && CPP=/usr/bin/cpp +[ -f /xenix ] && CPP="${CC} -E" +[ -f /lynx.os ] && CPP="${CC} -E" + +echo "#include " > conftest.c +$CPP conftest.c >/dev/null 2>/dev/null || CPP="${CC} -E" + + +echo Check if we can use asm code +OBJA="" +OCRCU8="" +if eval "$CPP match.S > _match.s 2>/dev/null"; then + if test ! -s _match.s || grep error < _match.s > /dev/null; then + : + elif eval "$CC -c _match.s >/dev/null 2>/dev/null" && [ -f _match.o ]; then + CFLAGS="${CFLAGS} -DASMV" + OBJA="match.o" + echo "int foo() { return 0;}" > conftest.c + $CC -c conftest.c >/dev/null 2>/dev/null + echo Check if compiler generates underlines + nm conftest.o | grep "(^|[^_])foo" >/dev/null 2>/dev/null + [ $? -eq 0 ] && CPP="${CPP} -DNO_UNDERLINE" + if eval "$CPP crc_i386.S > _crc_i386.s 2>/dev/null"; then + if eval "$CC -c _crc_i386.s >/dev/null 2>/dev/null" && [ -f _crc_i386.o ] + then + OBJA="$OBJA crc_i386.o" + OCRCU8="crc_i386.o" + CFLAGS="${CFLAGS} -DASM_CRC" + fi + fi + fi +fi +rm -f _match.s _match.o _crc_i386.s _crc_i386.o + + +# ANSI options for compilers that don't have __STDC__ defined by default +# Currently HPUX, pyramid, Dynix, AIX, OSF/1 and ultrix + +echo Check for ANSI options +cat > conftest.c << _EOF_ +int main() +{ +#ifndef __STDC__ + forget it +#endif + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null +if [ $? -ne 0 ]; then + for OPT in -Ae -Xa -qlanglvl=ansi -std1 -std + do + $CC $CFLAGS $OPT -c conftest.c > /dev/null 2>/dev/null + [ $? -eq 0 ] && CFLAGS="${CFLAGS} ${OPT}" && break + done +fi + + +echo Check for prototypes +echo "int main(int argc, char *argv[]) { return 0; }" > conftest.c +$CC $CFLAGS -c conftest.c > /dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_PROTO" + +# const check currently handles mips cc and non ANSI compilers. +# does it need more ? +echo Check the handling of const +cat > conftest.c << _EOF_ +typedef int charset[2]; +int main() +{ + const charset x; + const char *foo; + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_CONST" + + +echo Check for time_t +cat > conftest.c << _EOF_ +#include +#include +int main() +{ + time_t t; + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_TIME_T" + + +echo Check for size_t +cat > conftest.c << _EOF_ +#include +int main() +{ + size_t s; + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_SIZE_T" + + +echo Check for off_t +cat > conftest.c << _EOF_ +#include +int main() +{ + off_t s; + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_OFF_T" + + +echo Check size of UIDs and GIDs +echo "(Now zip stores variable size UIDs/GIDs using a new extra field. This" +echo " tests if this OS uses 16-bit UIDs/GIDs and so if the old 16-bit storage" +echo " should also be used for backward compatibility.)" +# Added 2008-04-15 CS +cat > conftest.c << _EOF_ +# define _LARGEFILE_SOURCE /* some OSes need this for fseeko */ +# define _LARGEFILE64_SOURCE +# define _FILE_OFFSET_BITS 64 /* select default interface as 64 bit */ +# define _LARGE_FILES /* some OSes need this for 64-bit off_t */ +#include +#include +#include +#include +int main() +{ + struct stat s; + + printf(" s.st_uid is %u bytes\n", sizeof(s.st_uid)); + printf(" s.st_gid is %u bytes\n", sizeof(s.st_gid)); + + /* see if have 16-bit UID */ + if (sizeof(s.st_uid) != 2) { + return 1; + } + /* see if have 16-bit GID */ + if (sizeof(s.st_gid) != 2) { + return 2; + } + return 3; +} +_EOF_ +# compile it +$CC -o conftest conftest.c >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + echo -- UID/GID test failed on compile - disabling old 16-bit UID/GID support + CFLAGS="${CFLAGS} -DUIDGID_NOT_16BIT" +else +# run it + ./conftest + r=$? + if [ $r -eq 1 ]; then + echo -- UID not 2 bytes - disabling old 16-bit UID/GID support + CFLAGS="${CFLAGS} -DUIDGID_NOT_16BIT" + elif [ $r -eq 2 ]; then + echo -- GID not 2 bytes - disabling old 16-bit UID/GID support + CFLAGS="${CFLAGS} -DUIDGID_NOT_16BIT" + elif [ $r -eq 3 ]; then + echo -- 16-bit UIDs and GIDs - keeping old 16-bit UID/GID support + else + echo -- test failed - conftest returned $r - disabling old 16-bit UID/GID support + CFLAGS="${CFLAGS} -DUIDGID_NOT_16BIT" + fi +fi + + +# Now we set the 64-bit file environment and check the size of off_t +# Added 11/4/2003 EG +# Revised 8/12/2004 EG + +echo Check for Large File Support +cat > conftest.c << _EOF_ +# define _LARGEFILE_SOURCE /* some OSes need this for fseeko */ +# define _LARGEFILE64_SOURCE +# define _FILE_OFFSET_BITS 64 /* select default interface as 64 bit */ +# define _LARGE_FILES /* some OSes need this for 64-bit off_t */ +#include +#include +#include +#include +int main() +{ + off_t offset; + struct stat s; + /* see if have 64-bit off_t */ + if (sizeof(offset) < 8) + return 1; + printf(" off_t is %d bytes\n", sizeof(off_t)); + /* see if have 64-bit stat */ + if (sizeof(s.st_size) < 8) { + printf(" s.st_size is %d bytes\n", sizeof(s.st_size)); + return 2; + } + return 3; +} +_EOF_ +# compile it +$CC -o conftest conftest.c >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + echo -- no Large File Support +else +# run it + ./conftest + r=$? + if [ $r -eq 1 ]; then + echo -- no Large File Support - no 64-bit off_t + elif [ $r -eq 2 ]; then + echo -- no Large File Support - no 64-bit stat + elif [ $r -eq 3 ]; then + echo -- yes we have Large File Support! + CFLAGS="${CFLAGS} -DLARGE_FILE_SUPPORT" + else + echo -- no Large File Support - conftest returned $r + fi +fi + + +# Check for wide char for Unicode support +# Added 11/24/2005 EG + +echo Check for wide char support +cat > conftest.c << _EOF_ +#include +#include +int main() +{ + int wsize; + wchar_t *wide_string; + + if ((wide_string = (wchar_t *)malloc(4 * sizeof(wchar_t))) == NULL) { + return 0; + } + /* get wide string */ + wsize = mbstowcs(wide_string, "foo", 3); + wide_string[wsize] = (wchar_t) NULL; + return 1; +} +_EOF_ +# compile it +$CC -o conftest conftest.c >/dev/null 2>/dev/null +# OCRCU8 is used by all utilities if Unicode is enabled +# OCRCTB is only used by zipcloak +if [ $? -ne 0 ]; then + echo -- no Unicode support + OCRCU8="" + OCRCTB="crc32_.o" +else +# have wide char support + echo -- have wchar_t - enabling Unicode support + CFLAGS="${CFLAGS} -DUNICODE_SUPPORT" + OCRCU8="crc32_.o ${OCRCU8}" + OCRCTB="" +fi + + +# from configure 2.4i (Onno) 12/5/04 + +echo Check for gcc no-builtin flag +# -fno-builtin since version 2 +cat > conftest.c << _EOF_ +int main() +{ +#if __GNUC__ >= 2 + return 0; +#else + forget it +#endif +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -eq 0 ] && BFLAG="-fno-builtin" + + +# Check for missing functions +# add NO_'function_name' to flags if missing + +for func in rmdir strchr strrchr rename mktemp mktime mkstemp +do + echo Check for $func + echo "int main(){ $func(); return 0; }" > conftest.c + $CC $BFLAG -o conftest conftest.c >/dev/null 2>/dev/null + [ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_`echo $func | tr '[a-z]' '[A-Z]'`" +done + + +echo Check for memset +echo "int main(){ char k; memset(&k,0,0); return 0; }" > conftest.c +$CC -o conftest conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DZMEM" + + +echo Check for memmove +cat > conftest.c << _EOF_ +#include +int main() { int a; int b = 0; memmove( &a, &b, sizeof( a)); return a; } +_EOF_ +$CC -o conftest conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNEED_MEMMOVE" + + +echo Check for strerror +cat > conftest.c << _EOF_ +#include +int main() { strerror( 0); return 0; } +_EOF_ +$CC -o conftest conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNEED_STRERROR" + +echo Check for errno declaration +cat > conftest.c << _EOF_ +#include +main() +{ + errno = 0; + return 0; +} +_EOF_ +$CC $CFLAGS -c conftest.c >/dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_ERRNO" + + +echo Check for directory libraries +cat > conftest.c << _EOF_ +int main() { return closedir(opendir(".")); } +_EOF_ + +$CC -o conftest conftest.c >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + OPT="" + for lib in ndir dir ucb bsd BSD PW x dirent + do + $CC -o conftest conftest.c -l$lib >/dev/null 2>/dev/null + [ $? -eq 0 ] && OPT=-l$lib && break + done + if [ ${OPT} ]; then + LFLAGS2="${LFLAGS2} ${OPT}" + else + CFLAGS="${CFLAGS} -DNO_DIR" + fi +fi + + +# Dynix/ptx 1.3 needed this + +echo Check for readlink +echo "int main(){ return readlink(); }" > conftest.c +$CC -o conftest conftest.c >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + $CC -o conftest conftest.c -lseq >/dev/null 2>/dev/null + [ $? -eq 0 ] && LFLAGS2="${LFLAGS2} -lseq" +fi + + +echo Check for directory include file +OPT="" +for inc in dirent.h sys/ndir.h ndir.h sys/dir.h +do + echo "#include <$inc>" > conftest.c + $CPP conftest.c > /dev/null 2>/dev/null + [ $? -eq 0 ] && OPT="-DHAVE_`echo $inc | tr '[a-z]./' '[A-Z]__'`" && break +done +CFLAGS="${CFLAGS} ${OPT}" + + +echo Check for nonexistent include files +for inc in stdlib.h stddef.h unistd.h fcntl.h string.h +do + echo "#include <$inc>" > conftest.c + $CPP conftest.c >/dev/null 2>/dev/null + [ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_`echo $inc | tr '[a-z]./' '[A-Z]__'`" +done + + +echo Check for term I/O include file +OPT="" +for inc in termios.h termio.h sgtty.h +do + echo "#include <$inc>" > conftest.c + $CPP conftest.c > /dev/null 2>/dev/null + [ $? -eq 0 ] && OPT="-DHAVE_`echo $inc | tr '[a-z]./' '[A-Z]__'`" && break +done +CFLAGS="${CFLAGS} ${OPT}" + + +# needed for AIX (and others ?) when mmap is used + +echo Check for valloc +cat > conftest.c << _EOF_ +main() +{ +#ifdef MMAP + valloc(); +#endif +} +_EOF_ +$CC ${CFLAGS} -c conftest.c > /dev/null 2>/dev/null +[ $? -ne 0 ] && CFLAGS="${CFLAGS} -DNO_VALLOC" + + +echo Check for /usr/local/bin and /usr/local/man +BINDIR=$HOME/bin +[ -d /usr/local/bin ] && BINDIR=/usr/local/bin + +MANDIR=manl +[ -d /usr/man/manl ] && MANDIR=/usr/man/manl +[ -d /usr/local/man/manl ] && MANDIR=/usr/local/man/manl +[ -d /usr/local/man/man1 ] && MANDIR=/usr/local/man/man1 + + +echo Check for OS-specific flags +if [ -f /usr/bin/hostinfo ]; then + if /usr/bin/hostinfo | grep NeXT > /dev/null; then + CFLAGS="${CFLAGS} -posix" + LFLAGS1="${LFLAGS1} -posix -object" + fi +# XXX ATT6300, Cray +elif [ -f /xenix ]; then + if uname -p | grep 286 > /dev/null; then + CFLAGS="${CFLAGS} -LARGE -Mel2 -DMEDIUM_MEM -DWSIZE=16384 -DNO_VOID" + LFLAGS1="${LFLAGS1} -LARGE -Mel2" + fi +elif uname -X >/dev/null 2>/dev/null; then +# SCO shared library check + echo "int main() { return 0;}" > conftest.c + $CC -o conftest conftest.c -lc_s -nointl >/dev/null 2> /dev/null + [ $? -eq 0 ] && LFLAGS2="-lc_s -nointl" +else + SYSTEM=`uname -s 2>/dev/null` || SYSTEM="unknown" + echo "int main() { return 0;}" > conftest.c + case $SYSTEM in + OSF1|ULTRIX) + echo Check for -Olimit option + $CC ${CFLAGS} -Olimit 1000 -o conftest conftest.c >/dev/null 2>/dev/null + [ $? -eq 0 ] && CFLAGS="${CFLAGS} -Olimit 1000" + ;; +### HP-UX) +### echo Check for +Onolimit option +### $CC ${CFLAGS} +Onolimit -o conftest conftest.c >/dev/null 2>/dev/null +### [ $? -eq 0 ] && CFLAGS="${CFLAGS} +Onolimit" +### ;; +### SunOS) +### CFLAGS="${CFLAGS} -D_FILE_OFFSET_BITS=64" +### ;; + esac +fi + + +echo Check for symbolic links +ln -s /dev/null null > /dev/null 2>/dev/null || LN=ln + + +rm -f a.out conftest.c conftest.o conftest null + +echo CC=\"${CC}\" CFLAGS=\"${CFLAGS}\" CPP=\"${CPP}\" OBJA=\"${OBJA}\" \ + OCRCU8=\"${OCRCU8}\" OCRCTB=\"${OCRCTB}\" \ + BINDIR=${BINDIR} MANDIR=${MANDIR} LFLAGS1=\"${LFLAGS1}\" \ + LFLAGS2=\"${LFLAGS2}\" LN=\"${LN}\" \ + CC_BZ=\"${CC_BZ}\" CFLAGS_BZ=\"${CFLAGS_BZ}\" \ + IZ_BZIP2=\"${IZ_BZIP2}\" LIB_BZ=\"${LIB_BZ}\" > flags + diff --git a/third_party/infozip/zip/unix/osdep.h b/third_party/infozip/zip/unix/osdep.h new file mode 100644 index 000000000..54fc3f3b0 --- /dev/null +++ b/third_party/infozip/zip/unix/osdep.h @@ -0,0 +1,82 @@ +/* clang-format off */ +/* + unix/osdep.h - Zip 3 + + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, both of these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ + +#ifdef NO_LARGE_FILE_SUPPORT +# ifdef LARGE_FILE_SUPPORT +# undef LARGE_FILE_SUPPORT +# endif +#endif + +#ifdef LARGE_FILE_SUPPORT + /* 64-bit Large File Support */ + + /* The following Large File Summit (LFS) defines turn on large file support on + Linux (probably 2.4 or later kernel) and many other unixen */ + +# define _LARGEFILE_SOURCE /* some OSes need this for fseeko */ +# define _LARGEFILE64_SOURCE +# define _FILE_OFFSET_BITS 64 /* select default interface as 64 bit */ +# define _LARGE_FILES /* some OSes need this for 64-bit off_t */ +#endif + +#include "libc/calls/weirdtypes.h" +#include "libc/calls/calls.h" +#include "libc/sysv/consts/s.h" + +/* printf format size prefix for zoff_t values */ +#ifdef LARGE_FILE_SUPPORT +# define ZOFF_T_FORMAT_SIZE_PREFIX "ll" +#else +# define ZOFF_T_FORMAT_SIZE_PREFIX "l" +#endif + +#ifdef NO_OFF_T + typedef long zoff_t; + typedef unsigned long uzoff_t; +#else + typedef off_t zoff_t; +# if defined(LARGE_FILE_SUPPORT) && !(defined(__alpha) && defined(__osf__)) + typedef unsigned long long uzoff_t; +# else + typedef unsigned long uzoff_t; +# endif +#endif + typedef struct stat z_stat; + + +/* Automatically set ZIP64_SUPPORT if LFS */ + +#ifdef LARGE_FILE_SUPPORT +# ifndef NO_ZIP64_SUPPORT +# ifndef ZIP64_SUPPORT +# define ZIP64_SUPPORT +# endif +# else +# ifdef ZIP64_SUPPORT +# undef ZIP64_SUPPORT +# endif +# endif +#endif + + +/* Process files in binary mode */ +#if defined(__DJGPP__) || defined(__CYGWIN__) +# define FOPR "rb" +# define FOPM "r+b" +# define FOPW "wb" +#endif + + +/* Enable the "UT" extra field (time info) */ +#if !defined(NO_EF_UT_TIME) && !defined(USE_EF_UT_TIME) +# define USE_EF_UT_TIME +#endif diff --git a/third_party/infozip/zip/unix/unix.c b/third_party/infozip/zip/unix/unix.c new file mode 100644 index 000000000..61e9a3e35 --- /dev/null +++ b/third_party/infozip/zip/unix/unix.c @@ -0,0 +1,1103 @@ +/* clang-format off */ +/* + unix/unix.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +#include "third_party/infozip/zip/zip.h" + +#ifndef UTIL /* the companion #endif is a bit of ways down ... */ + +#include "libc/time/time.h" +#include "libc/sysv/consts/_posix.h" + +#if defined(MINIX) || defined(__mpexl) +# ifdef S_IWRITE +# undef S_IWRITE +# endif /* S_IWRITE */ +# define S_IWRITE S_IWUSR +#endif /* MINIX */ + +#if (!defined(S_IWRITE) && defined(S_IWUSR)) +# define S_IWRITE S_IWUSR +#endif + +#if defined(HAVE_DIRENT_H) || defined(_POSIX_VERSION) +#include "libc/calls/calls.h" +#include "libc/sysv/consts/dt.h" +#else /* !HAVE_DIRENT_H */ +# ifdef HAVE_NDIR_H +# include +# endif /* HAVE_NDIR_H */ +# ifdef HAVE_SYS_NDIR_H +# include +# endif /* HAVE_SYS_NDIR_H */ +# ifdef HAVE_SYS_DIR_H +# include +# endif /* HAVE_SYS_DIR_H */ +# ifndef dirent +# define dirent direct +# endif +#endif /* HAVE_DIRENT_H || _POSIX_VERSION */ + +#define PAD 0 +#define PATH_END '/' + +/* Library functions not in (most) header files */ + +#if _POSIX_VERSION +# include +#else +/* int utime OF((char *, time_t *)); */ +#endif + +extern char *label; +local ulg label_time = 0; +local ulg label_mode = 0; +local time_t label_utim = 0; + +/* Local functions */ +local char *readd OF((DIR *)); + + +#ifdef NO_DIR /* for AT&T 3B1 */ +#include +#ifndef dirent +# define dirent direct +#endif +typedef FILE DIR; +/* +** Apparently originally by Rich Salz. +** Cleaned up and modified by James W. Birdsall. +*/ + +#define opendir(path) fopen(path, "r") + +struct dirent *readdir(dirp) +DIR *dirp; +{ + static struct dirent entry; + + if (dirp == NULL) + return NULL; + for (;;) + if (fread (&entry, sizeof (struct dirent), 1, dirp) == 0) + return NULL; + else if (entry.d_ino) + return (&entry); +} /* end of readdir() */ + +#define closedir(dirp) fclose(dirp) +#endif /* NO_DIR */ + + +local char *readd(d) +DIR *d; /* directory stream to read from */ +/* Return a pointer to the next name in the directory stream d, or NULL if + no more entries or an error occurs. */ +{ + struct dirent *e; + + e = readdir(d); + return e == NULL ? (char *) NULL : e->d_name; +} + +int procname(n, caseflag) +char *n; /* name to process */ +int caseflag; /* true to force case-sensitive match */ +/* Process a name or sh expression to operate on (or exclude). Return + an error code in the ZE_ class. */ +{ + char *a; /* path and name for recursion */ + DIR *d; /* directory stream from opendir() */ + char *e; /* pointer to name from readd() */ + int m; /* matched flag */ + char *p; /* path for recursion */ + z_stat s; /* result of stat() */ + struct zlist far *z; /* steps through zfiles list */ + + if (strcmp(n, "-") == 0) /* if compressing stdin */ + return newname(n, 0, caseflag); + else if (LSSTAT(n, &s)) + { + /* Not a file or directory--search for shell expression in zip file */ + p = ex2in(n, 0, (int *)NULL); /* shouldn't affect matching chars */ + m = 1; + for (z = zfiles; z != NULL; z = z->nxt) { + if (MATCH(p, z->iname, caseflag)) + { + z->mark = pcount ? filter(z->zname, caseflag) : 1; + if (verbose) + fprintf(mesg, "zip diagnostic: %scluding %s\n", + z->mark ? "in" : "ex", z->name); + m = 0; + } + } + free((zvoid *)p); + return m ? ZE_MISS : ZE_OK; + } + + /* Live name--use if file, recurse if directory */ +#ifdef OS390 + if (S_ISREG(s.st_mode) || S_ISLNK(s.st_mode)) +#else +# ifdef S_IFLNK + if ((s.st_mode & S_IFREG) == S_IFREG || (s.st_mode & S_IFLNK) == S_IFLNK) +# else + if ((s.st_mode & S_IFREG) == S_IFREG) +# endif +#endif + { + /* add or remove name of file */ + if ((m = newname(n, 0, caseflag)) != ZE_OK) + return m; + } +#ifdef OS390 + else if (S_ISDIR(s.st_mode)) +#else + else if ((s.st_mode & S_IFDIR) == S_IFDIR) +#endif + { + /* Add trailing / to the directory name */ + if ((p = malloc(strlen(n)+2)) == NULL) + return ZE_MEM; + if (strcmp(n, ".") == 0) { + *p = '\0'; /* avoid "./" prefix and do not create zip entry */ + } else { + strcpy(p, n); + a = p + strlen(p); + if (a[-1] != '/') + strcpy(a, "/"); + if (dirnames && (m = newname(p, 1, caseflag)) != ZE_OK) { + free((zvoid *)p); + return m; + } + } + /* recurse into directory */ + if (recurse && (d = opendir(n)) != NULL) + { + while ((e = readd(d)) != NULL) { + if (strcmp(e, ".") && strcmp(e, "..")) + { + if ((a = malloc(strlen(p) + strlen(e) + 1)) == NULL) + { + closedir(d); + free((zvoid *)p); + return ZE_MEM; + } + strcat(strcpy(a, p), e); + if ((m = procname(a, caseflag)) != ZE_OK) /* recurse on name */ + { + if (m == ZE_MISS) + zipwarn("name not matched: ", a); + else + ziperr(m, a); + } + free((zvoid *)a); + } + } + closedir(d); + } + free((zvoid *)p); + } /* (s.st_mode & S_IFDIR) */ +#ifdef OS390 + else if (S_ISFIFO(s.st_mode)) +#else + else if ((s.st_mode & S_IFIFO) == S_IFIFO) +#endif + { + if (allow_fifo) { + /* FIFO (Named Pipe) - handle as normal file */ + /* add or remove name of FIFO */ + /* zip will stop if FIFO is open and wait for pipe to be fed and closed */ + if (noisy) zipwarn("Reading FIFO (Named Pipe): ", n); + if ((m = newname(n, 0, caseflag)) != ZE_OK) + return m; + } else { + zipwarn("ignoring FIFO (Named Pipe) - use -FI to read: ", n); + return ZE_OK; + } + } /* S_IFIFO */ + else + zipwarn("ignoring special file: ", n); + return ZE_OK; +} + +char *ex2in(x, isdir, pdosflag) +char *x; /* external file name */ +int isdir; /* input: x is a directory */ +int *pdosflag; /* output: force MSDOS file attributes? */ +/* Convert the external file name to a zip file name, returning the malloc'ed + string or NULL if not enough memory. */ +{ + char *n; /* internal file name (malloc'ed) */ + char *t = NULL; /* shortened name */ + int dosflag; + + dosflag = dosify; /* default for non-DOS and non-OS/2 */ + + /* Find starting point in name before doing malloc */ + /* Strip "//host/share/" part of a UNC name */ + if (!strncmp(x,"//",2) && (x[2] != '\0' && x[2] != '/')) { + n = x + 2; + while (*n != '\0' && *n != '/') + n++; /* strip host name */ + if (*n != '\0') { + n++; + while (*n != '\0' && *n != '/') + n++; /* strip `share' name */ + } + if (*n != '\0') + t = n + 1; + } else + t = x; + while (*t == '/') + t++; /* strip leading '/' chars to get a relative path */ + while (*t == '.' && t[1] == '/') + t += 2; /* strip redundant leading "./" sections */ + + /* Make changes, if any, to the copied name (leave original intact) */ + if (!pathput) + t = last(t, PATH_END); + + /* Malloc space for internal name and copy it */ + if ((n = malloc(strlen(t) + 1)) == NULL) + return NULL; + strcpy(n, t); + + if (dosify) + msname(n); + +#ifdef EBCDIC + strtoasc(n, n); /* here because msname() needs native coding */ +#endif + + /* Returned malloc'ed name */ + if (pdosflag) + *pdosflag = dosflag; + + if (isdir) return n; /* avoid warning on unused variable */ + return n; +} + +char *in2ex(n) +char *n; /* internal file name */ +/* Convert the zip file name to an external file name, returning the malloc'ed + string or NULL if not enough memory. */ +{ + char *x; /* external file name */ + + if ((x = malloc(strlen(n) + 1 + PAD)) == NULL) + return NULL; +#ifdef EBCDIC + strtoebc(x, n); +#else + strcpy(x, n); +#endif + return x; +} + +/* + * XXX use ztimbuf in both POSIX and non POSIX cases ? + */ +void stamp(f, d) +char *f; /* name of file to change */ +ulg d; /* dos-style time to change it to */ +/* Set last updated and accessed time of file f to the DOS time d. */ +{ +#ifdef _POSIX_VERSION + struct utimbuf u; /* argument for utime() const ?? */ +#else + time_t u[2]; /* argument for utime() */ +#endif + + /* Convert DOS time to time_t format in u */ +#ifdef _POSIX_VERSION + u.actime = u.modtime = dos2unixtime(d); + utime(f, &u); +#else + u[0] = u[1] = dos2unixtime(d); + utime(f, u); +#endif + +} + +ulg filetime(f, a, n, t) + char *f; /* name of file to get info on */ + ulg *a; /* return value: file attributes */ + zoff_t *n; /* return value: file size */ + iztimes *t; /* return value: access, modific. and creation times */ +/* If file *f does not exist, return 0. Else, return the file's last + modified date and time as an MSDOS date and time. The date and + time is returned in a long with the date most significant to allow + unsigned integer comparison of absolute times. Also, if a is not + a NULL pointer, store the file attributes there, with the high two + bytes being the Unix attributes, and the low byte being a mapping + of that to DOS attributes. If n is not NULL, store the file size + there. If t is not NULL, the file's access, modification and creation + times are stored there as UNIX time_t values. + If f is "-", use standard input as the file. If f is a device, return + a file size of -1 */ +{ + z_stat s; /* results of stat() */ + /* converted to pointer from using FNMAX - 11/8/04 EG */ + char *name; + int len = strlen(f); + + if (f == label) { + if (a != NULL) + *a = label_mode; + if (n != NULL) + *n = -2L; /* convention for a label name */ + if (t != NULL) + t->atime = t->mtime = t->ctime = label_utim; + return label_time; + } + if ((name = malloc(len + 1)) == NULL) { + ZIPERR(ZE_MEM, "filetime"); + } + strcpy(name, f); + if (name[len - 1] == '/') + name[len - 1] = '\0'; + /* not all systems allow stat'ing a file with / appended */ + if (strcmp(f, "-") == 0) { + if (zfstat(fileno(stdin), &s) != 0) { + free(name); + error("fstat(stdin)"); + } + } + else if (LSSTAT(name, &s) != 0) { + /* Accept about any file kind including directories + * (stored with trailing / with -r option) + */ + free(name); + return 0; + } + free(name); + + if (a != NULL) { +#ifndef OS390 + *a = ((ulg)s.st_mode << 16) | !(s.st_mode & S_IWRITE); +#else +/* +** The following defines are copied from the unizip source and represent the +** legacy Unix mode flags. These fixed bit masks are no longer required +** by XOPEN standards - the S_IS### macros being the new recommended method. +** The approach here of setting the legacy flags by testing the macros should +** work under any _XOPEN_SOURCE environment (and will just rebuild the same bit +** mask), but is required if the legacy bit flags differ from legacy Unix. +*/ +#define UNX_IFDIR 0040000 /* Unix directory */ +#define UNX_IFREG 0100000 /* Unix regular file */ +#define UNX_IFSOCK 0140000 /* Unix socket (BSD, not SysV or Amiga) */ +#define UNX_IFLNK 0120000 /* Unix symbolic link (not SysV, Amiga) */ +#define UNX_IFBLK 0060000 /* Unix block special (not Amiga) */ +#define UNX_IFCHR 0020000 /* Unix character special (not Amiga) */ +#define UNX_IFIFO 0010000 /* Unix fifo (BCC, not MSC or Amiga) */ + { + mode_t legacy_modes; + + /* Initialize with permission bits--which are not implementation-optional */ + legacy_modes = s.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX); + if (S_ISDIR(s.st_mode)) + legacy_modes |= UNX_IFDIR; + if (S_ISREG(s.st_mode)) + legacy_modes |= UNX_IFREG; + if (S_ISLNK(s.st_mode)) + legacy_modes |= UNX_IFLNK; + if (S_ISBLK(s.st_mode)) + legacy_modes |= UNX_IFBLK; + if (S_ISCHR(s.st_mode)) + legacy_modes |= UNX_IFCHR; + if (S_ISFIFO(s.st_mode)) + legacy_modes |= UNX_IFIFO; + if (S_ISSOCK(s.st_mode)) + legacy_modes |= UNX_IFSOCK; + *a = ((ulg)legacy_modes << 16) | !(s.st_mode & S_IWRITE); + } +#endif + if ((s.st_mode & S_IFMT) == S_IFDIR) { + *a |= MSDOS_DIR_ATTR; + } + } + if (n != NULL) + *n = (s.st_mode & S_IFMT) == S_IFREG ? s.st_size : -1L; + if (t != NULL) { + t->atime = s.st_atime; + t->mtime = s.st_mtime; + t->ctime = t->mtime; /* best guess, (s.st_ctime: last status change!!) */ + } + return unix2dostime(&s.st_mtime); +} + + +#ifndef QLZIP /* QLZIP Unix2QDOS cross-Zip supplies an extended variant */ + +int set_new_unix_extra_field(z, s) + struct zlist far *z; + z_stat *s; + /* New unix extra field. + Currently only UIDs and GIDs are stored. */ +{ + int uid_size; + int gid_size; + int ef_data_size; + char *extra; + char *cextra; + ulg id; + int b; + + uid_size = sizeof(s->st_uid); + gid_size = sizeof(s->st_gid); + +/* New extra field + tag (2 bytes) + size (2 bytes) + version (1 byte) + uid_size (1 byte - size in bytes) + uid (variable) + gid_size (1 byte - size in bytes) + gid (variable) + */ + + ef_data_size = 1 + 1 + uid_size + 1 + gid_size; + + if ((extra = (char *)malloc(z->ext + 4 + ef_data_size)) == NULL) + return ZE_MEM; + if ((cextra = (char *)malloc(z->ext + 4 + ef_data_size)) == NULL) + return ZE_MEM; + + if (z->ext) + memcpy(extra, z->extra, z->ext); + if (z->cext) + memcpy(cextra, z->cextra, z->cext); + + free(z->extra); + z->extra = extra; + free(z->cextra); + z->cextra = cextra; + + z->extra[z->ext + 0] = 'u'; + z->extra[z->ext + 1] = 'x'; + z->extra[z->ext + 2] = (char)ef_data_size; /* length of data part */ + z->extra[z->ext + 3] = 0; + z->extra[z->ext + 4] = 1; /* version */ + + /* UID */ + z->extra[z->ext + 5] = (char)(uid_size); /* uid size in bytes */ + b = 6; + id = (ulg)(s->st_uid); + z->extra[z->ext + b] = (char)(id & 0xFF); + if (uid_size > 1) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + if (uid_size > 2) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + if (uid_size == 8) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + } + } + } + + /* GID */ + b++; + z->extra[z->ext + b] = (char)(gid_size); /* gid size in bytes */ + b++; + id = (ulg)(s->st_gid); + z->extra[z->ext + b] = (char)(id & 0xFF); + if (gid_size > 1) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + if (gid_size > 2) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + if (gid_size == 8) { + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + b++; + id = id >> 8; + z->extra[z->ext + b] = (char)(id & 0xFF); + } + } + } + + /* copy local extra field to central directory extra field */ + memcpy((z->cextra) + z->cext, (z->extra) + z->ext, 4 + ef_data_size); + + z->ext = z->ext + 4 + ef_data_size; + z->cext = z->cext + 4 + ef_data_size; + + return ZE_OK; +} + + +int set_extra_field(z, z_utim) + struct zlist far *z; + iztimes *z_utim; + /* store full data in local header but just modification time stamp info + in central header */ +{ + z_stat s; + char *name; + int len = strlen(z->name); + + /* For the full sized UT local field including the UID/GID fields, we + * have to stat the file again. */ + + if ((name = malloc(len + 1)) == NULL) { + ZIPERR(ZE_MEM, "set_extra_field"); + } + strcpy(name, z->name); + if (name[len - 1] == '/') + name[len - 1] = '\0'; + /* not all systems allow stat'ing a file with / appended */ + if (LSSTAT(name, &s)) { + free(name); + return ZE_OPEN; + } + free(name); + +#define EB_L_UT_SIZE (EB_HEADSIZE + EB_UT_LEN(2)) +#define EB_C_UT_SIZE (EB_HEADSIZE + EB_UT_LEN(1)) + +/* The flag UIDGID_NOT_16BIT should be set by the pre-compile configuration + script when it detects st_uid or st_gid sizes differing from 16-bit. + */ +#ifndef UIDGID_NOT_16BIT + /* The following "second-level" check for st_uid and st_gid members being + 16-bit wide is only added as a safety precaution in case the "first-level" + check failed to define the UIDGID_NOT_16BIT symbol. + The first-level check should have been implemented in the automatic + compile configuration process. + */ +# ifdef UIDGID_ARE_16B +# undef UIDGID_ARE_16B +# endif + /* The following expression is a compile-time constant and should (hopefully) + get optimized away by any sufficiently intelligent compiler! + */ +# define UIDGID_ARE_16B (sizeof(s.st_uid) == 2 && sizeof(s.st_gid) == 2) + +# define EB_L_UX2_SIZE (EB_HEADSIZE + EB_UX2_MINLEN) +# define EB_C_UX2_SIZE EB_HEADSIZE +# define EF_L_UNIX_SIZE (EB_L_UT_SIZE + (UIDGID_ARE_16B ? EB_L_UX2_SIZE : 0)) +# define EF_C_UNIX_SIZE (EB_C_UT_SIZE + (UIDGID_ARE_16B ? EB_C_UX2_SIZE : 0)) +#else +# define EF_L_UNIX_SIZE EB_L_UT_SIZE +# define EF_C_UNIX_SIZE EB_C_UT_SIZE +#endif /* !UIDGID_NOT_16BIT */ + + if ((z->extra = (char *)malloc(EF_L_UNIX_SIZE)) == NULL) + return ZE_MEM; + if ((z->cextra = (char *)malloc(EF_C_UNIX_SIZE)) == NULL) + return ZE_MEM; + + z->extra[0] = 'U'; + z->extra[1] = 'T'; + z->extra[2] = (char)EB_UT_LEN(2); /* length of data part of local e.f. */ + z->extra[3] = 0; + z->extra[4] = EB_UT_FL_MTIME | EB_UT_FL_ATIME; /* st_ctime != creation */ + z->extra[5] = (char)(s.st_mtime); + z->extra[6] = (char)(s.st_mtime >> 8); + z->extra[7] = (char)(s.st_mtime >> 16); + z->extra[8] = (char)(s.st_mtime >> 24); + z->extra[9] = (char)(s.st_atime); + z->extra[10] = (char)(s.st_atime >> 8); + z->extra[11] = (char)(s.st_atime >> 16); + z->extra[12] = (char)(s.st_atime >> 24); + +#ifndef UIDGID_NOT_16BIT + /* Only store the UID and GID in the old Ux extra field if the runtime + system provides them in 16-bit wide variables. */ + if (UIDGID_ARE_16B) { + z->extra[13] = 'U'; + z->extra[14] = 'x'; + z->extra[15] = (char)EB_UX2_MINLEN; /* length of data part of local e.f. */ + z->extra[16] = 0; + z->extra[17] = (char)(s.st_uid); + z->extra[18] = (char)(s.st_uid >> 8); + z->extra[19] = (char)(s.st_gid); + z->extra[20] = (char)(s.st_gid >> 8); + } +#endif /* !UIDGID_NOT_16BIT */ + + z->ext = EF_L_UNIX_SIZE; + + memcpy(z->cextra, z->extra, EB_C_UT_SIZE); + z->cextra[EB_LEN] = (char)EB_UT_LEN(1); +#ifndef UIDGID_NOT_16BIT + if (UIDGID_ARE_16B) { + /* Copy header of Ux extra field from local to central */ + memcpy(z->cextra+EB_C_UT_SIZE, z->extra+EB_L_UT_SIZE, EB_C_UX2_SIZE); + z->cextra[EB_LEN+EB_C_UT_SIZE] = 0; + } +#endif + z->cext = EF_C_UNIX_SIZE; + +#if 0 /* UID/GID presence is now signaled by central EF_IZUNIX2 field ! */ + /* lower-middle external-attribute byte (unused until now): + * high bit => (have GMT mod/acc times) >>> NO LONGER USED! <<< + * second-high bit => have Unix UID/GID info + * NOTE: The high bit was NEVER used in any official Info-ZIP release, + * but its future use should be avoided (if possible), since it + * was used as "GMT mod/acc times local extra field" flags in Zip beta + * versions 2.0j up to 2.0v, for about 1.5 years. + */ + z->atx |= 0x4000; +#endif /* never */ + + /* new unix extra field */ + set_new_unix_extra_field(z, &s); + + return ZE_OK; +} + +#endif /* !QLZIP */ + + +int deletedir(d) +char *d; /* directory to delete */ +/* Delete the directory *d if it is empty, do nothing otherwise. + Return the result of rmdir(), delete(), or system(). + For VMS, d must be in format [x.y]z.dir;1 (not [x.y.z]). + */ +{ +# ifdef NO_RMDIR + /* code from Greg Roelofs, who horked it from Mark Edwards (unzip) */ + int r, len; + char *s; /* malloc'd string for system command */ + + len = strlen(d); + if ((s = malloc(len + 34)) == NULL) + return 127; + + sprintf(s, "IFS=\" \t\n\" /bin/rmdir %s 2>/dev/null", d); + r = system(s); + free(s); + return r; +# else /* !NO_RMDIR */ + return rmdir(d); +# endif /* ?NO_RMDIR */ +} + +#endif /* !UTIL */ + + +/******************************/ +/* Function version_local() */ +/******************************/ + +#if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__386BSD__) || \ + defined(__OpenBSD__) || defined(__bsdi__) +#include /* for the BSD define */ +/* if we have something newer than NET/2 we'll use uname(3) */ +#if (BSD > 199103) +#include "libc/calls/calls.h" +#endif /* BSD > 199103 */ +#endif /* __{Net,Free,Open,386}BSD__ || __bsdi__ */ + +void version_local() +{ +#ifdef __GNUC__ +# ifdef NX_CURRENT_COMPILER_RELEASE + char compiler_name[80]; +# endif +#else +# if (defined( __SUNPRO_C)) + char compiler_name[33]; +# else +# if (defined( __HP_cc) || defined( __IBMC__)) + char compiler_name[33]; +# else +# if (defined( __DECC_VER)) + char compiler_name[33]; + int compiler_typ; +# else +# if ((defined(CRAY) || defined(cray)) && defined(_RELEASE)) + char compiler_name[40]; +# endif +# endif +# endif +# endif +#endif + +#ifdef BSD +# if (BSD > 199103) + struct utsname u; + char os_name[40]; +# else +# if defined(__NETBSD__)) + static ZCONST char *netbsd[] = { "_ALPHA", "", "A", "B" }; + char os_name[40]; +# endif /* __NETBSD__ */ +# endif /* BSD > 199103 */ +#else /* !BSD */ +#if ((defined(CRAY) || defined(cray)) && defined(_UNICOS)) + char os_name[40]; +#endif /* (CRAY && defined(_UNICOS)) */ +#endif /* ?BSD */ + +/* Define the compiler name and version string */ +#ifdef __GNUC__ +# ifdef NX_CURRENT_COMPILER_RELEASE + sprintf(compiler_name, "NeXT DevKit %d.%02d (gcc " __VERSION__ ")", + NX_CURRENT_COMPILER_RELEASE/100, NX_CURRENT_COMPILER_RELEASE%100); +# define COMPILER_NAME compiler_name +# else +# define COMPILER_NAME "gcc " __VERSION__ +# endif +#else /* !__GNUC__ */ +# if defined(__SUNPRO_C) + sprintf( compiler_name, "Sun C version %x", __SUNPRO_C); +# define COMPILER_NAME compiler_name +# else +# if (defined( __HP_cc)) + if ((__HP_cc% 100) == 0) + { + sprintf( compiler_name, "HP C version A.%02d.%02d", + (__HP_cc/ 10000), ((__HP_cc% 10000)/ 100)); + } + else + { + sprintf( compiler_name, "HP C version A.%02d.%02d.%02d", + (__HP_cc/ 10000), ((__HP_cc% 10000)/ 100), (__HP_cc% 100)); + } +# define COMPILER_NAME compiler_name +# else +# if (defined( __DECC_VER)) + sprintf( compiler_name, "DEC C version %c%d.%d-%03d", + ((compiler_typ = (__DECC_VER / 10000) % 10) == 6 ? 'T' : + (compiler_typ == 8 ? 'S' : 'V')), + __DECC_VER / 10000000, + (__DECC_VER % 10000000) / 100000, __DECC_VER % 1000); +# define COMPILER_NAME compiler_name +# else +# if ((defined(CRAY) || defined(cray)) && defined(_RELEASE)) + sprintf(compiler_name, "cc version %d", _RELEASE); +# define COMPILER_NAME compiler_name +# else +# ifdef __IBMC__ + sprintf( compiler_name, "IBM C version %d.%d.%d", + (__IBMC__/ 100), ((__IBMC__/ 10)% 10), (__IBMC__% 10)); +# define COMPILER_NAME compiler_name +# else +# ifdef __VERSION__ +# define COMPILER_NAME "cc " __VERSION__ +# else +# define COMPILER_NAME "cc " +# endif +# endif +# endif +# endif +# endif +# endif +#endif /* ?__GNUC__ */ + + +/* Define the name to use for the OS we're compiling on */ +#if defined(sgi) || defined(__sgi) +# define OS_NAME "Silicon Graphics IRIX" +#else +#ifdef sun +# ifdef sparc +# ifdef __SVR4 +# define OS_NAME "Sun SPARC/Solaris" +# else /* may or may not be SunOS */ +# define OS_NAME "Sun SPARC" +# endif +# else +# if defined(sun386) || defined(i386) +# define OS_NAME "Sun 386i" +# else +# if defined(mc68020) || defined(__mc68020__) +# define OS_NAME "Sun 3" +# else /* mc68010 or mc68000: Sun 2 or earlier */ +# define OS_NAME "Sun 2" +# endif +# endif +# endif +#else +#ifdef __hpux +# define OS_NAME "HP-UX" +#else +#ifdef __osf__ +# define OS_NAME "DEC OSF/1" +#else +#ifdef _AIX +# define OS_NAME "IBM AIX" +#else +#ifdef aiws +# define OS_NAME "IBM RT/AIX" +#else +#if defined(CRAY) || defined(cray) +# ifdef _UNICOS + sprintf(os_name, "Cray UNICOS release %d", _UNICOS); +# define OS_NAME os_name +# else +# define OS_NAME "Cray UNICOS" +# endif +#else +#if defined(uts) || defined(UTS) +# define OS_NAME "Amdahl UTS" +#else +#ifdef NeXT +# ifdef mc68000 +# define OS_NAME "NeXTStep/black" +# else +# define OS_NAME "NeXTStep for Intel" +# endif +#else +#if defined(linux) || defined(__linux__) +# ifdef __ELF__ +# define OS_NAME "Linux ELF" +# else +# define OS_NAME "Linux a.out" +# endif +#else +#ifdef MINIX +# define OS_NAME "Minix" +#else +#ifdef M_UNIX +# define OS_NAME "SCO Unix" +#else +#ifdef M_XENIX +# define OS_NAME "SCO Xenix" +#else +#ifdef BSD +# if (BSD > 199103) +# define OS_NAME os_name + uname(&u); + sprintf(os_name, "%s %s", u.sysname, u.release); +# else +# ifdef __NetBSD__ +# define OS_NAME os_name +# ifdef NetBSD0_8 + sprintf(os_name, "NetBSD 0.8%s", netbsd[NetBSD0_8]); +# else +# ifdef NetBSD0_9 + sprintf(os_name, "NetBSD 0.9%s", netbsd[NetBSD0_9]); +# else +# ifdef NetBSD1_0 + sprintf(os_name, "NetBSD 1.0%s", netbsd[NetBSD1_0]); +# endif /* NetBSD1_0 */ +# endif /* NetBSD0_9 */ +# endif /* NetBSD0_8 */ +# else +# ifdef __FreeBSD__ +# define OS_NAME "FreeBSD 1.x" +# else +# ifdef __bsdi__ +# define OS_NAME "BSD/386 1.0" +# else +# ifdef __386BSD__ +# define OS_NAME "386BSD" +# else +# define OS_NAME "Unknown BSD" +# endif /* __386BSD__ */ +# endif /* __bsdi__ */ +# endif /* FreeBSD */ +# endif /* NetBSD */ +# endif /* BSD > 199103 */ +#else +#ifdef __CYGWIN__ +# define OS_NAME "Cygwin" +#else +#if defined(i686) || defined(__i686) || defined(__i686__) +# define OS_NAME "Intel 686" +#else +#if defined(i586) || defined(__i586) || defined(__i586__) +# define OS_NAME "Intel 586" +#else +#if defined(i486) || defined(__i486) || defined(__i486__) +# define OS_NAME "Intel 486" +#else +#if defined(i386) || defined(__i386) || defined(__i386__) +# define OS_NAME "Intel 386" +#else +#ifdef pyr +# define OS_NAME "Pyramid" +#else +#if defined(ultrix) || defined(__ultrix) +# if defined(mips) || defined(__mips) +# define OS_NAME "DEC/MIPS" +# else +# if defined(vax) || defined(__vax) +# define OS_NAME "DEC/VAX" +# else /* __alpha? */ +# define OS_NAME "DEC/Alpha" +# endif +# endif +#else +#ifdef gould +# define OS_NAME "Gould" +#else +#ifdef MTS +# define OS_NAME "MTS" +#else +#ifdef __convexc__ +# define OS_NAME "Convex" +#else +#ifdef __QNX__ +# define OS_NAME "QNX 4" +#else +#ifdef __QNXNTO__ +# define OS_NAME "QNX Neutrino" +#else +#ifdef __APPLE__ +# ifdef __i386__ +# define OS_NAME "Mac OS X Intel" +# else /* __i386__ */ +# ifdef __ppc__ +# define OS_NAME "Mac OS X PowerPC" +# else /* __ppc__ */ +# ifdef __ppc64__ +# define OS_NAME "Mac OS X PowerPC64" +# else /* __ppc64__ */ +# define OS_NAME "Mac OS X" +# endif /* __ppc64__ */ +# endif /* __ppc__ */ +# endif /* __i386__ */ +#else +# define OS_NAME "Unknown" +#endif /* Apple */ +#endif /* QNX Neutrino */ +#endif /* QNX 4 */ +#endif /* Convex */ +#endif /* MTS */ +#endif /* Gould */ +#endif /* DEC */ +#endif /* Pyramid */ +#endif /* 386 */ +#endif /* 486 */ +#endif /* 586 */ +#endif /* 686 */ +#endif /* Cygwin */ +#endif /* BSD */ +#endif /* SCO Xenix */ +#endif /* SCO Unix */ +#endif /* Minix */ +#endif /* Linux */ +#endif /* NeXT */ +#endif /* Amdahl */ +#endif /* Cray */ +#endif /* RT/AIX */ +#endif /* AIX */ +#endif /* OSF/1 */ +#endif /* HP-UX */ +#endif /* Sun */ +#endif /* SGI */ + + +/* Define the compile date string */ +#ifdef __DATE__ +# define COMPILE_DATE " on " __DATE__ +#else +# define COMPILE_DATE "" +#endif + + printf("Compiled with %s for Unix (%s)%s.\n\n", + COMPILER_NAME, OS_NAME, COMPILE_DATE); + +} /* end function version_local() */ + + +/* 2006-03-23 SMS. + * Emergency replacement for strerror(). (Useful on SunOS 4.*.) + * Enable by specifying "LOCAL_UNZIP=-DNEED_STRERROR=1" on the "make" + * command line. + */ + +#ifdef NEED_STRERROR + +char *strerror( err) + int err; +{ + extern char *sys_errlist[]; + extern int sys_nerr; + + static char no_msg[ 64]; + + if ((err >= 0) && (err < sys_nerr)) + { + return sys_errlist[ err]; + } + else + { + sprintf( no_msg, "(no message, code = %d.)", err); + return no_msg; + } +} + +#endif /* def NEED_STRERROR */ + + +/* 2006-03-23 SMS. + * Emergency replacement for memmove(). (Useful on SunOS 4.*.) + * Enable by specifying "LOCAL_UNZIP=-DNEED_MEMMOVE=1" on the "make" + * command line. + */ + +#ifdef NEED_MEMMOVE + +/* memmove.c -- copy memory. + Copy LENGTH bytes from SOURCE to DEST. Does not null-terminate. + In the public domain. + By David MacKenzie . + Adjusted by SMS. +*/ + +void *memmove(dest0, source0, length) + void *dest0; + void const *source0; + size_t length; +{ + char *dest = dest0; + char const *source = source0; + if (source < dest) + /* Moving from low mem to hi mem; start at end. */ + for (source += length, dest += length; length; --length) + *--dest = *--source; + else if (source != dest) + { + /* Moving from hi mem to low mem; start at beginning. */ + for (; length; --length) + *dest++ = *source++; + } + return dest0; +} + +#endif /* def NEED_MEMMOVE */ diff --git a/third_party/infozip/zip/unix/zipup.h b/third_party/infozip/zip/unix/zipup.h new file mode 100644 index 000000000..08b2ca740 --- /dev/null +++ b/third_party/infozip/zip/unix/zipup.h @@ -0,0 +1,25 @@ +/* clang-format off */ +/* + unix/zipup.h - Zip 3 + + Copyright (c) 1990-2005 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2005-Feb-10 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, both of these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +#ifndef O_RDONLY +# define O_RDONLY 0 +#endif +#ifndef O_BINARY +# define O_BINARY 0 +#endif +#define fhow (O_RDONLY|O_BINARY) +#define fbad (-1) +typedef int ftype; +#define zopen(n,p) open(n,p) +#define zread(f,b,n) read(f,b,n) +#define zclose(f) close(f) +#define zerr(f) (k == (extent)(-1L)) +#define zstdin 0 diff --git a/third_party/infozip/zip/util.c b/third_party/infozip/zip/util.c new file mode 100644 index 000000000..44fe0a9f7 --- /dev/null +++ b/third_party/infozip/zip/util.c @@ -0,0 +1,1453 @@ +/* clang-format off */ +/* + util.c + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * util.c by Mark Adler. + */ +#define __UTIL_C + +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/ebcdic.h" +#include "libc/str/str.h" +#include "libc/fmt/fmt.h" +#include "libc/fmt/conv.h" + +#ifdef MSDOS16 +# include +#endif + +#ifdef NO_MKTIME +# ifndef IZ_MKTIME_ONLY +# define IZ_MKTIME_ONLY /* only mktime() related code is pulled in */ +# endif +# include "timezone.c" +#endif + +uch upper[256], lower[256]; +/* Country-dependent case map table */ + + +#ifndef UTIL /* UTIL picks out namecmp code (all utils) */ + +/* RISC OS uses # as its single-character wildcard */ +#ifdef RISCOS +# define WILDCHR_SINGLE '#' +# define WILDCHR_MULTI '*' +# define DIRSEP_CHR '.' +#endif + +#ifdef VMS +# define WILDCHR_SINGLE '%' +# define WILDCHR_MULTI '*' +# define DIRSEP_CHR '.' +#endif + +#ifndef WILDCHR_SINGLE +# define WILDCHR_SINGLE '?' +#endif +#ifndef WILDCHR_MULTI +# define WILDCHR_MULTI '*' +#endif +#ifndef DIRSEP_CHR +# define DIRSEP_CHR '/' +#endif + +/* Local functions */ +local int recmatch OF((ZCONST char *, ZCONST char *, int)); +#if defined(UNICODE_SUPPORT) && defined(WIN32) + local long recmatchw OF((ZCONST wchar_t *, ZCONST wchar_t *, int)); +#endif +local int count_args OF((char *s)); + +#ifdef MSDOS16 + local unsigned ident OF((unsigned chr)); +#endif + +#ifndef HAVE_FSEEKABLE + +/* 2004-11-12 SMS. + Changed to use z*o() functions, and ftell() test from >= 0 to != -1. + This solves problems with negative 32-bit offsets, even on small-file + products. +*/ +int fseekable( fp) +FILE *fp; +{ + zoff_t x; + + return (fp == NULL || + ((zfseeko( fp, ((zoff_t) -1), SEEK_CUR) == 0) && /* Seek ok. */ + ((x = zftello( fp)) != ((zoff_t) -1)) && /* Tell ok. */ + (zfseeko( fp, ((zoff_t) 1), SEEK_CUR) == 0) && /* Seek ok. */ + (zftello( fp) == x+ 1))); /* Tells agree. */ +} +#endif /* HAVE_FSEEKABLE */ + + +char *isshexp(p) +char *p; /* candidate sh expression */ +/* If p is a sh expression, a pointer to the first special character is + returned. Otherwise, NULL is returned. */ +{ + for (; *p; INCSTR(p)) + if (*p == '\\' && *(p+1)) + p++; +#ifdef VMS + else if (*p == WILDCHR_SINGLE || *p == WILDCHR_MULTI) +#else /* !VMS */ + else if (*p == WILDCHR_SINGLE || *p == WILDCHR_MULTI || *p == '[') +#endif /* ?VMS */ + return p; + return NULL; +} + +#ifdef UNICODE_SUPPORT +# ifdef WIN32 + +wchar_t *isshexpw(pw) + wchar_t *pw; /* candidate sh expression */ +/* If pw is a sh expression, a pointer to the first special character is + returned. Otherwise, NULL is returned. */ +{ + for (; *pw; pw++) + if (*pw == (wchar_t)'\\' && *(pw+1)) + pw++; + else if (*pw == (wchar_t)WILDCHR_SINGLE || *pw == (wchar_t)WILDCHR_MULTI || + *pw == (wchar_t)'[') + return pw; + return NULL; +} + +# endif +#endif + + +#ifdef UNICODE_SUPPORT +# ifdef WIN32 + +local long recmatchw(pw, sw, cs) +ZCONST wchar_t *pw; /* sh pattern to match */ +ZCONST wchar_t *sw; /* string to match it to */ +int cs; /* flag: force case-sensitive matching */ +/* Recursively compare the sh pattern p with the string s and return 1 if + they match, and 0 or 2 if they don't or if there is a syntax error in the + pattern. This routine recurses on itself no deeper than the number of + characters in the pattern. */ +{ + long c; /* pattern char or start of range in [-] loop */ + /* Get first character, the pattern for new recmatch calls follows */ + + c = (long)*(pw++); + + /* If that was the end of the pattern, match if string empty too */ + if (c == 0) + return *sw == 0; + + /* '?' matches any character (but not an empty string) */ + if ((wchar_t)c == (wchar_t)WILDCHR_SINGLE) { + if (wild_stop_at_dir) + return (*sw && *sw != (wchar_t)DIRSEP_CHR) ? recmatchw(pw, sw + 1, cs) : 0; + else + return *sw ? recmatchw(pw, sw + 1, cs) : 0; + } + + /* WILDCHR_MULTI ('*') matches any number of characters, including zero */ + if (!no_wild && (wchar_t)c == (wchar_t)WILDCHR_MULTI) + { + if (wild_stop_at_dir) { + /* Check for an immediately following WILDCHR_MULTI */ + if (*pw != (wchar_t)WILDCHR_MULTI) { + /* Single WILDCHR_MULTI ('*'): this doesn't match slashes */ + for (; *sw && *sw != (wchar_t)DIRSEP_CHR; sw++) + if ((c = recmatchw(pw, sw, cs)) != 0) + return c; + /* end of pattern: matched if at end of string, else continue */ + if (*pw == 0) + return (*sw == 0); + /* continue to match if at DIRSEP_CHR in pattern, else give up */ + return (*pw == (wchar_t)DIRSEP_CHR || (*pw == (wchar_t)'\\' && + pw[1] == (wchar_t)DIRSEP_CHR)) + ? recmatchw(pw, sw, cs) : 2; + } + /* Two consecutive WILDCHR_MULTI ("**"): this matches DIRSEP_CHR ('/') */ + pw++; /* move p past the second WILDCHR_MULTI */ + /* continue with the normal non-WILD_STOP_AT_DIR code */ + } /* wild_stop_at_dir */ + + /* Not wild_stop_at_dir */ + if (*pw == 0) + return 1; + if (!isshexpw((wchar_t *)pw)) + { + /* optimization for rest of pattern being a literal string */ + + /* optimization to handle patterns like *.txt */ + /* if the first char in the pattern is '*' and there */ + /* are no other shell expression chars, i.e. a literal string */ + /* then just compare the literal string at the end */ + + ZCONST wchar_t *swrest; + + swrest = sw + (wcslen(sw) - wcslen(pw)); + if (swrest - sw < 0) + /* remaining literal string from pattern is longer than rest of + test string, there can't be a match + */ + return 0; + else + /* compare the remaining literal pattern string with the last bytes + of the test string to check for a match */ + return ((cs ? wcscmp(pw, swrest) : _wcsicmp(pw, swrest)) == 0); + } + else + { + /* pattern contains more wildcards, continue with recursion... */ + for (; *sw; sw++) + if ((c = recmatchw(pw, sw, cs)) != 0) + return c; + return 2; /* 2 means give up--shmatch will return false */ + } + } + + /* Parse and process the list of characters and ranges in brackets */ + if (!no_wild && allow_regex && (wchar_t)c == '[') + { + int e; /* flag true if next char to be taken literally */ + ZCONST wchar_t *qw; /* pointer to end of [-] group */ + int r; /* flag true to match anything but the range */ + + if (*sw == 0) /* need a character to match */ + return 0; + pw += (r = (*pw == (wchar_t)'!' || *pw == (wchar_t)'^')); /* see if reverse */ + for (qw = pw, e = 0; *qw; qw++) /* find closing bracket */ + if (e) + e = 0; + else + if (*qw == (wchar_t)'\\') + e = 1; + else if (*qw == (wchar_t)']') + break; + if (*qw != (wchar_t)']') /* nothing matches if bad syntax */ + return 0; + for (c = 0, e = *pw == (wchar_t)'-'; pw < qw; pw++) /* go through the list */ + { + if (e == 0 && *pw == (wchar_t)'\\') /* set escape flag if \ */ + e = 1; + else if (e == 0 && *pw == (wchar_t)'-') /* set start of range if - */ + c = *(pw-1); + else + { + wchar_t cc = (cs ? *sw : towupper(*sw)); + wchar_t uc = (wchar_t) c; + + if (*(pw+1) != (wchar_t)'-') + for (uc = uc ? uc : *pw; cc <= *pw; uc++) + /* compare range */ + if ((cs ? uc : towupper(uc)) == cc) + return r ? 0 : recmatchw(qw + 1, sw + 1, cs); + c = e = 0; /* clear range, escape flags */ + } + } + return r ? recmatchw(qw + 1, sw + 1, cs) : 0; + /* bracket match failed */ + } + + /* If escape ('\'), just compare next character */ + if (!no_wild && (wchar_t)c == (wchar_t)'\\') + if ((c = *pw++) == '\0') /* if \ at end, then syntax error */ + return 0; + + /* Just a character--compare it */ + return (cs ? (wchar_t)c == *sw : towupper((wchar_t)c) == towupper(*sw)) ? + recmatchw(pw, sw + 1, cs) : 0; +} + +# endif +#endif + + +local int recmatch(p, s, cs) +ZCONST char *p; /* sh pattern to match */ +ZCONST char *s; /* string to match it to */ +int cs; /* flag: force case-sensitive matching */ +/* Recursively compare the sh pattern p with the string s and return 1 if + they match, and 0 or 2 if they don't or if there is a syntax error in the + pattern. This routine recurses on itself no deeper than the number of + characters in the pattern. */ +{ + int c; /* pattern char or start of range in [-] loop */ + /* Get first character, the pattern for new recmatch calls follows */ + + /* This fix provided by akt@m5.dion.ne.jp for Japanese. + See 21 July 2006 mail. + It only applies when p is pointing to a doublebyte character and + things like / and wildcards are not doublebyte. This probably + should not be needed. */ + +#ifdef _MBCS + if (CLEN(p) == 2) { + if (CLEN(s) == 2) { + return (*p == *s && *(p+1) == *(s+1)) ? + recmatch(p + 2, s + 2, cs) : 0; + } else { + return 0; + } + } +#endif /* ?_MBCS */ + + c = *POSTINCSTR(p); + + /* If that was the end of the pattern, match if string empty too */ + if (c == 0) + return *s == 0; + + /* '?' (or '%' or '#') matches any character (but not an empty string) */ + if (c == WILDCHR_SINGLE) { + if (wild_stop_at_dir) + return (*s && *s != DIRSEP_CHR) ? recmatch(p, s + CLEN(s), cs) : 0; + else + return *s ? recmatch(p, s + CLEN(s), cs) : 0; + } + + /* WILDCHR_MULTI ('*') matches any number of characters, including zero */ +#ifdef AMIGA + if (!no_wild && c == '#' && *p == '?') /* "#?" is Amiga-ese for "*" */ + c = WILDCHR_MULTI, p++; +#endif /* AMIGA */ + if (!no_wild && c == WILDCHR_MULTI) + { + if (wild_stop_at_dir) { + /* Check for an immediately following WILDCHR_MULTI */ +# ifdef AMIGA + if ((c = p[0]) == '#' && p[1] == '?') /* "#?" is Amiga-ese for "*" */ + c = WILDCHR_MULTI, p++; + if (c != WILDCHR_MULTI) { +# else /* !AMIGA */ + if (*p != WILDCHR_MULTI) { +# endif /* ?AMIGA */ + /* Single WILDCHR_MULTI ('*'): this doesn't match slashes */ + for (; *s && *s != DIRSEP_CHR; INCSTR(s)) + if ((c = recmatch(p, s, cs)) != 0) + return c; + /* end of pattern: matched if at end of string, else continue */ + if (*p == 0) + return (*s == 0); + /* continue to match if at DIRSEP_CHR in pattern, else give up */ + return (*p == DIRSEP_CHR || (*p == '\\' && p[1] == DIRSEP_CHR)) + ? recmatch(p, s, cs) : 2; + } + /* Two consecutive WILDCHR_MULTI ("**"): this matches DIRSEP_CHR ('/') */ + p++; /* move p past the second WILDCHR_MULTI */ + /* continue with the normal non-WILD_STOP_AT_DIR code */ + } /* wild_stop_at_dir */ + + /* Not wild_stop_at_dir */ + if (*p == 0) + return 1; + if (!isshexp((char *)p)) + { + /* optimization for rest of pattern being a literal string */ + + /* optimization to handle patterns like *.txt */ + /* if the first char in the pattern is '*' and there */ + /* are no other shell expression chars, i.e. a literal string */ + /* then just compare the literal string at the end */ + + ZCONST char *srest; + + srest = s + (strlen(s) - strlen(p)); + if (srest - s < 0) + /* remaining literal string from pattern is longer than rest of + test string, there can't be a match + */ + return 0; + else + /* compare the remaining literal pattern string with the last bytes + of the test string to check for a match */ +#ifdef _MBCS + { + ZCONST char *q = s; + + /* MBCS-aware code must not scan backwards into a string from + * the end. + * So, we have to move forward by character from our well-known + * character position s in the test string until we have advanced + * to the srest position. + */ + while (q < srest) + INCSTR(q); + /* In case the byte *srest is a trailing byte of a multibyte + * character, we have actually advanced past the position (srest). + * For this case, the match has failed! + */ + if (q != srest) + return 0; + return ((cs ? strcmp(p, q) : namecmp(p, q)) == 0); + } +#else /* !_MBCS */ + return ((cs ? strcmp(p, srest) : namecmp(p, srest)) == 0); +#endif /* ?_MBCS */ + } + else + { + /* pattern contains more wildcards, continue with recursion... */ + for (; *s; INCSTR(s)) + if ((c = recmatch(p, s, cs)) != 0) + return c; + return 2; /* 2 means give up--shmatch will return false */ + } + } + +#ifndef VMS /* No bracket matching in VMS */ + /* Parse and process the list of characters and ranges in brackets */ + if (!no_wild && allow_regex && c == '[') + { + int e; /* flag true if next char to be taken literally */ + ZCONST char *q; /* pointer to end of [-] group */ + int r; /* flag true to match anything but the range */ + + if (*s == 0) /* need a character to match */ + return 0; + p += (r = (*p == '!' || *p == '^')); /* see if reverse */ + for (q = p, e = 0; *q; q++) /* find closing bracket */ + if (e) + e = 0; + else + if (*q == '\\') + e = 1; + else if (*q == ']') + break; + if (*q != ']') /* nothing matches if bad syntax */ + return 0; + for (c = 0, e = *p == '-'; p < q; p++) /* go through the list */ + { + if (e == 0 && *p == '\\') /* set escape flag if \ */ + e = 1; + else if (e == 0 && *p == '-') /* set start of range if - */ + c = *(p-1); + else + { + uch cc = (cs ? (uch)*s : case_map((uch)*s)); + uch uc = (uch) c; + if (*(p+1) != '-') + for (uc = uc ? uc : (uch)*p; uc <= (uch)*p; uc++) + /* compare range */ + if ((cs ? uc : case_map(uc)) == cc) + return r ? 0 : recmatch(q + CLEN(q), s + CLEN(s), cs); + c = e = 0; /* clear range, escape flags */ + } + } + return r ? recmatch(q + CLEN(q), s + CLEN(s), cs) : 0; + /* bracket match failed */ + } +#endif /* !VMS */ + + /* If escape ('\'), just compare next character */ + if (!no_wild && c == '\\') + if ((c = *p++) == '\0') /* if \ at end, then syntax error */ + return 0; + +#ifdef VMS + /* 2005-11-06 SMS. + Handle "..." wildcard in p with "." or "]" in s. + */ + if ((c == '.') && (*p == '.') && (*(p+ CLEN( p)) == '.') && + ((*s == '.') || (*s == ']'))) + { + /* Match "...]" with "]". Continue after "]" in both. */ + if ((*(p+ 2* CLEN( p)) == ']') && (*s == ']')) + return recmatch( (p+ 3* CLEN( p)), (s+ CLEN( s)), cs); + + /* Else, look for a reduced match in s, until "]" in or end of s. */ + for (; *s && (*s != ']'); INCSTR(s)) + if (*s == '.') + /* If reduced match, then continue after "..." in p, "." in s. */ + if ((c = recmatch( (p+ CLEN( p)), s, cs)) != 0) + return (int)c; + + /* Match "...]" with "]". Continue after "]" in both. */ + if ((*(p+ 2* CLEN( p)) == ']') && (*s == ']')) + return recmatch( (p+ 3* CLEN( p)), (s+ CLEN( s)), cs); + + /* No reduced match. Quit. */ + return 2; + } + +#endif /* def VMS */ + + /* Just a character--compare it */ + return (cs ? c == *s : case_map((uch)c) == case_map((uch)*s)) ? + recmatch(p, s + CLEN(s), cs) : 0; +} + + +int shmatch(p, s, cs) +ZCONST char *p; /* sh pattern to match */ +ZCONST char *s; /* string to match it to */ +int cs; /* force case-sensitive match if TRUE */ +/* Compare the sh pattern p with the string s and return true if they match, + false if they don't or if there is a syntax error in the pattern. */ +{ + return recmatch(p, s, cs) == 1; +} + + +#if defined(DOS) || defined(WIN32) + +#ifdef UNICODE_SUPPORT + +int dosmatchw(pw, sw, cs) +ZCONST wchar_t *pw; /* dos pattern to match */ +ZCONST wchar_t *sw; /* string to match it to */ +int cs; /* force case-sensitive match if TRUE */ +/* Treat filenames without periods as having an implicit trailing period */ +{ + wchar_t *sw1; /* revised string to match */ + int r; /* result */ + + if (wcschr(pw, (wchar_t)'.') && !wcschr(sw, (wchar_t)'.') && + ((sw1 = (wchar_t *)malloc((wcslen(sw) + 2) * sizeof(wchar_t))) != NULL)) + { + wcscpy(sw1, sw); + wcscat(sw1, L"."); + } + else + { + /* will usually be OK */ + sw1 = (wchar_t *)sw; + } + + r = recmatchw(pw, sw1, cs) == 1; + if (sw != sw1) + free((zvoid *)sw1); + return r == 1; +} + +#endif + +/* XXX also suitable for OS2? Atari? Human68K? TOPS-20?? */ + +int dosmatch(p, s, cs) +ZCONST char *p; /* dos pattern to match */ +ZCONST char *s; /* string to match it to */ +int cs; /* force case-sensitive match if TRUE */ +/* Treat filenames without periods as having an implicit trailing period */ +{ + char *s1; /* revised string to match */ + int r; /* result */ + + if (strchr(p, '.') && !strchr(s, '.') && + ((s1 = malloc(strlen(s) + 2)) != NULL)) + { + strcpy(s1, s); + strcat(s1, "."); + } + else + { + /* will usually be OK */ + s1 = (char *)s; + } + + r = recmatch(p, s1, cs) == 1; + if (s != s1) + free((zvoid *)s1); + return r == 1; +} + +#endif /* DOS || WIN32 */ + +zvoid far **search(b, a, n, cmp) +ZCONST zvoid *b; /* pointer to value to search for */ +ZCONST zvoid far **a; /* table of pointers to values, sorted */ +extent n; /* number of pointers in a[] */ +int (*cmp) OF((ZCONST zvoid *, ZCONST zvoid far *)); /* comparison function */ + +/* Search for b in the pointer list a[0..n-1] using the compare function + cmp(b, c) where c is an element of a[i] and cmp() returns negative if + *b < *c, zero if *b == *c, or positive if *b > *c. If *b is found, + search returns a pointer to the entry in a[], else search() returns + NULL. The nature and size of *b and *c (they can be different) are + left up to the cmp() function. A binary search is used, and it is + assumed that the list is sorted in ascending order. */ +{ + ZCONST zvoid far **i; /* pointer to midpoint of current range */ + ZCONST zvoid far **l; /* pointer to lower end of current range */ + int r; /* result of (*cmp)() call */ + ZCONST zvoid far **u; /* pointer to upper end of current range */ + + l = (ZCONST zvoid far **)a; u = l + (n-1); + while (u >= l) { + i = l + ((unsigned)(u - l) >> 1); + if ((r = (*cmp)(b, (ZCONST char far *)*(struct zlist far **)i)) < 0) + u = i - 1; + else if (r > 0) + l = i + 1; + else + return (zvoid far **)i; + } + return NULL; /* If b were in list, it would belong at l */ +} + +#endif /* !UTIL */ + +#ifdef MSDOS16 + +local unsigned ident(unsigned chr) +{ + return chr; /* in al */ +} + +void init_upper() +{ + static struct country { + uch ignore[18]; + int (far *casemap)(int); + uch filler[16]; + } country_info; + + struct country far *info = &country_info; + union REGS regs; + struct SREGS sregs; + unsigned int c; + + regs.x.ax = 0x3800; /* get country info */ + regs.x.dx = FP_OFF(info); + sregs.ds = FP_SEG(info); + intdosx(®s, ®s, &sregs); + for (c = 0; c < 128; c++) { + upper[c] = (uch) toupper(c); + lower[c] = (uch) c; + } + for (; c < sizeof(upper); c++) { + upper[c] = (uch) (*country_info.casemap)(ident(c)); + /* ident() required because casemap takes its parameter in al */ + lower[c] = (uch) c; + } + for (c = 0; c < sizeof(upper); c++ ) { + unsigned int u = upper[c]; + if (u != c && lower[u] == (uch) u) { + lower[u] = (uch)c; + } + } + for (c = 'A'; c <= 'Z'; c++) { + lower[c] = (uch) (c - 'A' + 'a'); + } +} +#else /* !MSDOS16 */ +# ifndef OS2 + +void init_upper() +{ + unsigned int c; +#if defined(ATARI) || defined(CMS_MVS) +#include +/* this should be valid for all other platforms too. (HD 11/11/95) */ + for (c = 0; c< sizeof(upper); c++) { + upper[c] = islower(c) ? toupper(c) : c; + lower[c] = isupper(c) ? tolower(c) : c; + } +#else + for (c = 0; c < sizeof(upper); c++) upper[c] = lower[c] = (uch)c; + for (c = 'a'; c <= 'z'; c++) upper[c] = (uch)(c - 'a' + 'A'); + for (c = 'A'; c <= 'Z'; c++) lower[c] = (uch)(c - 'A' + 'a'); +#endif +} +# endif /* !OS2 */ + +#endif /* ?MSDOS16 */ + +int namecmp(string1, string2) + ZCONST char *string1, *string2; +/* Compare the two strings ignoring case, and correctly taking into + * account national language characters. For operating systems with + * case sensitive file names, this function is equivalent to strcmp. + */ +{ + int d; + + for (;;) + { + d = (int) (uch) case_map(*string1) + - (int) (uch) case_map(*string2); + + if (d || *string1 == 0 || *string2 == 0) + return d; + + string1++; + string2++; + } +} + +#ifdef EBCDIC +char *strtoasc(char *str1, ZCONST char *str2) +{ + char *old; + old = str1; + while (*str1++ = (char)ascii[(uch)(*str2++)]); + return old; +} + +char *strtoebc(char *str1, ZCONST char *str2) +{ + char *old; + old = str1; + while (*str1++ = (char)ebcdic[(uch)(*str2++)]); + return old; +} + +char *memtoasc(char *mem1, ZCONST char *mem2, unsigned len) +{ + char *old; + old = mem1; + while (len--) + *mem1++ = (char)ascii[(uch)(*mem2++)]; + return old; +} + +char *memtoebc(char *mem1, ZCONST char *mem2, unsigned len) +{ + char *old; + old = mem1; + while (len--) + *mem1++ = (char)ebcdic[(uch)(*mem2++)]; + return old; +} +#endif /* EBCDIC */ + +#ifdef IZ_ISO2OEM_ARRAY +char *str_iso_to_oem(dst, src) + ZCONST char *src; + char *dst; +{ + char *dest_start = dst; + while (*dst++ = (char)iso2oem[(uch)(*src++)]); + return dest_start; +} +#endif + +#ifdef IZ_OEM2ISO_ARRAY +char *str_oem_to_iso(dst, src) + ZCONST char *src; + char *dst; +{ + char *dest_start = dst; + while (*dst++ = (char)oem2iso[(uch)(*src++)]); + return dest_start; +} +#endif + + + +/* DBCS support for Info-ZIP's zip (mainly for japanese (-: ) + * by Yoshioka Tsuneo (QWF00133@nifty.ne.jp,tsuneo-y@is.aist-nara.ac.jp) + * This code is public domain! Date: 1998/12/20 + */ +#ifdef _MBCS + +char *___tmp_ptr; + +int lastchar(ptr) + ZCONST char *ptr; +{ + ZCONST char *oldptr = ptr; + while(*ptr != '\0'){ + oldptr = ptr; + INCSTR(ptr); + } + return (int)(unsigned)*oldptr; +} + +unsigned char *zmbschr(str, c) + ZCONST unsigned char *str; + unsigned int c; +{ + while(*str != '\0'){ + if (*str == c) {return (unsigned char *)str;} + INCSTR(str); + } + return NULL; +} + +unsigned char *zmbsrchr(str, c) + ZCONST unsigned char *str; + unsigned int c; +{ + unsigned char *match = NULL; + while(*str != '\0'){ + if (*str == c) {match = (unsigned char*)str;} + INCSTR(str); + } + return match; +} +#endif /* _MBCS */ + + + +#ifndef UTIL + +/***************************************************************** + | envargs - add default options from environment to command line + |---------------------------------------------------------------- + | Author: Bill Davidsen, original 10/13/91, revised 23 Oct 1991. + | This program is in the public domain. + |---------------------------------------------------------------- + | Minor program notes: + | 1. Yes, the indirection is a tad complex + | 2. Parenthesis were added where not needed in some cases + | to make the action of the code less obscure. + ****************************************************************/ + +void envargs(Pargc, Pargv, envstr, envstr2) + int *Pargc; + char ***Pargv; + char *envstr; + char *envstr2; +{ + char *envptr; /* value returned by getenv */ + char *bufptr; /* copy of env info */ + int argc; /* internal arg count */ + register int ch; /* spare temp value */ + char **argv; /* internal arg vector */ + char **argvect; /* copy of vector address */ + + /* see if anything in the environment */ + envptr = getenv(envstr); + if (envptr != NULL) /* usual var */ + while (isspace((uch)*envptr)) /* we must discard leading spaces */ + envptr++; + if (envptr == NULL || *envptr == '\0') + if ((envptr = getenv(envstr2)) != NULL) /* alternate */ + while (isspace((uch)*envptr)) + envptr++; + if (envptr == NULL || *envptr == '\0') + return; + + /* count the args so we can allocate room for them */ + argc = count_args(envptr); + bufptr = malloc(1 + strlen(envptr)); + if (bufptr == NULL) + ziperr(ZE_MEM, "Can't get memory for arguments"); + strcpy(bufptr, envptr); + + /* allocate a vector large enough for all args */ + argv = (char **)malloc((argc + *Pargc + 1) * sizeof(char *)); + if (argv == NULL) { + free(bufptr); + ziperr(ZE_MEM, "Can't get memory for arguments"); + } + argvect = argv; + + /* copy the program name first, that's always true */ + *(argv++) = *((*Pargv)++); + + /* copy the environment args first, may be changed */ + do { +#if defined(AMIGA) || defined(UNIX) + if (*bufptr == '"') { + char *argstart = ++bufptr; + *(argv++) = argstart; + for (ch = *bufptr; ch != '\0' && ch != '\"'; + ch = *PREINCSTR(bufptr)) + if (ch == '\\' && bufptr[1] != '\0') + ++bufptr; /* skip to char after backslash */ + if (ch != '\0') /* overwrite trailing '"' */ + *(bufptr++) = '\0'; + + /* remove escape characters */ + while ((argstart = MBSCHR(argstart, '\\')) != NULL) { + strcpy(argstart, argstart + 1); + if (*argstart) + ++argstart; + } + } else { + *(argv++) = bufptr; + while ((ch = *bufptr) != '\0' && !isspace((uch)ch)) INCSTR(bufptr); + if (ch != '\0') *(bufptr++) = '\0'; + } +#else +# ifdef WIN32 + /* We do not support backslash-quoting of quotes in quoted */ + /* strings under Win32, because backslashes are directory */ + /* separators and double quotes are illegal in filenames. */ + if (*bufptr == '"') { + *(argv++) = ++bufptr; + while ((ch = *bufptr) != '\0' && ch != '\"') INCSTR(bufptr); + if (ch != '\0') *(bufptr++) = '\0'; + } else { + *(argv++) = bufptr; + while ((ch = *bufptr) != '\0' && !isspace((uch)ch)) INCSTR(bufptr); + if (ch != '\0') *(bufptr++) = '\0'; + } +# else + *(argv++) = bufptr; + while ((ch = *bufptr) != '\0' && !isspace((uch)ch)) INCSTR(bufptr); + if (ch != '\0') *(bufptr++) = '\0'; +# endif +#endif /* ?(AMIGA || UNIX) */ + while ((ch = *bufptr) != '\0' && isspace((uch)ch)) INCSTR(bufptr); + } while (ch); + + /* now save old argc and copy in the old args */ + argc += *Pargc; + while (--(*Pargc)) *(argv++) = *((*Pargv)++); + + /* finally, add a NULL after the last arg, like UNIX */ + *argv = NULL; + + /* save the values and return */ + *Pargv = argvect; + *Pargc = argc; +} + +local int count_args(s) +char *s; +{ + int count = 0; + char ch; + + do { + /* count and skip args */ + ++count; +#if defined(AMIGA) || defined(UNIX) + if (*s == '\"') { + for (ch = *PREINCSTR(s); ch != '\0' && ch != '\"'; + ch = *PREINCSTR(s)) + if (ch == '\\' && s[1] != '\0') + INCSTR(s); + if (*s) INCSTR(s); /* trailing quote */ + } else + while ((ch = *s) != '\0' && !isspace((uch)ch)) INCSTR(s); +#else +# ifdef WIN32 + if (*s == '\"') { + ++s; /* leading quote */ + while ((ch = *s) != '\0' && ch != '\"') INCSTR(s); + if (*s) INCSTR(s); /* trailing quote */ + } else + while ((ch = *s) != '\0' && !isspace((uch)ch)) INCSTR(s); +# else + while ((ch = *s) != '\0' && !isspace((uch)ch)) INCSTR(s); +# endif +#endif /* ?(AMIGA || UNIX) */ + while ((ch = *s) != '\0' && isspace((uch)ch)) INCSTR(s); + } while (ch); + + return(count); +} + + + +/* Extended argument processing -- by Rich Wales + * This function currently deals only with the MKS shell, but could be + * extended later to understand other conventions. + * + * void expand_args(int *argcp, char ***argvp) + * + * Substitutes the extended command line argument list produced by + * the MKS Korn Shell in place of the command line info from DOS. + * + * The MKS shell gets around DOS's 128-byte limit on the length of + * a command line by passing the "real" command line in the envi- + * ronment. The "real" arguments are flagged by prepending a tilde + * (~) to each one. + * + * This "expand_args" routine creates a new argument list by scanning + * the environment from the beginning, looking for strings begin- + * ning with a tilde character. The new list replaces the original + * "argv" (pointed to by "argvp"), and the number of arguments + * in the new list replaces the original "argc" (pointed to by + * "argcp"). + */ +void expand_args(argcp, argvp) + int *argcp; + char ***argvp; +{ +#ifdef DOS + +/* Do NEVER include (re)definiton of `environ' variable with any version + of MSC or BORLAND/Turbo C. These compilers supply an incompatible + definition in . */ +#if defined(__GO32__) || defined(__EMX__) + extern char **environ; /* environment */ +#endif /* __GO32__ || __EMX__ */ + char **envp; /* pointer into environment */ + char **newargv; /* new argument list */ + char **argp; /* pointer into new arg list */ + int newargc; /* new argument count */ + + /* sanity check */ + if (environ == NULL + || argcp == NULL + || argvp == NULL || *argvp == NULL) + return; + /* find out how many environment arguments there are */ + for (envp = environ, newargc = 0; + *envp != NULL && (*envp)[0] == '~'; + envp++, newargc++) ; + if (newargc == 0) + return; /* no environment arguments */ + /* set up new argument list */ + newargv = (char **) malloc(sizeof(char **) * (newargc+1)); + if (newargv == NULL) + return; /* malloc failed */ + for (argp = newargv, envp = environ; + *envp != NULL && (*envp)[0] == '~'; + *argp++ = &(*envp++)[1]) ; + *argp = NULL; /* null-terminate the list */ + /* substitute new argument list in place of old one */ + *argcp = newargc; + *argvp = newargv; +#else /* !DOS */ + if (argcp || argvp) return; +#endif /* ?DOS */ +} + + +/* Fast routine for detection of plain text + * (ASCII or an ASCII-compatible extension such as ISO-8859, UTF-8, etc.) + * Author: Cosmin Truta. + * See "proginfo/txtvsbin.txt" for more information. + * + * This function returns the same result as set_file_type() in "trees.c". + * Unlike in set_file_type(), however, the speed depends on the buffer size, + * so the optimal implementation is different. + */ +int is_text_buf(buf_ptr, buf_size) + ZCONST char *buf_ptr; + unsigned buf_size; +{ + int result = 0; + unsigned i; + unsigned char c; + + for (i = 0; i < buf_size; ++i) + { + c = (unsigned char)buf_ptr[i]; + if (c >= 32) /* speed up the loop by checking this first */ + result = 1; /* white-listed character found; keep looping */ + else /* speed up the loop by inlining the following check */ + if ((c <= 6) || (c >= 14 && c <= 25) || (c >= 28 && c <= 31)) + return 0; /* black-listed character found; stop */ + } + + return result; +} + +#endif /* UTIL */ + + +#ifdef DEBUGNAMES +#undef free +int Free(x) +void *x; +{ + if (x == (void *) 0xdeadbeef) + exit(-1); + free(x); + return 0; +} + +int printnames() +{ + struct zlist far *z; + + for (z = zfiles; z != NULL; z = z->nxt) + fprintf(mesg, "%s %s %s %p %p %p %08x %08x %08x\n", + z->name, z->zname, z->iname, + z->name, z->zname, z->iname, + *((int *) z->name), *((int *) z->zname), + *((int *) z->iname)); + return 0; +} + +#endif /* DEBUGNAMES */ + + +/* Below is used to format zoff_t values, which can be either long or long long + depending on if LARGE FILES are supported. Function provided by SMS. + 10/17/04 EG */ + +/* 2004-12-01 SMS. + * Brought in fancy fzofft() from UnZip. + */ + +/* This implementation assumes that no more than FZOFF_NUM values will be + needed in any printf using it. */ + +/* zip_fzofft(): Format a zoff_t value in a cylindrical buffer set. + This version renamed from fzofft because of name conflict in unzip + when combined in WiZ. */ + +/* 2004-12-19 SMS. + * I still claim than the smart move would have been to disable one or + * the other instance with #if for Wiz. But fine. We'll change the + * name. + */ + +/* This is likely not thread safe. Needs to be done without static storage. + 12/29/04 EG */ + +/* zip_fzofft(): Format a zoff_t value in a cylindrical buffer set. */ + +#define FZOFFT_NUM 4 /* Number of chambers. */ +#define FZOFFT_LEN 24 /* Number of characters/chamber. */ + + +/* Format a zoff_t value in a cylindrical buffer set. */ + +char *zip_fzofft( val, pre, post) + zoff_t val; + char *pre; + char *post; +{ + /* Storage cylinder. */ + static char fzofft_buf[ FZOFFT_NUM][ FZOFFT_LEN]; + static int fzofft_index = 0; + + /* Temporary format string storage. */ + static char fmt[ 16] = "%"; + + /* Assemble the format string. */ + fmt[ 1] = '\0'; /* Start after initial "%". */ + if (pre == FZOFFT_HEX_WID) /* Special hex width. */ + { + strcat( fmt, FZOFFT_HEX_WID_VALUE); + } + else if (pre == FZOFFT_HEX_DOT_WID) /* Special hex ".width". */ + { + strcat( fmt, "."); + strcat( fmt, FZOFFT_HEX_WID_VALUE); + } + else if (pre != NULL) /* Caller's prefix (width). */ + { + strcat( fmt, pre); + } + + strcat( fmt, FZOFFT_FMT); /* Long or long-long or whatever. */ + + if (post == NULL) + strcat( fmt, "d"); /* Default radix = decimal. */ + else + strcat( fmt, post); /* Caller's radix. */ + + /* Advance the cylinder. */ + fzofft_index = (fzofft_index+ 1)% FZOFFT_NUM; + + /* Write into the current chamber. */ + sprintf( fzofft_buf[ fzofft_index], fmt, val); + + /* Return a pointer to this chamber. */ + return fzofft_buf[ fzofft_index]; +} + + +/* Format a uzoff_t value in a cylindrical buffer set. */ +/* Added to support uzoff_t type. 12/29/04 */ + +char *zip_fuzofft( val, pre, post) + uzoff_t val; + char *pre; + char *post; +{ + /* Storage cylinder. */ + static char fuzofft_buf[ FZOFFT_NUM][ FZOFFT_LEN]; + static int fuzofft_index = 0; + + /* Temporary format string storage. */ + static char fmt[ 16] = "%"; + + /* Assemble the format string. */ + fmt[ 1] = '\0'; /* Start after initial "%". */ + if (pre == FZOFFT_HEX_WID) /* Special hex width. */ + { + strcat( fmt, FZOFFT_HEX_WID_VALUE); + } + else if (pre == FZOFFT_HEX_DOT_WID) /* Special hex ".width". */ + { + strcat( fmt, "."); + strcat( fmt, FZOFFT_HEX_WID_VALUE); + } + else if (pre != NULL) /* Caller's prefix (width). */ + { + strcat( fmt, pre); + } + + strcat( fmt, FZOFFT_FMT); /* Long or long-long or whatever. */ + + if (post == NULL) + strcat( fmt, "u"); /* Default radix = decimal. */ + else + strcat( fmt, post); /* Caller's radix. */ + + /* Advance the cylinder. */ + fuzofft_index = (fuzofft_index+ 1)% FZOFFT_NUM; + + /* Write into the current chamber. */ + sprintf( fuzofft_buf[ fuzofft_index], fmt, val); + + /* Return a pointer to this chamber. */ + return fuzofft_buf[ fuzofft_index]; +} + + +/* Display number to mesg stream + 5/15/05 EG */ + +int DisplayNumString(file, i) + FILE *file; + uzoff_t i; +{ + char tempstrg[100]; + int j; + char *s = tempstrg; + + WriteNumString(i, tempstrg); + /* skip spaces */ + for (j = 0; j < 3; j++) { + if (*s != ' ') break; + s++; + } + fprintf(file, "%s", s); + + return 0; +} + +/* Read numbers with trailing size multiplier (like 10M) and return number. + 10/30/04 EG */ + +uzoff_t ReadNumString( numstring ) + char *numstring; +{ + zoff_t num = 0; + char multchar = ' '; + int i; + uzoff_t mult = 1; + + /* check if valid number (currently no negatives) */ + if (numstring == NULL) { + zipwarn("Unable to read empty number in ReadNumString", ""); + return (uzoff_t)-1; + } + if (numstring[0] < '0' || numstring[0] > '9') { + zipwarn("Unable to read number (must start with digit): ", numstring); + return (uzoff_t)-1; + } + if (strlen(numstring) > 8) { + zipwarn("Number too long to read (8 characters max): ", numstring); + return (uzoff_t)-1; + } + + /* get the number part */ + num = atoi(numstring); + + /* find trailing multiplier */ + for (i = 0; numstring[i] && isdigit(numstring[i]); i++) ; + + /* return if no multiplier */ + if (numstring[i] == '\0') { + return (uzoff_t)num; + } + + /* nothing follows multiplier */ + if (numstring[i + 1]) { + return (uzoff_t)-1; + } + + /* get multiplier */ + multchar = toupper(numstring[i]); + + if (multchar == 'K') { + mult <<= 10; + } else if (multchar == 'M') { + mult <<= 20; + } else if (multchar == 'G') { + mult <<= 30; +#ifdef LARGE_FILE_SUPPORT + } else if (multchar == 'T') { + mult <<= 40; +#endif + } else { + return (uzoff_t)-1; + } + + return (uzoff_t)num * mult; +} + + +/* Write the number as a string with a multiplier (like 10M) to outstring. + Always writes no more than 3 digits followed maybe by a multiplier and + returns the characters written or -1 if error. + 10/30/04 EG */ + +int WriteNumString( num, outstring ) + uzoff_t num; + char *outstring; +{ + int mult; + int written = 0; + int i; + int j; + char digits[4]; + int dig; + + *outstring = '\0'; + + /* shift number 1 K until less than 10000 */ + for (mult = 0; num >= 10240; mult++) { + num >>= 10; + } + + /* write digits as " 0" */ + for (i = 1; i < 4; i++) { + digits[i] = ' '; + } + digits[0] = '0'; + + if (num >= 1000) { + i = 3; + num *= 10; + num >>= 10; + mult++; + digits[0] = (char) (num % 10) + '0'; + digits[1] = '.'; + digits[2] = (char) (num / 10) + '0'; + } else { + for (i = 0; num; i++) { + dig = (int) (num % 10); + num /= 10; + digits[i] = dig + '0'; + } + } + if (i == 0) i = 1; + for (j = i; j > 0; j--) { + *outstring = digits[j - 1]; + outstring++; + written++; + } + + /* output multiplier */ + if (mult == 0) { + } else if (mult == 1) { + *outstring = 'K'; + outstring++; + written++; + } else if (mult == 2) { + *outstring = 'M'; + outstring++; + written++; + } else if (mult == 3) { + *outstring = 'G'; + outstring++; + written++; + } else if (mult == 4) { + *outstring = 'T'; + outstring++; + written++; + } else { + *outstring = '?'; + outstring++; + written++; + } + + *outstring = '\0'; + + return written; +} + + +#if 0 /* not used anywhere, should get removed by next release... */ + +/* Apply the Adler-16 checksum to a set of bytes. + * Use this function as you would use crc32(): + * - First call this function by passing a NULL pointer instead of buf + * OR initialize the checksum register with ADLERVAL_INITIAL. + * - Iteratively call this function for each buffer fragment. + * This function returns the updated checksum. + * + * IN assertion: chksum is a valid Adler-16 checksum: + * (chksum & 0xffU) < ADLER16_BASE && ((chksum >> 8) & 0xffU) < ADLER16_BASE + * + * Author: Cosmin Truta. + * See "proginfo/adler16.txt" for more information. + */ + +#define ADLER16_BASE 251 /* The largest prime smaller than 256 */ + +unsigned int adler16(chksum, buf, len) + unsigned int chksum; + ZCONST uch *buf; + extent len; +{ + unsigned int sum1 = chksum & 0xff; + unsigned int sum2 = (chksum >> 8) & 0xff; + extent i; + + Assert((sum1 < ADLER16_BASE) && (sum2 < ADLER16_BASE), + "adler16: invalid checksum"); + + if (buf == NULL) + return 1; + + for (i = 0; i < len; ++i) + { + sum1 += buf[i]; + if (sum1 >= ADLER16_BASE) /* this is faster than modulo ADLER16_BASE */ + sum1 -= ADLER16_BASE; + sum2 += sum1; + if (sum2 >= ADLER16_BASE) /* ditto */ + sum2 -= ADLER16_BASE; + } + + return (sum2 << 8) | sum1; +} + +#endif /* 0, not used anywhere */ + + +/* returns true if abbrev is abbreviation for matchstring */ +int abbrevmatch (matchstring, abbrev, case_sensitive, minmatch) + char *matchstring; + char *abbrev; + int case_sensitive; + int minmatch; +{ + int cnt = 0; + char *m; + char *a; + + m = matchstring; + a = abbrev; + + for (; *m && *a; m++, a++) { + cnt++; + if (case_sensitive) { + if (*m != *a) { + /* mismatch */ + return 0; + } + } else { + if (toupper(*m) != toupper(*a)) { + /* mismatch */ + return 0; + } + } + } + if (cnt < minmatch) { + /* not big enough string */ + return 0; + } + if (*a != '\0') { + /* abbreviation longer than match string */ + return 0; + } + /* either abbreviation or match */ + return 1; +} diff --git a/third_party/infozip/zip/zbz2err.c b/third_party/infozip/zip/zbz2err.c new file mode 100644 index 000000000..a304f73e1 --- /dev/null +++ b/third_party/infozip/zip/zbz2err.c @@ -0,0 +1,62 @@ +/* clang-format off */ +/* + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/*--------------------------------------------------------------------------- + + zbz2err.c + + This file contains the "fatal error" callback routine required by the + "minimal" (silent, non-stdio) setup of the bzip2 compression library. + + The fatal bzip2 error bail-out routine is provided in a separate code + module, so that it can be easily overridden when the Zip package is + used as a static link library. One example is the WinDLL static library + usage for building a monolithic binary of the Windows application "WiZ" + that supports bzip2 both in compression and decompression operations. + + Contains: bz_internal_error() (BZIP2_SUPPORT only) + + Adapted from UnZip ubz2err.c, with all the DLL fine print stripped + out. + + ---------------------------------------------------------------------------*/ + + +#define __ZBZ2ERR_C /* identifies this source module */ + +#include "zip.h" + +#ifdef BZIP2_SUPPORT +# ifdef BZIP2_USEBZIP2DIR +# include "bzip2/bzlib.h" +# else + /* If IZ_BZIP2 is defined as the location of the bzip2 files then + assume the location has been added to include path. For Unix + this is done by the configure script. */ + /* Also do not need path for bzip2 include if OS includes support + for bzip2 library. */ +# include "bzlib.h" +# endif + +/**********************************/ +/* Function bz_internal_error() */ +/**********************************/ + +/* Call-back function for the bzip2 decompression code (compiled with + * BZ_NO_STDIO), required to handle fatal internal bug-type errors of + * the bzip2 library. + */ +void bz_internal_error(errcode) + int errcode; +{ + sprintf(errbuf, "fatal error (code %d) in bzip2 library", errcode); + ziperr(ZE_LOGIC, errbuf); +} /* end function bz_internal_error() */ + +#endif /* def BZIP2_SUPPORT */ diff --git a/third_party/infozip/zip/zip.c b/third_party/infozip/zip/zip.c new file mode 100644 index 000000000..1c9cfc0a2 --- /dev/null +++ b/third_party/infozip/zip/zip.c @@ -0,0 +1,6022 @@ +/* clang-format off */ +/* + zip.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * zip.c by Mark Adler. + */ +#define __ZIP_C + +#include "third_party/infozip/zip/zip.h" +#include "libc/time/time.h" /* for tzset() declaration */ +#if defined(WIN32) || defined(WINDLL) +# define WIN32_LEAN_AND_MEAN +# include +#endif +#ifdef WINDLL +# include +# include "windll/windll.h" +#endif +#define DEFCPYRT /* main module: enable copyright string defines! */ +#include "third_party/infozip/zip/revision.h" +#include "third_party/infozip/zip/crc32.h" +#include "third_party/infozip/zip/crypt.h" +#include "third_party/infozip/zip/ttyio.h" +#include "libc/str/str.h" +#include "libc/errno.h" +#ifdef VMS +# include +# include "vms/vmsmunch.h" +# include "vms/vms.h" +#endif + +#ifdef MACOS +# include "macglob.h" + extern MacZipGlobals MacZip; + extern int error_level; +#endif + +#if (defined(MSDOS) && !defined(__GO32__)) || defined(__human68k__) +# include +# if (!defined(P_WAIT) && defined(_P_WAIT)) +# define P_WAIT _P_WAIT +# endif +#endif + +#include "libc/calls/calls.h" +#include "libc/fmt/fmt.h" +#include "libc/log/log.h" +#include "libc/stdio/stdio.h" +#include "libc/stdio/temp.h" + +#ifdef UNICODE_TEST +# ifdef WIN32 +# include +# endif +#endif + +#ifdef BZIP2_SUPPORT + /* If IZ_BZIP2 is defined as the location of the bzip2 files then + assume the location has been added to include path. For Unix + this is done by the configure script. */ + /* Also do not need path for bzip2 include if OS includes support + for bzip2 library. */ +# include "bzlib.h" +#endif + +#define MAXCOM 256 /* Maximum one-line comment size */ + + +/* Local option flags */ +#ifndef DELETE +#define DELETE 0 +#endif +#define ADD 1 +#define UPDATE 2 +#define FRESHEN 3 +#define ARCHIVE 4 +local int action = ADD; /* one of ADD, UPDATE, FRESHEN, DELETE, or ARCHIVE */ +local int comadd = 0; /* 1=add comments for new files */ +local int zipedit = 0; /* 1=edit zip comment and all file comments */ +local int latest = 0; /* 1=set zip file time to time of latest file */ +local int test = 0; /* 1=test zip file with unzip -t */ +local char *unzip_path = NULL; /* where to find unzip */ +local int tempdir = 0; /* 1=use temp directory (-b) */ +local int junk_sfx = 0; /* 1=junk the sfx prefix */ +#if defined(AMIGA) || defined(MACOS) +local int filenotes = 0; /* 1=take comments from AmigaDOS/MACOS filenotes */ +#endif + +#ifdef EBCDIC +int aflag = __EBCDIC; /* Convert EBCDIC to ASCII or stay EBCDIC ? */ +#endif +#ifdef CMS_MVS +int bflag = 0; /* Use text mode as default */ +#endif + +#ifdef QDOS +char _version[] = VERSION; +#endif + +#ifdef WINDLL +jmp_buf zipdll_error_return; +#ifdef ZIP64_SUPPORT + unsigned long low, high; /* returning 64 bit values for systems without an _int64 */ + uzoff_t filesize64; +#endif +#endif + +#if CRYPT +/* Pointer to crc_table, needed in crypt.c */ +# if (!defined(USE_ZLIB) || defined(USE_OWN_CRCTAB)) +ZCONST ulg near *crc_32_tab; +# else +ZCONST uLongf *crc_32_tab; +# endif +#endif /* CRYPT */ + +/* Local functions */ + +local void freeup OF((void)); +local int finish OF((int)); +#if (!defined(MACOS) && !defined(WINDLL)) +local void handler OF((int)); +local void license OF((void)); +#ifndef VMSCLI +local void help OF((void)); +local void help_extended OF((void)); +#endif /* !VMSCLI */ +#endif /* !MACOS && !WINDLL */ + +/* prereading of arguments is not supported in new command + line interpreter get_option() so read filters as arguments + are processed and convert to expected array later */ +local int add_filter OF((int flag, char *pattern)); +local int filterlist_to_patterns OF((void)); +/* not used + local int get_filters OF((int argc, char **argv)); +*/ + +/* list to store file arguments */ +local long add_name OF((char *filearg)); + + +local int DisplayRunningStats OF((void)); +local int BlankRunningStats OF((void)); + +#if !defined(WINDLL) +local void version_info OF((void)); +# if !defined(MACOS) +local void zipstdout OF((void)); +# endif /* !MACOS */ +local int check_unzip_version OF((char *unzippath)); +local void check_zipfile OF((char *zipname, char *zippath)); +#endif /* !WINDLL */ + +/* structure used by add_filter to store filters */ +struct filterlist_struct { + char flag; + char *pattern; + struct filterlist_struct *next; +}; +struct filterlist_struct *filterlist = NULL; /* start of list */ +struct filterlist_struct *lastfilter = NULL; /* last filter in list */ + +/* structure used by add_filearg to store file arguments */ +struct filelist_struct { + char *name; + struct filelist_struct *next; +}; +long filearg_count = 0; +struct filelist_struct *filelist = NULL; /* start of list */ +struct filelist_struct *lastfile = NULL; /* last file in list */ + +local void freeup() +/* Free all allocations in the 'found' list, the 'zfiles' list and + the 'patterns' list. */ +{ + struct flist far *f; /* steps through found list */ + struct zlist far *z; /* pointer to next entry in zfiles list */ + + for (f = found; f != NULL; f = fexpel(f)) + ; + while (zfiles != NULL) + { + z = zfiles->nxt; + if (zfiles->zname && zfiles->zname != zfiles->name) + free((zvoid *)(zfiles->zname)); + if (zfiles->name) + free((zvoid *)(zfiles->name)); + if (zfiles->iname) + free((zvoid *)(zfiles->iname)); + if (zfiles->cext && zfiles->cextra && zfiles->cextra != zfiles->extra) + free((zvoid *)(zfiles->cextra)); + if (zfiles->ext && zfiles->extra) + free((zvoid *)(zfiles->extra)); + if (zfiles->com && zfiles->comment) + free((zvoid *)(zfiles->comment)); + if (zfiles->oname) + free((zvoid *)(zfiles->oname)); +#ifdef UNICODE_SUPPORT + if (zfiles->uname) + free((zvoid *)(zfiles->uname)); + if (zfiles->zuname) + free((zvoid *)(zfiles->zuname)); + if (zfiles->ouname) + free((zvoid *)(zfiles->ouname)); +# ifdef WIN32 + if (zfiles->namew) + free((zvoid *)(zfiles->namew)); + if (zfiles->inamew) + free((zvoid *)(zfiles->inamew)); + if (zfiles->znamew) + free((zvoid *)(zfiles->znamew)); +# endif +#endif + farfree((zvoid far *)zfiles); + zfiles = z; + zcount--; + } + + if (patterns != NULL) { + while (pcount-- > 0) { + if (patterns[pcount].zname != NULL) + free((zvoid *)(patterns[pcount].zname)); + } + free((zvoid *)patterns); + patterns = NULL; + } + + /* close logfile */ + if (logfile) { + fclose(logfile); + } +} + +local int finish(e) +int e; /* exit code */ +/* Process -o and -m options (if specified), free up malloc'ed stuff, and + exit with the code e. */ +{ + int r; /* return value from trash() */ + ulg t; /* latest time in zip file */ + struct zlist far *z; /* pointer into zfile list */ + + /* If latest, set time to zip file to latest file in zip file */ + if (latest && zipfile && strcmp(zipfile, "-")) + { + diag("changing time of zip file to time of latest file in it"); + /* find latest time in zip file */ + if (zfiles == NULL) + zipwarn("zip file is empty, can't make it as old as latest entry", ""); + else { + t = 0; + for (z = zfiles; z != NULL; z = z->nxt) + /* Ignore directories in time comparisons */ +#ifdef USE_EF_UT_TIME + if (z->iname[z->nam-1] != (char)0x2f) /* ascii '/' */ + { + iztimes z_utim; + ulg z_tim; + + z_tim = ((get_ef_ut_ztime(z, &z_utim) & EB_UT_FL_MTIME) ? + unix2dostime(&z_utim.mtime) : z->tim); + if (t < z_tim) + t = z_tim; + } +#else /* !USE_EF_UT_TIME */ + if (z->iname[z->nam-1] != (char)0x2f /* ascii '/' */ + && t < z->tim) + t = z->tim; +#endif /* ?USE_EF_UT_TIME */ + /* set modified time of zip file to that time */ + if (t != 0) + stamp(zipfile, t); + else + zipwarn( + "zip file has only directories, can't make it as old as latest entry", + ""); + } + } + if (tempath != NULL) + { + free((zvoid *)tempath); + tempath = NULL; + } + if (zipfile != NULL) + { + free((zvoid *)zipfile); + zipfile = NULL; + } + if (in_file != NULL) + { + fclose(in_file); + in_file = NULL; + } + if (in_path != NULL) + { + free((zvoid *)in_path); + in_path = NULL; + } + if (out_path != NULL) + { + free((zvoid *)out_path); + out_path = NULL; + } + if (zcomment != NULL) + { + free((zvoid *)zcomment); + zcomment = NULL; + } + + + /* If dispose, delete all files in the zfiles list that are marked */ + if (dispose) + { + diag("deleting files that were added to zip file"); + if ((r = trash()) != ZE_OK) + ZIPERR(r, "was deleting moved files and directories"); + } + + + /* Done! */ + freeup(); + return e; +} + +void ziperr(c, h) +int c; /* error code from the ZE_ class */ +ZCONST char *h; /* message about how it happened */ +/* Issue a message for the error, clean up files and memory, and exit. */ +{ +#ifndef WINDLL +#ifndef MACOS + static int error_level = 0; +#endif + + if (error_level++ > 0) + /* avoid recursive ziperr() printouts (his should never happen) */ + EXIT(ZE_LOGIC); /* ziperr recursion is an internal logic error! */ +#endif /* !WINDLL */ + + if (mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } + if (logfile && logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + } + if (h != NULL) { + if (PERR(c)) + fprintf(mesg, "zip I/O error: %s", strerror(errno)); + /* perror("zip I/O error"); */ + fflush(mesg); + fprintf(mesg, "\nzip error: %s (%s)\n", ZIPERRORS(c), h); +#ifdef DOS + check_for_windows("Zip"); +#endif + if (logfile) { + if (PERR(c)) + fprintf(logfile, "zip I/O error: %s\n", strerror(errno)); + fprintf(logfile, "\nzip error: %s (%s)\n", ZIPERRORS(c), h); + logfile_line_started = 0; + } + } + if (tempzip != NULL) + { + if (tempzip != zipfile) { + if (current_local_file) + fclose(current_local_file); + if (y != current_local_file && y != NULL) + fclose(y); +#ifndef DEBUG + destroy(tempzip); +#endif + free((zvoid *)tempzip); + } else { + /* -g option, attempt to restore the old file */ + + /* zip64 support 09/05/2003 R.Nausedat */ + uzoff_t k = 0; /* keep count for end header */ + uzoff_t cb = cenbeg; /* get start of central */ + + struct zlist far *z; /* steps through zfiles linked list */ + + fprintf(mesg, "attempting to restore %s to its previous state\n", + zipfile); + if (logfile) + fprintf(logfile, "attempting to restore %s to its previous state\n", + zipfile); + + zfseeko(y, cenbeg, SEEK_SET); + + tempzn = cenbeg; + for (z = zfiles; z != NULL; z = z->nxt) + { + putcentral(z); + tempzn += 4 + CENHEAD + z->nam + z->cext + z->com; + k++; + } + putend(k, tempzn - cb, cb, zcomlen, zcomment); + fclose(y); + y = NULL; + } + } + + if (key != NULL) { + free((zvoid *)key); + key = NULL; + } + if (tempath != NULL) { + free((zvoid *)tempath); + tempath = NULL; + } + if (zipfile != NULL) { + free((zvoid *)zipfile); + zipfile = NULL; + } + if (out_path != NULL) { + free((zvoid *)out_path); + out_path = NULL; + } + if (zcomment != NULL) { + free((zvoid *)zcomment); + zcomment = NULL; + } + + freeup(); +#ifndef WINDLL + EXIT(c); +#else + longjmp(zipdll_error_return, c); +#endif +} + + +void error(h) + ZCONST char *h; +/* Internal error, should never happen */ +{ + ziperr(ZE_LOGIC, h); +} + +#if (!defined(MACOS) && !defined(WINDLL)) +local void handler(s) +int s; /* signal number (ignored) */ +/* Upon getting a user interrupt, turn echo back on for tty and abort + cleanly using ziperr(). */ +{ +#if defined(AMIGA) && defined(__SASC) + _abort(); +#else +#if !defined(MSDOS) && !defined(__human68k__) && !defined(RISCOS) + echon(); + putc('\n', mesg); +#endif /* !MSDOS */ +#endif /* AMIGA && __SASC */ + ziperr(ZE_ABORT, "aborting"); + s++; /* keep some compilers happy */ +} +#endif /* !MACOS && !WINDLL */ + +void zipmessage_nl(a, nl) +ZCONST char *a; /* message string to output */ +int nl; /* 1 = add nl to end */ +/* If nl false, print a message to mesg without new line. + If nl true, print and add new line. If logfile is + open then also write message to log file. */ +{ + if (noisy) { + if (a && strlen(a)) { + fprintf(mesg, "%s", a); + mesg_line_started = 1; + } + if (nl) { + if (mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } + } else if (a && strlen(a)) { + mesg_line_started = 1; + } + fflush(mesg); + } + if (logfile) { + if (a && strlen(a)) { + fprintf(logfile, "%s", a); + logfile_line_started = 1; + } + if (nl) { + if (logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + } + } else if (a && strlen(a)) { + logfile_line_started = 1; + } + fflush(logfile); + } +} + +void zipmessage(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a message to mesg and flush. Also write to log file if + open. Write new line first if current line has output already. */ +{ + if (noisy) { + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "%s%s\n", a, b); + mesg_line_started = 0; + fflush(mesg); + } + if (logfile) { + if (logfile_line_started) + fprintf(logfile, "\n"); + fprintf(logfile, "%s%s\n", a, b); + logfile_line_started = 0; + fflush(logfile); + } +} + +void zipwarn(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a warning message to mesg (usually stderr) and return. */ +{ + if (noisy) { + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "\tzip warning: %s%s\n", a, b); + mesg_line_started = 0; + fflush(mesg); + } + if (logfile) { + if (logfile_line_started) + fprintf(logfile, "\n"); + fprintf(logfile, "\tzip warning: %s%s\n", a, b); + logfile_line_started = 0; + fflush(logfile); + } +} + +#ifndef WINDLL +local void license() +/* Print license information to stdout. */ +{ + extent i; /* counter for copyright array */ + + for (i = 0; i < sizeof(swlicense)/sizeof(char *); i++) + puts(swlicense[i]); +} + +#ifdef VMSCLI +void help() +#else +local void help() +#endif +/* Print help (along with license info) to stdout. */ +{ + extent i; /* counter for help array */ + + /* help array */ + static ZCONST char *text[] = { +#ifdef VMS +"Zip %s (%s). Usage: zip == \"$ disk:[dir]zip.exe\"", +#else +"Zip %s (%s). Usage:", +#endif +#ifdef MACOS +"zip [-options] [-b fm] [-t mmddyyyy] [-n suffixes] [zipfile list] [-xi list]", +" The default action is to add or replace zipfile entries from list.", +" ", +" -f freshen: only changed files -u update: only changed or new files", +" -d delete entries in zipfile -m move into zipfile (delete OS files)", +" -r recurse into directories -j junk (don't record) directory names", +" -0 store only -l convert LF to CR LF (-ll CR LF to LF)", +" -1 compress faster -9 compress better", +" -q quiet operation -v verbose operation/print version info", +" -c add one-line comments -z add zipfile comment", +" -o make zipfile as old as latest entry", +" -F fix zipfile (-FF try harder) -D do not add directory entries", +" -T test zipfile integrity -X eXclude eXtra file attributes", +# if CRYPT +" -e encrypt -n don't compress these suffixes" +# else +" -h show this help -n don't compress these suffixes" +# endif +," -h2 show more help", +" Macintosh specific:", +" -jj record Fullpath (+ Volname) -N store finder-comments as comments", +" -df zip only datafork of a file -S include finder invisible/system files" +#else /* !MACOS */ +#ifdef VM_CMS +"zip [-options] [-b fm] [-t mmddyyyy] [-n suffixes] [zipfile list] [-xi list]", +#else /* !VM_CMS */ +"zip [-options] [-b path] [-t mmddyyyy] [-n suffixes] [zipfile list] [-xi list]", +#endif /* ?VM_CMS */ +" The default action is to add or replace zipfile entries from list, which", +" can include the special name - to compress standard input.", +" If zipfile and list are omitted, zip compresses stdin to stdout.", +" -f freshen: only changed files -u update: only changed or new files", +" -d delete entries in zipfile -m move into zipfile (delete OS files)", +" -r recurse into directories -j junk (don't record) directory names", +#ifdef THEOS +" -0 store only -l convert CR to CR LF (-ll CR LF to CR)", +#else +" -0 store only -l convert LF to CR LF (-ll CR LF to LF)", +#endif +" -1 compress faster -9 compress better", +" -q quiet operation -v verbose operation/print version info", +" -c add one-line comments -z add zipfile comment", +" -@ read names from stdin -o make zipfile as old as latest entry", +" -x exclude the following names -i include only the following names", +#ifdef EBCDIC +#ifdef CMS_MVS +" -a translate to ASCII -B force binary read (text is default)", +#else /* !CMS_MVS */ +" -a translate to ASCII", +#endif /* ?CMS_MVS */ +#endif /* EBCDIC */ +#ifdef TANDEM +" -Bn set Enscribe formatting options", +#endif +" -F fix zipfile (-FF try harder) -D do not add directory entries", +" -A adjust self-extracting exe -J junk zipfile prefix (unzipsfx)", +" -T test zipfile integrity -X eXclude eXtra file attributes", +#ifdef VMS +" -C preserve case of file names -C- down-case all file names", +" -C2 preserve case of ODS2 names -C2- down-case ODS2 file names* (*=default)", +" -C5 preserve case of ODS5 names* -C5- down-case ODS5 file names", +" -V save VMS file attributes (-VV also save allocated blocks past EOF)", +" -w store file version numbers\ + -ww store file version numbers as \".nnn\"", +#endif /* def VMS */ +#ifdef NTSD_EAS +" -! use privileges (if granted) to obtain all aspects of WinNT security", +#endif /* NTSD_EAS */ +#ifdef OS2 +" -E use the .LONGNAME Extended attribute (if found) as filename", +#endif /* OS2 */ +#ifdef S_IFLNK +" -y store symbolic links as the link instead of the referenced file", +#endif /* !S_IFLNK */ +/* +" -R PKZIP recursion (see manual)", +*/ +#if defined(MSDOS) || defined(OS2) +" -$ include volume label -S include system and hidden files", +#endif +#ifdef AMIGA +# if CRYPT +" -N store filenotes as comments -e encrypt", +" -h show this help -n don't compress these suffixes" +# else +" -N store filenotes as comments -n don't compress these suffixes" +# endif +#else /* !AMIGA */ +# if CRYPT +" -e encrypt -n don't compress these suffixes" +# else +" -h show this help -n don't compress these suffixes" +# endif +#endif /* ?AMIGA */ +#ifdef RISCOS +," -h2 show more help -I don't scan thru Image files" +#else +," -h2 show more help" +#endif +#endif /* ?MACOS */ +#ifdef VMS +," (Must quote upper-case options, like \"-V\", unless SET PROC/PARSE=EXTEND)" +#endif /* def VMS */ +," " + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) + { + printf(copyright[i], "zip"); + putchar('\n'); + } + for (i = 0; i < sizeof(text)/sizeof(char *); i++) + { + printf(text[i], VERSION, REVDATE); + putchar('\n'); + } +} + +#ifdef VMSCLI +void help_extended() +#else +local void help_extended() +#endif +/* Print extended help to stdout. */ +{ + extent i; /* counter for help array */ + + /* help array */ + static ZCONST char *text[] = { +"", +"Extended Help for Zip", +"", +"See the Zip Manual for more detailed help", +"", +"", +"Zip stores files in zip archives. The default action is to add or replace", +"zipfile entries.", +"", +"Basic command line:", +" zip options archive_name file file ...", +"", +"Some examples:", +" Add file.txt to z.zip (create z if needed): zip z file.txt", +" Zip all files in current dir: zip z *", +" Zip files in current dir and subdirs also: zip -r z .", +"", +"Basic modes:", +" External modes (selects files from file system):", +" add - add new files/update existing files in archive (default)", +" -u update - add new files/update existing files only if later date", +" -f freshen - update existing files only (no files added)", +" -FS filesync - update if date or size changed, delete if no OS match", +" Internal modes (selects entries in archive):", +" -d delete - delete files from archive (see below)", +" -U copy - select files in archive to copy (use with --out)", +"", +"Basic options:", +" -r recurse into directories (see Recursion below)", +" -m after archive created, delete original files (move into archive)", +" -j junk directory names (store just file names)", +" -q quiet operation", +" -v verbose operation (just \"zip -v\" shows version information)", +" -c prompt for one-line comment for each entry", +" -z prompt for comment for archive (end with just \".\" line or EOF)", +" -@ read names to zip from stdin (one path per line)", +" -o make zipfile as old as latest entry", +"", +"", +"Syntax:", +" The full command line syntax is:", +"", +" zip [-shortopts ...] [--longopt ...] [zipfile [path path ...]] [-xi list]", +"", +" Any number of short option and long option arguments are allowed", +" (within limits) as well as any number of path arguments for files", +" to zip up. If zipfile exists, the archive is read in. If zipfile", +" is \"-\", stream to stdout. If any path is \"-\", zip stdin.", +"", +"Options and Values:", +" For short options that take values, use -ovalue or -o value or -o=value", +" For long option values, use either --longoption=value or --longoption value", +" For example:", +" zip -ds 10 --temp-dir=path zipfile path1 path2 --exclude pattern pattern", +" Avoid -ovalue (no space between) to avoid confusion", +" In particular, be aware of 2-character options. For example:", +" -d -s is (delete, split size) while -ds is (dot size)", +" Usually better to break short options across multiple arguments by function", +" zip -r -dbdcds 10m -lilalf logfile archive input_directory -ll", +"", +" All args after just \"--\" arg are read verbatim as paths and not options.", +" zip zipfile path path ... -- verbatimpath verbatimpath ...", +" Use -nw to also disable wildcards, so paths are read literally:", +" zip zipfile -nw -- \"-leadingdashpath\" \"a[path].c\" \"path*withwildcard\"", +" You may still have to escape or quote arguments to avoid shell expansion", +"", +"Wildcards:", +" Internally zip supports the following wildcards:", +" ? (or %% or #, depending on OS) matches any single character", +" * matches any number of characters, including zero", +" [list] matches char in list (regex), can do range [ac-f], all but [!bf]", +" If port supports [], must escape [ as [[] or use -nw to turn off wildcards", +" For shells that expand wildcards, escape (\\* or \"*\") so zip can recurse", +" zip zipfile -r . -i \"*.h\"", +"", +" Normally * crosses dir bounds in path, e.g. 'a*b' can match 'ac/db'. If", +" -ws option used, * does not cross dir bounds but ** does", +"", +" For DOS and Windows, [list] is now disabled unless the new option", +" -RE enable [list] (regular expression) matching", +" is used to avoid problems with file paths containing \"[\" and \"]\":", +" zip files_ending_with_number -RE foo[0-9].c", +"", +"Include and Exclude:", +" -i pattern pattern ... include files that match a pattern", +" -x pattern pattern ... exclude files that match a pattern", +" Patterns are paths with optional wildcards and match paths as stored in", +" archive. Exclude and include lists end at next option, @, or end of line.", +" zip -x pattern pattern @ zipfile path path ...", +"", +"Case matching:", +" On most OS the case of patterns must match the case in the archive, unless", +" the -ic option is used.", +" -ic ignore case of archive entries", +" This option not available on case-sensitive file systems. On others, case", +" ignored when matching files on file system but matching against archive", +" entries remains case sensitive for modes -f (freshen), -U (archive copy),", +" and -d (delete) because archive paths are always case sensitive. With", +" -ic, all matching ignores case, but it's then possible multiple archive", +" entries that differ only in case will match.", +"", +"End Of Line Translation (text files only):", +" -l change CR or LF (depending on OS) line end to CR LF (Unix->Win)", +" -ll change CR LF to CR or LF (depending on OS) line end (Win->Unix)", +" If first buffer read from file contains binary the translation is skipped", +"", +"Recursion:", +" -r recurse paths, include files in subdirs: zip -r a path path ...", +" -R recurse current dir and match patterns: zip -R a ptn ptn ...", +" Use -i and -x with either to include or exclude paths", +" Path root in archive starts at current dir, so if /a/b/c/file and", +" current dir is /a/b, 'zip -r archive .' puts c/file in archive", +"", +"Date filtering:", +" -t date exclude before (include files modified on this date and later)", +" -tt date include before (include files modified before date)", +" Can use both at same time to set a date range", +" Dates are mmddyyyy or yyyy-mm-dd", +"", +"Deletion, File Sync:", +" -d delete files", +" Delete archive entries matching internal archive paths in list", +" zip archive -d pattern pattern ...", +" Can use -t and -tt to select files in archive, but NOT -x or -i, so", +" zip archive -d \"*\" -t 2005-12-27", +" deletes all files from archive.zip with date of 27 Dec 2005 and later", +" Note the * (escape as \"*\" on Unix) to select all files in archive", +"", +" -FS file sync", +" Similar to update, but files updated if date or size of entry does not", +" match file on OS. Also deletes entry from archive if no matching file", +" on OS.", +" zip archive_to_update -FS -r dir_used_before", +" Result generally same as creating new archive, but unchanged entries", +" are copied instead of being read and compressed so can be faster.", +" WARNING: -FS deletes entries so make backup copy of archive first", +"", +"Compression:", +" -0 store files (no compression)", +" -1 to -9 compress fastest to compress best (default is 6)", +" -Z cm set compression method to cm:", +" store - store without compression, same as option -0", +" deflate - original zip deflate, same as -1 to -9 (default)", +" if bzip2 is enabled:", +" bzip2 - use bzip2 compression (need modern unzip)", +"", +"Encryption:", +" -e use standard (weak) PKZip 2.0 encryption, prompt for password", +" -P pswd use standard encryption, password is pswd", +"", +"Splits (archives created as a set of split files):", +" -s ssize create split archive with splits of size ssize, where ssize nm", +" n number and m multiplier (kmgt, default m), 100k -> 100 kB", +" -sp pause after each split closed to allow changing disks", +" WARNING: Archives created with -sp use data descriptors and should", +" work with most unzips but may not work with some", +" -sb ring bell when pause", +" -sv be verbose about creating splits", +" Split archives CANNOT be updated, but see --out and Copy Mode below", +"", +"Using --out (output to new archive):", +" --out oa output to new archive oa", +" Instead of updating input archive, create new output archive oa.", +" Result is same as without --out but in new archive. Input archive", +" unchanged.", +" WARNING: --out ALWAYS overwrites any existing output file", +" For example, to create new_archive like old_archive but add newfile1", +" and newfile2:", +" zip old_archive newfile1 newfile2 --out new_archive", +" Cannot update split archive, so use --out to out new archive:", +" zip in_split_archive newfile1 newfile2 --out out_split_archive", +" If input is split, output will default to same split size", +" Use -s=0 or -s- to turn off splitting to convert split to single file:", +" zip in_split_archive -s 0 --out out_single_file_archive", +" WARNING: If overwriting old split archive but need less splits,", +" old splits not overwritten are not needed but remain", +"", +"Copy Mode (copying from archive to archive):", +" -U (also --copy) select entries in archive to copy (reverse delete)", +" Copy Mode copies entries from old to new archive with --out and is used by", +" zip when either no input files on command line or -U (--copy) used.", +" zip inarchive --copy pattern pattern ... --out outarchive", +" To copy only files matching *.c into new archive, excluding foo.c:", +" zip old_archive --copy \"*.c\" --out new_archive -x foo.c", +" If no input files and --out, copy all entries in old archive:", +" zip old_archive --out new_archive", +"", +"Streaming and FIFOs:", +" prog1 | zip -ll z - zip output of prog1 to zipfile z, converting CR LF", +" zip - -R \"*.c\" | prog2 zip *.c files in current dir and stream to prog2 ", +" prog1 | zip | prog2 zip in pipe with no in or out acts like zip - -", +" If Zip is Zip64 enabled, streaming stdin creates Zip64 archives by default", +" that need PKZip 4.5 unzipper like UnZip 6.0", +" WARNING: Some archives created with streaming use data descriptors and", +" should work with most unzips but may not work with some", +" Can use -fz- to turn off Zip64 if input not large (< 4 GB):", +" prog_with_small_output | zip archive -fz-", +"", +" Zip now can read Unix FIFO (named pipes). Off by default to prevent zip", +" from stopping unexpectedly on unfed pipe, use -FI to enable:", +" zip -FI archive fifo", +"", +"Dots, counts:", +" -db display running count of bytes processed and bytes to go", +" (uncompressed size, except delete and copy show stored size)", +" -dc display running count of entries done and entries to go", +" -dd display dots every 10 MB (or dot size) while processing files", +" -dg display dots globally for archive instead of for each file", +" zip -qdgds 10m will turn off most output except dots every 10 MB", +" -ds siz each dot is siz processed where siz is nm as splits (0 no dots)", +" -du display original uncompressed size for each entry as added", +" -dv display volume (disk) number in format in_disk>out_disk", +" Dot size is approximate, especially for dot sizes less than 1 MB", +" Dot options don't apply to Scanning files dots (dot/2sec) (-q turns off)", +"", +"Logging:", +" -lf path open file at path as logfile (overwrite existing file)", +" -la append to existing logfile", +" -li include info messages (default just warnings and errors)", +"", +"Testing archives:", +" -T test completed temp archive with unzip before updating archive", +" -TT cmd use command cmd instead of 'unzip -tqq' to test archive", +" On Unix, to use unzip in current directory, could use:", +" zip archive file1 file2 -T -TT \"./unzip -tqq\"", +" In cmd, {} replaced by temp archive path, else temp appended.", +" The return code is checked for success (0 on Unix)", +"", +"Fixing archives:", +" -F attempt to fix a mostly intact archive (try this first)", +" -FF try to salvage what can (may get more but less reliable)", +" Fix options copy entries from potentially bad archive to new archive.", +" -F tries to read archive normally and copy only intact entries, while", +" -FF tries to salvage what can and may result in incomplete entries.", +" Must use --out option to specify output archive:", +" zip -F bad.zip --out fixed.zip", +" Use -v (verbose) with -FF to see details:", +" zip reallybad.zip -FF -v --out fixed.zip", +" Currently neither option fixes bad entries, as from text mode ftp get.", +"", +"Difference mode:", +" -DF (also --dif) only include files that have changed or are", +" new as compared to the input archive", +" Difference mode can be used to create incremental backups. For example:", +" zip --dif full_backup.zip -r somedir --out diff.zip", +" will store all new files, as well as any files in full_backup.zip where", +" either file time or size have changed from that in full_backup.zip,", +" in new diff.zip. Output archive not excluded automatically if exists,", +" so either use -x to exclude it or put outside what is being zipped.", +"", +"DOS Archive bit (Windows only):", +" -AS include only files with the DOS Archive bit set", +" -AC after archive created, clear archive bit of included files", +" WARNING: Once the archive bits are cleared they are cleared", +" Use -T to test the archive before the bits are cleared", +" Can also use -sf to save file list before zipping files", +"", +"Show files:", +" -sf show files to operate on and exit (-sf- logfile only)", +" -su as -sf but show escaped UTF-8 Unicode names also if exist", +" -sU as -sf but show escaped UTF-8 Unicode names instead", +" Any character not in the current locale is escaped as #Uxxxx, where x", +" is hex digit, if 16-bit code is sufficient, or #Lxxxxxx if 24-bits", +" are needed. If add -UN=e, Zip escapes all non-ASCII characters.", +"", +"Unicode:", +" If compiled with Unicode support, Zip stores UTF-8 path of entries.", +" This is backward compatible. Unicode paths allow better conversion", +" of entry names between different character sets.", +"", +" New Unicode extra field includes checksum to verify Unicode path", +" goes with standard path for that entry (as utilities like ZipNote", +" can rename entries). If these do not match, use below options to", +" set what Zip does:", +" -UN=Quit - if mismatch, exit with error", +" -UN=Warn - if mismatch, warn, ignore UTF-8 (default)", +" -UN=Ignore - if mismatch, quietly ignore UTF-8", +" -UN=No - ignore any UTF-8 paths, use standard paths for all", +" An exception to -UN=N are entries with new UTF-8 bit set (instead", +" of using extra fields). These are always handled as Unicode.", +"", +" Normally Zip escapes all chars outside current char set, but leaves", +" as is supported chars, which may not be OK in path names. -UN=Escape", +" escapes any character not ASCII:", +" zip -sU -UN=e archive", +" Can use either normal path or escaped Unicode path on command line", +" to match files in archive.", +"", +" Zip now stores UTF-8 in entry path and comment fields on systems", +" where UTF-8 char set is default, such as most modern Unix, and", +" and on other systems in new extra fields with escaped versions in", +" entry path and comment fields for backward compatibility.", +" Option -UN=UTF8 will force storing UTF-8 in entry path and comment", +" fields:", +" -UN=UTF8 - store UTF-8 in entry path and comment fields", +" This option can be useful for multi-byte char sets on Windows where", +" escaped paths and comments can be too long to be valid as the UTF-8", +" versions tend to be shorter.", +"", +" Only UTF-8 comments on UTF-8 native systems supported. UTF-8 comments", +" for other systems planned in next release.", +"", +"Self extractor:", +" -A Adjust offsets - a self extractor is created by prepending", +" the extractor executable to archive, but internal offsets", +" are then off. Use -A to fix offsets.", +" -J Junk sfx - removes prepended extractor executable from", +" self extractor, leaving a plain zip archive.", +"", +"More option highlights (see manual for additional options and details):", +" -b dir when creating or updating archive, create the temp archive in", +" dir, which allows using seekable temp file when writing to a", +" write once CD, such archives compatible with more unzips", +" (could require additional file copy if on another device)", +" -MM input patterns must match at least one file and matched files", +" must be readable or exit with OPEN error and abort archive", +" (without -MM, both are warnings only, and if unreadable files", +" are skipped OPEN error (18) returned after archive created)", +" -nw no wildcards (wildcards are like any other character)", +" -sc show command line arguments as processed and exit", +" -sd show debugging as Zip does each step", +" -so show all available options on this system", +" -X default=strip old extra fields, -X- keep old, -X strip most", +" -ws wildcards don't span directory boundaries in paths", +"" + }; + + for (i = 0; i < sizeof(text)/sizeof(char *); i++) + { + printf(text[i]); + putchar('\n'); + } +#ifdef DOS + check_for_windows("Zip"); +#endif +} + +/* + * XXX version_info() in a separate file + */ +local void version_info() +/* Print verbose info about program version and compile time options + to stdout. */ +{ + extent i; /* counter in text arrays */ + char *envptr; + + /* Bzip2 option string storage (with version). */ + +#ifdef BZIP2_SUPPORT + static char bz_opt_ver[81]; + static char bz_opt_ver2[81]; + static char bz_opt_ver3[81]; +#endif + + /* Options info array */ + static ZCONST char *comp_opts[] = { +#ifdef ASM_CRC + "ASM_CRC", +#endif +#ifdef ASMV + "ASMV", +#endif +#ifdef DYN_ALLOC + "DYN_ALLOC", +#endif +#ifdef MMAP + "MMAP", +#endif +#ifdef BIG_MEM + "BIG_MEM", +#endif +#ifdef MEDIUM_MEM + "MEDIUM_MEM", +#endif +#ifdef SMALL_MEM + "SMALL_MEM", +#endif +#ifdef DEBUG + "DEBUG", +#endif +#ifdef USE_EF_UT_TIME + "USE_EF_UT_TIME (store Universal Time)", +#endif +#ifdef NTSD_EAS + "NTSD_EAS (store NT Security Descriptor)", +#endif +#if defined(WIN32) && defined(NO_W32TIMES_IZFIX) + "NO_W32TIMES_IZFIX", +#endif +#ifdef VMS +#ifdef VMSCLI + "VMSCLI", +#endif +#ifdef VMS_IM_EXTRA + "VMS_IM_EXTRA", +#endif +#ifdef VMS_PK_EXTRA + "VMS_PK_EXTRA", +#endif +#endif /* VMS */ +#ifdef WILD_STOP_AT_DIR + "WILD_STOP_AT_DIR (wildcards do not cross directory boundaries)", +#endif +#ifdef WIN32_OEM + "WIN32_OEM (store file paths on Windows as OEM)", +#endif +#ifdef BZIP2_SUPPORT + bz_opt_ver, + bz_opt_ver2, + bz_opt_ver3, +#endif +#ifdef S_IFLNK +# ifdef VMS + "SYMLINK_SUPPORT (symbolic links supported, if C RTL permits)", +# else + "SYMLINK_SUPPORT (symbolic links supported)", +# endif +#endif +#ifdef LARGE_FILE_SUPPORT +# ifdef USING_DEFAULT_LARGE_FILE_SUPPORT + "LARGE_FILE_SUPPORT (default settings)", +# else + "LARGE_FILE_SUPPORT (can read and write large files on file system)", +# endif +#endif +#ifdef ZIP64_SUPPORT + "ZIP64_SUPPORT (use Zip64 to store large files in archives)", +#endif +#ifdef UNICODE_SUPPORT + "UNICODE_SUPPORT (store and read UTF-8 Unicode paths)", +#endif + +#ifdef UNIX + "STORE_UNIX_UIDs_GIDs (store UID/GID sizes/values using new extra field)", +# ifdef UIDGID_NOT_16BIT + "UIDGID_NOT_16BIT (old Unix 16-bit UID/GID extra field not used)", +# else + "UIDGID_16BIT (old Unix 16-bit UID/GID extra field also used)", +# endif +#endif + +#if CRYPT && defined(PASSWD_FROM_STDIN) + "PASSWD_FROM_STDIN", +#endif /* CRYPT & PASSWD_FROM_STDIN */ + NULL + }; + + static ZCONST char *zipenv_names[] = { +#ifndef VMS +# ifndef RISCOS + "ZIP" +# else /* RISCOS */ + "Zip$Options" +# endif /* ?RISCOS */ +#else /* VMS */ + "ZIP_OPTS" +#endif /* ?VMS */ + ,"ZIPOPT" +#ifdef AZTEC_C + , /* extremely lame compiler bug workaround */ +#endif +#ifndef __RSXNT__ +# ifdef __EMX__ + ,"EMX" + ,"EMXOPT" +# endif +# if (defined(__GO32__) && (!defined(__DJGPP__) || __DJGPP__ < 2)) + ,"GO32" + ,"GO32TMP" +# endif +# if (defined(__DJGPP__) && __DJGPP__ >= 2) + ,"TMPDIR" +# endif +#endif /* !__RSXNT__ */ +#ifdef RISCOS + ,"Zip$Exts" +#endif + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) + { + printf(copyright[i], "zip"); + putchar('\n'); + } + + for (i = 0; i < sizeof(versinfolines)/sizeof(char *); i++) + { + printf(versinfolines[i], "Zip", VERSION, REVDATE); + putchar('\n'); + } + + version_local(); + + puts("Zip special compilation options:"); +#if WSIZE != 0x8000 + printf("\tWSIZE=%u\n", WSIZE); +#endif + + /* Fill in bzip2 version. (32-char limit valid as of bzip 1.0.3.) */ +#ifdef BZIP2_SUPPORT + sprintf( bz_opt_ver, + "BZIP2_SUPPORT (bzip2 library version %.32s)", BZ2_bzlibVersion()); + sprintf( bz_opt_ver2, + " bzip2 code and library copyright (c) Julian R Seward"); + sprintf( bz_opt_ver3, + " (See the bzip2 license for terms of use)"); +#endif + + for (i = 0; (int)i < (int)(sizeof(comp_opts)/sizeof(char *) - 1); i++) + { + printf("\t%s\n",comp_opts[i]); + } +#ifdef USE_ZLIB + if (strcmp(ZLIB_VERSION, zlibVersion()) == 0) + printf("\tUSE_ZLIB [zlib version %s]\n", ZLIB_VERSION); + else + printf("\tUSE_ZLIB [compiled with version %s, using version %s]\n", + ZLIB_VERSION, zlibVersion()); + i++; /* zlib use means there IS at least one compilation option */ +#endif +#if CRYPT + printf("\t[encryption, version %d.%d%s of %s] (modified for Zip 3)\n\n", + CR_MAJORVER, CR_MINORVER, CR_BETA_VER, CR_VERSION_DATE); + for (i = 0; i < sizeof(cryptnote)/sizeof(char *); i++) + { + printf(cryptnote[i]); + putchar('\n'); + } + ++i; /* crypt support means there IS at least one compilation option */ +#endif /* CRYPT */ + if (i == 0) + puts("\t[none]"); + + puts("\nZip environment options:"); + for (i = 0; i < sizeof(zipenv_names)/sizeof(char *); i++) + { + envptr = getenv(zipenv_names[i]); + printf("%16s: %s\n", zipenv_names[i], + ((envptr == (char *)NULL || *envptr == 0) ? "[none]" : envptr)); + } +#ifdef DOS + check_for_windows("Zip"); +#endif +} +#endif /* !WINDLL */ + + +#ifndef PROCNAME +/* Default to case-sensitive matching of archive entries for the modes + that specifically operate on archive entries, as this archive may + have come from a system that allows paths in the archive to differ + only by case. Except for adding ARCHIVE (copy mode), this is how it + was done before. Note that some case-insensitive ports (WIN32, VMS) + define their own PROCNAME() in their respective osdep.h that use the + filter_match_case flag set to FALSE by the -ic option to enable + case-insensitive archive entry mathing. */ +# define PROCNAME(n) procname(n, (action == ARCHIVE || action == DELETE \ + || action == FRESHEN) \ + && filter_match_case) +#endif /* PROCNAME */ + +#ifndef WINDLL +#ifndef MACOS +local void zipstdout() +/* setup for writing zip file on stdout */ +{ + mesg = stderr; + if (isatty(1)) + ziperr(ZE_PARMS, "cannot write zip file to terminal"); + if ((zipfile = malloc(4)) == NULL) + ziperr(ZE_MEM, "was processing arguments"); + strcpy(zipfile, "-"); + /* + if ((r = readzipfile()) != ZE_OK) + ziperr(r, zipfile); + */ +} +#endif /* !MACOS */ + +local int check_unzip_version(unzippath) + char *unzippath; +{ +#ifdef ZIP64_SUPPORT + /* Here is where we need to check for the version of unzip the user + * has. If creating a Zip64 archive need UnZip 6 or may fail. + */ + char cmd[4004]; + FILE *unzip_out = NULL; + char buf[1001]; + float UnZip_Version = 0.0; + + cmd[0] = '\0'; + strncat(cmd, unzippath, 4000); + strcat(cmd, " -v"); + + if ((unzip_out = popen(cmd, "r")) == NULL) { + perror("unzip pipe error"); + } else { + if (fgets(buf, 1000, unzip_out) == NULL) { + zipwarn("failed to get information from UnZip", ""); + } else { + /* the first line should start with the version */ + if (sscanf(buf, "UnZip %f ", &UnZip_Version) < 1) { + zipwarn("unexpected output of UnZip -v", ""); + } else { + /* printf("UnZip %f\n", UnZip_Version); */ + + while (fgets(buf, 1000, unzip_out)) { + } + } + } + pclose(unzip_out); + } + if (UnZip_Version < 6.0 && zip64_archive) { + sprintf(buf, "Found UnZip version %4.2f", UnZip_Version); + zipwarn(buf, ""); + zipwarn("Need UnZip 6.00 or later to test this Zip64 archive", ""); + return 0; + } +#endif + return 1; +} + +local void check_zipfile(zipname, zippath) + char *zipname; + char *zippath; + /* Invoke unzip -t on the given zip file */ +{ +#if (defined(MSDOS) && !defined(__GO32__)) || defined(__human68k__) + int status, len; + char *path, *p; + char *zipnam; + + if ((zipnam = (char *)malloc(strlen(zipname) + 3)) == NULL) + ziperr(ZE_MEM, "was creating unzip zipnam"); + +# ifdef MSDOS + /* Add quotes for MSDOS. 8/11/04 */ + strcpy(zipnam, "\""); /* accept spaces in name and path */ + strcat(zipnam, zipname); + strcat(zipnam, "\""); +# else + strcpy(zipnam, zipname); +# endif + + if (unzip_path) { + /* if user gave us the unzip to use go with it */ + char *here; + int len; + char *cmd; + + /* Replace first {} with archive name. If no {} append name to string. */ + here = strstr(unzip_path, "{}"); + + if ((cmd = (char *)malloc(strlen(unzip_path) + strlen(zipnam) + 3)) == NULL) + ziperr(ZE_MEM, "was creating unzip cmd"); + + if (here) { + /* have {} so replace with temp name */ + len = here - unzip_path; + strcpy(cmd, unzip_path); + cmd[len] = '\0'; + strcat(cmd, " "); + strcat(cmd, zipnam); + strcat(cmd, " "); + strcat(cmd, here + 2); + } else { + /* No {} so append temp name to end */ + strcpy(cmd, unzip_path); + strcat(cmd, " "); + strcat(cmd, zipnam); + } + + status = system(cmd); + + free(unzip_path); + unzip_path = NULL; + free(cmd); + } else { + /* Here is where we need to check for the version of unzip the user + * has. If creating a Zip64 archive need UnZip 6 or may fail. + */ + if (check_unzip_version("unzip") == 0) + ZIPERR(ZE_TEST, zipfile); + + status = spawnlp(P_WAIT, "unzip", "unzip", verbose ? "-t" : "-tqq", + zipnam, NULL); +# ifdef __human68k__ + if (status == -1) + perror("unzip"); +# else +/* + * unzip isn't in PATH range, assume an absolute path to zip in argv[0] + * and hope that unzip is in the same directory. + */ + if (status == -1) { + p = MBSRCHR(zippath, '\\'); + path = MBSRCHR((p == NULL ? zippath : p), '/'); + if (path != NULL) + p = path; + if (p != NULL) { + len = (int)(p - zippath) + 1; + if ((path = malloc(len + sizeof("unzip.exe"))) == NULL) + ziperr(ZE_MEM, "was creating unzip path"); + memcpy(path, zippath, len); + strcpy(&path[len], "unzip.exe"); + + if (check_unzip_version(path) == 0) + ZIPERR(ZE_TEST, zipfile); + + status = spawnlp(P_WAIT, path, "unzip", verbose ? "-t" : "-tqq", + zipnam, NULL); + free(path); + } + if (status == -1) + perror("unzip"); + } + } +# endif /* ?__human68k__ */ + free(zipnam); + if (status != 0) { + +#else /* (MSDOS && !__GO32__) || __human68k__ */ + char *cmd; + int result; + + /* Tell picky compilers to shut up about unused variables */ + zippath = zippath; + + if (unzip_path) { + /* user gave us a path to some unzip (may not be UnZip) */ + char *here; + int len; + + /* Replace first {} with archive name. If no {} append name to string. */ + here = strstr(unzip_path, "{}"); + + if ((cmd = malloc(strlen(unzip_path) + strlen(zipname) + 3)) == NULL) { + ziperr(ZE_MEM, "building command string for testing archive"); + } + + if (here) { + /* have {} so replace with temp name */ + len = here - unzip_path; + strcpy(cmd, unzip_path); + cmd[len] = '\0'; + strcat(cmd, " "); +# ifdef UNIX + strcat(cmd, "'"); /* accept space or $ in name */ + strcat(cmd, zipname); + strcat(cmd, "'"); +# else + strcat(cmd, zipname); +# endif + strcat(cmd, " "); + strcat(cmd, here + 2); + } else { + /* No {} so append temp name to end */ + strcpy(cmd, unzip_path); + strcat(cmd, " "); +# ifdef UNIX + strcat(cmd, "'"); /* accept space or $ in name */ + strcat(cmd, zipname); + strcat(cmd, "'"); +# else + strcat(cmd, zipname); +# endif + } + free(unzip_path); + unzip_path = NULL; + + } else { + if ((cmd = malloc(20 + strlen(zipname))) == NULL) { + ziperr(ZE_MEM, "building command string for testing archive"); + } + + strcpy(cmd, "unzip -t "); +# ifdef QDOS + strcat(cmd, "-Q4 "); +# endif + if (!verbose) strcat(cmd, "-qq "); + if (check_unzip_version("unzip") == 0) + ZIPERR(ZE_TEST, zipfile); + +# ifdef UNIX + strcat(cmd, "'"); /* accept space or $ in name */ + strcat(cmd, zipname); + strcat(cmd, "'"); +# else + strcat(cmd, zipname); +# endif + } + + result = system(cmd); +# ifdef VMS + /* Convert success severity to 0, others to non-zero. */ + result = ((result & STS$M_SEVERITY) != STS$M_SUCCESS); +# endif /* def VMS */ + free(cmd); + cmd = NULL; + if (result) { +#endif /* ?((MSDOS && !__GO32__) || __human68k__) */ + + fprintf(mesg, "test of %s FAILED\n", zipfile); + ziperr(ZE_TEST, "original files unmodified"); + } + if (noisy) { + fprintf(mesg, "test of %s OK\n", zipfile); + fflush(mesg); + } + if (logfile) { + fprintf(logfile, "test of %s OK\n", zipfile); + fflush(logfile); + } +} +#endif /* !WINDLL */ + +/* get_filters() is replaced by the following +local int get_filters(argc, argv) +*/ + +/* The filter patterns for options -x, -i, and -R are + returned by get_option() one at a time, so use a linked + list to store until all args are processed. Then convert + to array for processing. + */ + +/* add a filter to the linked list */ +local int add_filter(flag, pattern) + int flag; + char *pattern; +{ + char *iname, *p = NULL; + FILE *fp; + struct filterlist_struct *filter = NULL; + + /* should never happen */ + if (flag != 'R' && flag != 'x' && flag != 'i') { + ZIPERR(ZE_LOGIC, "bad flag to add_filter"); + } + if (pattern == NULL) { + ZIPERR(ZE_LOGIC, "null pattern to add_filter"); + } + + if (pattern[0] == '@') { + /* read file with 1 pattern per line */ + if (pattern[1] == '\0') { + ZIPERR(ZE_PARMS, "missing file after @"); + } + fp = fopen(pattern + 1, "r"); + if (fp == NULL) { + sprintf(errbuf, "%c pattern file '%s'", flag, pattern); + ZIPERR(ZE_OPEN, errbuf); + } + while ((p = getnam(fp)) != NULL) { + if ((filter = (struct filterlist_struct *) malloc(sizeof(struct filterlist_struct))) == NULL) { + ZIPERR(ZE_MEM, "adding filter"); + } + if (filterlist == NULL) { + /* first filter */ + filterlist = filter; /* start of list */ + lastfilter = filter; + } else { + lastfilter->next = filter; /* link to last filter in list */ + lastfilter = filter; + } + iname = ex2in(p, 0, (int *)NULL); + free(p); + if (iname != NULL) { + lastfilter->pattern = in2ex(iname); + free(iname); + } else { + lastfilter->pattern = NULL; + } + lastfilter->flag = flag; + pcount++; + lastfilter->next = NULL; + } + fclose(fp); + } else { + /* single pattern */ + if ((filter = (struct filterlist_struct *) malloc(sizeof(struct filterlist_struct))) == NULL) { + ZIPERR(ZE_MEM, "adding filter"); + } + if (filterlist == NULL) { + /* first pattern */ + filterlist = filter; /* start of list */ + lastfilter = filter; + } else { + lastfilter->next = filter; /* link to last filter in list */ + lastfilter = filter; + } + iname = ex2in(pattern, 0, (int *)NULL); + if (iname != NULL) { + lastfilter->pattern = in2ex(iname); + free(iname); + } else { + lastfilter->pattern = NULL; + } + lastfilter->flag = flag; + pcount++; + lastfilter->next = NULL; + } + + return pcount; +} + +/* convert list to patterns array */ +local int filterlist_to_patterns() +{ + unsigned i; + struct filterlist_struct *next = NULL; + + if (pcount == 0) { + patterns = NULL; + return 0; + } + if ((patterns = (struct plist *) malloc((pcount + 1) * sizeof(struct plist))) + == NULL) { + ZIPERR(ZE_MEM, "was creating pattern list"); + } + + for (i = 0; i < pcount && filterlist != NULL; i++) { + switch (filterlist->flag) { + case 'i': + icount++; + break; + case 'R': + Rcount++; + break; + } + patterns[i].select = filterlist->flag; + patterns[i].zname = filterlist->pattern; + next = filterlist->next; + free(filterlist); + filterlist = next; + } + + return pcount; +} + + +/* add a file argument to linked list */ +local long add_name(filearg) + char *filearg; +{ + char *name = NULL; + struct filelist_struct *fileentry = NULL; + + if ((fileentry = (struct filelist_struct *) malloc(sizeof(struct filelist_struct))) == NULL) { + ZIPERR(ZE_MEM, "adding file"); + } + if ((name = malloc(strlen(filearg) + 1)) == NULL) { + ZIPERR(ZE_MEM, "adding file"); + } + strcpy(name, filearg); + fileentry->next = NULL; + fileentry->name = name; + if (filelist == NULL) { + /* first file argument */ + filelist = fileentry; /* start of list */ + lastfile = fileentry; + } else { + lastfile->next = fileentry; /* link to last filter in list */ + lastfile = fileentry; + } + filearg_count++; + + return filearg_count; +} + + +/* Running Stats + 10/30/04 */ + +local int DisplayRunningStats() +{ + char tempstrg[100]; + + if (mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } + if (logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + } + if (display_volume) { + if (noisy) { + fprintf(mesg, "%lu>%lu: ", current_in_disk + 1, current_disk + 1); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, "%lu>%lu: ", current_in_disk + 1, current_disk + 1); + logfile_line_started = 1; + } + } + if (display_counts) { + if (noisy) { + fprintf(mesg, "%3ld/%3ld ", files_so_far, files_total - files_so_far); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, "%3ld/%3ld ", files_so_far, files_total - files_so_far); + logfile_line_started = 1; + } + } + if (display_bytes) { + /* since file sizes can change as we go, use bytes_so_far from + initial scan so all adds up */ + WriteNumString(bytes_so_far, tempstrg); + if (noisy) { + fprintf(mesg, "[%4s", tempstrg); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, "[%4s", tempstrg); + logfile_line_started = 1; + } + if (bytes_total >= bytes_so_far) { + WriteNumString(bytes_total - bytes_so_far, tempstrg); + if (noisy) + fprintf(mesg, "/%4s] ", tempstrg); + if (logall) + fprintf(logfile, "/%4s] ", tempstrg); + } else { + WriteNumString(bytes_so_far - bytes_total, tempstrg); + if (noisy) + fprintf(mesg, "-%4s] ", tempstrg); + if (logall) + fprintf(logfile, "-%4s] ", tempstrg); + } + } + if (noisy) + fflush(mesg); + if (logall) + fflush(logfile); + + return 0; +} + +local int BlankRunningStats() +{ + if (display_volume) { + if (noisy) { + fprintf(mesg, "%lu>%lu: ", current_in_disk + 1, current_disk + 1); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, "%lu>%lu: ", current_in_disk + 1, current_disk + 1); + logfile_line_started = 1; + } + } + if (display_counts) { + if (noisy) { + fprintf(mesg, " / "); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, " / "); + logfile_line_started = 1; + } + } + if (display_bytes) { + if (noisy) { + fprintf(mesg, " / "); + mesg_line_started = 1; + } + if (logall) { + fprintf(logfile, " / "); + logfile_line_started = 1; + } + } + if (noisy) + fflush(mesg); + if (logall) + fflush(logfile); + + return 0; +} + +#if CRYPT +#ifndef WINDLL +int encr_passwd(modeflag, pwbuf, size, zfn) +int modeflag; +char *pwbuf; +int size; +ZCONST char *zfn; +{ + char *prompt; + + /* Tell picky compilers to shut up about unused variables */ + zfn = zfn; + + prompt = (modeflag == ZP_PW_VERIFY) ? + "Verify password: " : "Enter password: "; + + if (getp(prompt, pwbuf, size) == NULL) { + ziperr(ZE_PARMS, "stderr is not a tty"); + } + return IZ_PW_ENTERED; +} +#endif /* !WINDLL */ +#else /* !CRYPT */ +int encr_passwd(modeflag, pwbuf, size, zfn) +int modeflag; +char *pwbuf; +int size; +ZCONST char *zfn; +{ + /* Tell picky compilers to shut up about unused variables */ + modeflag = modeflag; pwbuf = pwbuf; size = size; zfn = zfn; + + return ZE_LOGIC; /* This function should never be called! */ +} +#endif /* CRYPT */ + + +/* rename a split + * A split has a tempfile name until it is closed, then + * here rename it as out_path the final name for the split. + */ +int rename_split(temp_name, out_path) + char *temp_name; + char *out_path; +{ + int r; + /* Replace old zip file with new zip file, leaving only the new one */ + if ((r = replace(out_path, temp_name)) != ZE_OK) + { + zipwarn("new zip file left as: ", temp_name); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing split file"); + } + if (zip_attributes) { + setfileattr(out_path, zip_attributes); + } + return ZE_OK; +} + + +int set_filetype(out_path) + char *out_path; +{ +#ifdef __BEOS__ + /* Set the filetype of the zipfile to "application/zip" */ + setfiletype( out_path, "application/zip" ); +#endif + +#ifdef __ATHEOS__ + /* Set the filetype of the zipfile to "application/x-zip" */ + setfiletype(out_path, "application/x-zip"); +#endif + +#ifdef MACOS + /* Set the Creator/Type of the zipfile to 'IZip' and 'ZIP ' */ + setfiletype(out_path, 'IZip', 'ZIP '); +#endif + +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(out_path, 0xDDC); +#endif + return ZE_OK; +} + + +/* + ------------------------------------------------------- + Command Line Options + ------------------------------------------------------- + + Valid command line options. + + The function get_option() uses this table to check if an + option is valid and if it takes a value (also called an + option argument). To add an option to zip just add it + to this table and add a case in the main switch to handle + it. If either shortopt or longopt not used set to "". + + The fields: + shortopt - short option name (1 or 2 chars) + longopt - long option name + value_type - see zip.h for constants + negatable - option is negatable with trailing - + ID - unsigned long int returned for option + name - short description of option which is + returned on some errors and when options + are listed with -so option, can be NULL +*/ + +/* Most option IDs are set to the shortopt char. For + multichar short options set to arbitrary unused constant. */ +#define o_AC 0x101 +#define o_AS 0x102 +#define o_C2 0x103 +#define o_C5 0x104 +#define o_db 0x105 +#define o_dc 0x106 +#define o_dd 0x107 +#define o_des 0x108 +#define o_df 0x109 +#define o_DF 0x110 +#define o_dg 0x111 +#define o_ds 0x112 +#define o_du 0x113 +#define o_dv 0x114 +#define o_FF 0x115 +#define o_FI 0x116 +#define o_FS 0x117 +#define o_h2 0x118 +#define o_ic 0x119 +#define o_jj 0x120 +#define o_la 0x121 +#define o_lf 0x122 +#define o_li 0x123 +#define o_ll 0x124 +#define o_mm 0x125 +#define o_MM 0x126 +#define o_nw 0x127 +#define o_RE 0x128 +#define o_sb 0x129 +#define o_sc 0x130 +#define o_sd 0x131 +#define o_sf 0x132 +#define o_so 0x133 +#define o_sp 0x134 +#define o_su 0x135 +#define o_sU 0x136 +#define o_sv 0x137 +#define o_tt 0x138 +#define o_TT 0x139 +#define o_UN 0x140 +#define o_ve 0x141 +#define o_VV 0x142 +#define o_ws 0x143 +#define o_ww 0x144 +#define o_z64 0x145 +#ifdef UNICODE_TEST +#define o_sC 0x146 +#endif + + +/* the below is mainly from the old main command line + switch with a few changes */ +struct option_struct far options[] = { + /* short longopt value_type negatable ID name */ +#ifdef EBCDIC + {"a", "ascii", o_NO_VALUE, o_NOT_NEGATABLE, 'a', "to ascii"}, +#endif /* EBCDIC */ +#ifdef CMS_MVS + {"B", "binary", o_NO_VALUE, o_NOT_NEGATABLE, 'B', "binary"}, +#endif /* CMS_MVS */ +#ifdef TANDEM + {"B", "", o_NUMBER_VALUE, o_NOT_NEGATABLE, 'B', "nsk"}, +#endif + {"0", "store", o_NO_VALUE, o_NOT_NEGATABLE, '0', "store"}, + {"1", "compress-1", o_NO_VALUE, o_NOT_NEGATABLE, '1', "compress 1"}, + {"2", "compress-2", o_NO_VALUE, o_NOT_NEGATABLE, '2', "compress 2"}, + {"3", "compress-3", o_NO_VALUE, o_NOT_NEGATABLE, '3', "compress 3"}, + {"4", "compress-4", o_NO_VALUE, o_NOT_NEGATABLE, '4', "compress 4"}, + {"5", "compress-5", o_NO_VALUE, o_NOT_NEGATABLE, '5', "compress 5"}, + {"6", "compress-6", o_NO_VALUE, o_NOT_NEGATABLE, '6', "compress 6"}, + {"7", "compress-7", o_NO_VALUE, o_NOT_NEGATABLE, '7', "compress 7"}, + {"8", "compress-8", o_NO_VALUE, o_NOT_NEGATABLE, '8', "compress 8"}, + {"9", "compress-9", o_NO_VALUE, o_NOT_NEGATABLE, '9', "compress 9"}, + {"A", "adjust-sfx", o_NO_VALUE, o_NOT_NEGATABLE, 'A', "adjust self extractor offsets"}, +#if defined(WIN32) + {"AC", "archive-clear", o_NO_VALUE, o_NOT_NEGATABLE, o_AC, "clear DOS archive bit of included files"}, + {"AS", "archive-set", o_NO_VALUE, o_NOT_NEGATABLE, o_AS, "include only files with archive bit set"}, +#endif + {"b", "temp-path", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'b', "dir to use for temp archive"}, + {"c", "entry-comments", o_NO_VALUE, o_NOT_NEGATABLE, 'c', "add comments for each entry"}, +#ifdef VMS + {"C", "preserve-case", o_NO_VALUE, o_NEGATABLE, 'C', "Preserve (C-: down-) case all on VMS"}, + {"C2", "preserve-case-2", o_NO_VALUE, o_NEGATABLE, o_C2, "Preserve (C2-: down-) case ODS2 on VMS"}, + {"C5", "preserve-case-5", o_NO_VALUE, o_NEGATABLE, o_C5, "Preserve (C5-: down-) case ODS5 on VMS"}, +#endif /* VMS */ + {"d", "delete", o_NO_VALUE, o_NOT_NEGATABLE, 'd', "delete entries from archive"}, + {"db", "display-bytes", o_NO_VALUE, o_NEGATABLE, o_db, "display running bytes"}, + {"dc", "display-counts", o_NO_VALUE, o_NEGATABLE, o_dc, "display running file count"}, + {"dd", "display-dots", o_NO_VALUE, o_NEGATABLE, o_dd, "display dots as process each file"}, + {"dg", "display-globaldots",o_NO_VALUE, o_NEGATABLE, o_dg, "display dots for archive instead of files"}, + {"ds", "dot-size", o_REQUIRED_VALUE, o_NOT_NEGATABLE, o_ds, "set progress dot size - default 10M bytes"}, + {"du", "display-usize", o_NO_VALUE, o_NEGATABLE, o_du, "display uncompressed size in bytes"}, + {"dv", "display-volume", o_NO_VALUE, o_NEGATABLE, o_dv, "display volume (disk) number"}, +#ifdef MACOS + {"df", "datafork", o_NO_VALUE, o_NOT_NEGATABLE, o_df, "save datafork"}, +#endif /* MACOS */ + {"D", "no-dir-entries", o_NO_VALUE, o_NOT_NEGATABLE, 'D', "no entries for dirs themselves (-x */)"}, + {"DF", "difference-archive",o_NO_VALUE, o_NOT_NEGATABLE, o_DF, "create diff archive with changed/new files"}, + {"e", "encrypt", o_NO_VALUE, o_NOT_NEGATABLE, 'e', "encrypt entries, ask for password"}, +#ifdef OS2 + {"E", "longnames", o_NO_VALUE, o_NOT_NEGATABLE, 'E', "use OS2 longnames"}, +#endif + {"F", "fix", o_NO_VALUE, o_NOT_NEGATABLE, 'F', "fix mostly intact archive (try first)"}, + {"FF", "fixfix", o_NO_VALUE, o_NOT_NEGATABLE, o_FF, "try harder to fix archive (not as reliable)"}, + {"FI", "fifo", o_NO_VALUE, o_NEGATABLE, o_FI, "read Unix FIFO (zip will wait on open pipe)"}, + {"FS", "filesync", o_NO_VALUE, o_NOT_NEGATABLE, o_FS, "add/delete entries to make archive match OS"}, + {"f", "freshen", o_NO_VALUE, o_NOT_NEGATABLE, 'f', "freshen existing archive entries"}, + {"fd", "force-descriptors", o_NO_VALUE, o_NOT_NEGATABLE, o_des,"force data descriptors as if streaming"}, +#ifdef ZIP64_SUPPORT + {"fz", "force-zip64", o_NO_VALUE, o_NEGATABLE, o_z64,"force use of Zip64 format, negate prevents"}, +#endif + {"g", "grow", o_NO_VALUE, o_NOT_NEGATABLE, 'g', "grow existing archive instead of replace"}, +#ifndef WINDLL + {"h", "help", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + {"H", "", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + {"?", "", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + {"h2", "more-help", o_NO_VALUE, o_NOT_NEGATABLE, o_h2, "extended help"}, +#endif /* !WINDLL */ + {"i", "include", o_VALUE_LIST, o_NOT_NEGATABLE, 'i', "include only files matching patterns"}, +#if defined(VMS) || defined(WIN32) + {"ic", "ignore-case", o_NO_VALUE, o_NEGATABLE, o_ic, "ignore case when matching archive entries"}, +#endif +#ifdef RISCOS + {"I", "no-image", o_NO_VALUE, o_NOT_NEGATABLE, 'I', "no image"}, +#endif + {"j", "junk-paths", o_NO_VALUE, o_NOT_NEGATABLE, 'j', "strip paths and just store file names"}, +#ifdef MACOS + {"jj", "absolute-path", o_NO_VALUE, o_NOT_NEGATABLE, o_jj, "MAC absolute path"}, +#endif /* ?MACOS */ + {"J", "junk-sfx", o_NO_VALUE, o_NOT_NEGATABLE, 'J', "strip self extractor from archive"}, + {"k", "DOS-names", o_NO_VALUE, o_NOT_NEGATABLE, 'k', "force use of 8.3 DOS names"}, + {"l", "to-crlf", o_NO_VALUE, o_NOT_NEGATABLE, 'l', "convert text file line ends - LF->CRLF"}, + {"ll", "from-crlf", o_NO_VALUE, o_NOT_NEGATABLE, o_ll, "convert text file line ends - CRLF->LF"}, + {"lf", "logfile-path",o_REQUIRED_VALUE, o_NOT_NEGATABLE, o_lf, "log to log file at path (default overwrite)"}, + {"la", "log-append", o_NO_VALUE, o_NEGATABLE, o_la, "append to existing log file"}, + {"li", "log-info", o_NO_VALUE, o_NEGATABLE, o_li, "include informational messages in log"}, +#ifndef WINDLL + {"L", "license", o_NO_VALUE, o_NOT_NEGATABLE, 'L', "display license"}, +#endif + {"m", "move", o_NO_VALUE, o_NOT_NEGATABLE, 'm', "add files to archive then delete files"}, + {"mm", "", o_NO_VALUE, o_NOT_NEGATABLE, o_mm, "not used"}, + {"MM", "must-match", o_NO_VALUE, o_NOT_NEGATABLE, o_MM, "error if in file not matched/not readable"}, + {"n", "suffixes", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'n', "suffixes to not compress: .gz:.zip"}, + {"nw", "no-wild", o_NO_VALUE, o_NOT_NEGATABLE, o_nw, "no wildcards during add or update"}, +#if defined(AMIGA) || defined(MACOS) + {"N", "notes", o_NO_VALUE, o_NOT_NEGATABLE, 'N', "add notes as entry comments"}, +#endif + {"o", "latest-time", o_NO_VALUE, o_NOT_NEGATABLE, 'o', "use latest entry time as archive time"}, + {"O", "output-file", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'O', "set out zipfile different than in zipfile"}, + {"p", "paths", o_NO_VALUE, o_NOT_NEGATABLE, 'p', "store paths"}, + {"P", "password", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'P', "encrypt entries, option value is password"}, +#if defined(QDOS) || defined(QLZIP) + {"Q", "Q-flag", o_NUMBER_VALUE, o_NOT_NEGATABLE, 'Q', "Q flag"}, +#endif + {"q", "quiet", o_NO_VALUE, o_NOT_NEGATABLE, 'q', "quiet"}, + {"r", "recurse-paths", o_NO_VALUE, o_NOT_NEGATABLE, 'r', "recurse down listed paths"}, + {"R", "recurse-patterns", o_NO_VALUE, o_NOT_NEGATABLE, 'R', "recurse current dir and match patterns"}, + {"RE", "regex", o_NO_VALUE, o_NOT_NEGATABLE, o_RE, "allow [list] matching (regex)"}, + {"s", "split-size", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 's', "do splits, set split size (-s=0 no splits)"}, + {"sp", "split-pause", o_NO_VALUE, o_NOT_NEGATABLE, o_sp, "pause while splitting to select destination"}, + {"sv", "split-verbose", o_NO_VALUE, o_NOT_NEGATABLE, o_sv, "be verbose about creating splits"}, + {"sb", "split-bell", o_NO_VALUE, o_NOT_NEGATABLE, o_sb, "when pause for next split ring bell"}, + {"sc", "show-command",o_NO_VALUE, o_NOT_NEGATABLE, o_sc, "show command line"}, +#ifdef UNICODE_TEST + {"sC", "create-files",o_NO_VALUE, o_NOT_NEGATABLE, o_sC, "create empty files using archive names"}, +#endif + {"sd", "show-debug", o_NO_VALUE, o_NOT_NEGATABLE, o_sd, "show debug"}, + {"sf", "show-files", o_NO_VALUE, o_NEGATABLE, o_sf, "show files to operate on and exit"}, + {"so", "show-options",o_NO_VALUE, o_NOT_NEGATABLE, o_so, "show options"}, +#ifdef UNICODE_SUPPORT + {"su", "show-unicode", o_NO_VALUE, o_NEGATABLE, o_su, "as -sf but also show escaped Unicode"}, + {"sU", "show-just-unicode", o_NO_VALUE, o_NEGATABLE, o_sU, "as -sf but only show escaped Unicode"}, +#endif +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(ATARI) + {"S", "", o_NO_VALUE, o_NOT_NEGATABLE, 'S', "include system and hidden"}, +#endif /* MSDOS || OS2 || WIN32 || ATARI */ + {"t", "from-date", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 't', "exclude before date"}, + {"tt", "before-date", o_REQUIRED_VALUE, o_NOT_NEGATABLE, o_tt, "include before date"}, + {"T", "test", o_NO_VALUE, o_NOT_NEGATABLE, 'T', "test updates before replacing archive"}, + {"TT", "unzip-command", o_REQUIRED_VALUE,o_NOT_NEGATABLE,o_TT, "unzip command to use, name is added to end"}, + {"u", "update", o_NO_VALUE, o_NOT_NEGATABLE, 'u', "update existing entries and add new"}, + {"U", "copy-entries", o_NO_VALUE, o_NOT_NEGATABLE, 'U', "select from archive instead of file system"}, +#ifdef UNICODE_SUPPORT + {"UN", "unicode", o_REQUIRED_VALUE, o_NOT_NEGATABLE, o_UN, "UN=quit, warn, ignore, no, escape"}, +#endif + {"v", "verbose", o_NO_VALUE, o_NOT_NEGATABLE, 'v', "display additional information"}, + {"", "version", o_NO_VALUE, o_NOT_NEGATABLE, o_ve, "(if no other args) show version information"}, +#ifdef VMS + {"V", "VMS-portable", o_NO_VALUE, o_NOT_NEGATABLE, 'V', "Store VMS attributes, portable file format"}, + {"VV", "VMS-specific", o_NO_VALUE, o_NOT_NEGATABLE, o_VV, "Store VMS attributes, VMS specific format"}, + {"w", "VMS-versions", o_NO_VALUE, o_NOT_NEGATABLE, 'w', "store VMS versions"}, + {"ww", "VMS-dot-versions", o_NO_VALUE, o_NOT_NEGATABLE, o_ww, "store VMS versions as \".nnn\""}, +#endif /* VMS */ + {"ws", "wild-stop-dirs", o_NO_VALUE, o_NOT_NEGATABLE, o_ws, "* stops at /, ** includes any /"}, + {"x", "exclude", o_VALUE_LIST, o_NOT_NEGATABLE, 'x', "exclude files matching patterns"}, +/* {"X", "no-extra", o_NO_VALUE, o_NOT_NEGATABLE, 'X', "no extra"}, +*/ + {"X", "strip-extra", o_NO_VALUE, o_NEGATABLE, 'X', "-X- keep all ef, -X strip but critical ef"}, +#ifdef S_IFLNK + {"y", "symlinks", o_NO_VALUE, o_NOT_NEGATABLE, 'y', "store symbolic links"}, +#endif /* S_IFLNK */ + {"z", "archive-comment", o_NO_VALUE, o_NOT_NEGATABLE, 'z', "ask for archive comment"}, + {"Z", "compression-method", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'Z', "compression method"}, +#if defined(MSDOS) || defined(OS2) + {"$", "volume-label", o_NO_VALUE, o_NOT_NEGATABLE, '$', "store volume label"}, +#endif +#ifndef MACOS + {"@", "names-stdin", o_NO_VALUE, o_NOT_NEGATABLE, '@', "get file names from stdin, one per line"}, +#endif /* !MACOS */ +#ifdef NTSD_EAS + {"!", "use-privileges", o_NO_VALUE, o_NOT_NEGATABLE, '!', "use privileges"}, +#endif +#ifdef RISCOS + {"/", "exts-to-swap", o_REQUIRED_VALUE, o_NOT_NEGATABLE, '/', "override Zip$Exts"}, +#endif + /* the end of the list */ + {NULL, NULL, o_NO_VALUE, o_NOT_NEGATABLE, 0, NULL} /* end has option_ID = 0 */ + }; + + + +#ifndef USE_ZIPMAIN +int main(argc, argv) +#else +int zipmain(argc, argv) +#endif +int argc; /* number of tokens in command line */ +char **argv; /* command line tokens */ +/* Add, update, freshen, or delete zip entries in a zip file. See the + command help in help() above. */ +{ + int d; /* true if just adding to a zip file */ + char *e; /* malloc'd comment buffer */ + struct flist far *f; /* steps through found linked list */ + int i; /* arg counter, root directory flag */ + int kk; /* next arg type (formerly another re-use of "k") */ + + /* zip64 support 09/05/2003 R.Nausedat */ + uzoff_t c; /* start of central directory */ + uzoff_t t; /* length of central directory */ + zoff_t k; /* marked counter, comment size, entry count */ + uzoff_t n; /* total of entry len's */ + + int o; /* true if there were any ZE_OPEN errors */ + char *p; /* steps through option arguments */ + char *pp; /* temporary pointer */ + int r; /* temporary variable */ + int s; /* flag to read names from stdin */ + uzoff_t csize; /* compressed file size for stats */ + uzoff_t usize; /* uncompressed file size for stats */ + ulg tf; /* file time */ + int first_listarg = 0;/* index of first arg of "process these files" list */ + struct zlist far *v; /* temporary variable */ + struct zlist far * far *w; /* pointer to last link in zfiles list */ + FILE *x /*, *y */; /* input and output zip files (y global) */ + struct zlist far *z; /* steps through zfiles linked list */ + int bad_open_is_error = 0; /* if read fails, 0=warning, 1=error */ +#if 0 + /* does not seem used */ +#ifdef WINDLL + int retcode; /* return code for dll */ +#endif /* WINDLL */ +#endif +#if (!defined(VMS) && !defined(CMS_MVS)) + char *zipbuf; /* stdio buffer for the zip file */ +#endif /* !VMS && !CMS_MVS */ + FILE *comment_stream; /* set to stderr if anything is read from stdin */ + int all_current; /* used by File Sync to determine if all entries are current */ + + struct filelist_struct *filearg; + +/* used by get_option */ + unsigned long option; /* option ID returned by get_option */ + int argcnt = 0; /* current argcnt in args */ + int argnum = 0; /* arg number */ + int optchar = 0; /* option state */ + char *value = NULL; /* non-option arg, option value or NULL */ + int negated = 0; /* 1 = option negated */ + int fna = 0; /* current first non-opt arg */ + int optnum = 0; /* index in table */ + + int show_options = 0; /* show options */ + int show_what_doing = 0; /* show what doing */ + int show_args = 0; /* show command line */ + int seen_doubledash = 0; /* seen -- argument */ + int key_needed = 0; /* prompt for encryption key */ + int have_out = 0; /* if set in_path and out_path different archive */ +#ifdef UNICODE_TEST + int create_files = 0; +#endif + + char **args = NULL; /* could be wide argv */ + + +#ifdef THEOS + /* the argument expansion from the standard library is full of bugs */ + /* use mine instead */ + _setargv(&argc, &argv); + setlocale(LC_CTYPE, "I"); +#else + SETLOCALE(LC_CTYPE, ""); +#endif + +#ifdef UNICODE_SUPPORT +# ifdef UNIX + /* For Unix, set the locale to UTF-8. Any UTF-8 locale is + OK and they should all be the same. This allows seeing, + writing, and displaying (if the fonts are loaded) all + characters in UTF-8. */ + { + char *loc; + + /* + loc = setlocale(LC_CTYPE, NULL); + printf(" Initial language locale = '%s'\n", loc); + */ + + loc = setlocale(LC_CTYPE, "en_US.UTF-8"); + + /* + printf("langinfo %s\n", nl_langinfo(CODESET)); + */ + + if (loc != NULL) { + /* using UTF-8 character set so can set UTF-8 GPBF bit 11 */ + using_utf8 = 1; + /* + printf(" Locale set to %s\n", loc); + */ + } else { + /* + printf(" Could not set Unicode UTF-8 locale\n"); + */ + } + } +# endif +#endif + +#if defined(__IBMC__) && defined(__DEBUG_ALLOC__) + { + extern void DebugMalloc(void); + atexit(DebugMalloc); + } +#endif + +#ifdef QDOS + { + extern void QDOSexit(void); + atexit(QDOSexit); + } +#endif + +#ifdef NLM + { + extern void NLMexit(void); + atexit(NLMexit); + } +#endif + +#ifdef RISCOS + set_prefix(); +#endif + +#ifdef __human68k__ + fflush(stderr); + setbuf(stderr, NULL); +#endif + +/* Re-initialize global variables to make the zip dll re-entrant. It is + * possible that we could get away with not re-initializing all of these + * but better safe than sorry. + */ +#if defined(MACOS) || defined(WINDLL) || defined(USE_ZIPMAIN) + action = ADD; /* one of ADD, UPDATE, FRESHEN, DELETE, or ARCHIVE */ + comadd = 0; /* 1=add comments for new files */ + zipedit = 0; /* 1=edit zip comment and all file comments */ + latest = 0; /* 1=set zip file time to time of latest file */ + before = 0; /* 0=ignore, else exclude files before this time */ + after = 0; /* 0=ignore, else exclude files newer than this time */ + test = 0; /* 1=test zip file with unzip -t */ + unzip_path = NULL; /* where to look for unzip command path */ + tempdir = 0; /* 1=use temp directory (-b) */ + junk_sfx = 0; /* 1=junk the sfx prefix */ +#if defined(AMIGA) || defined(MACOS) + filenotes = 0;/* 1=take comments from AmigaDOS/MACOS filenotes */ +#endif +#ifndef USE_ZIPMAIN + zipstate = -1; +#endif + tempzip = NULL; + fcount = 0; + recurse = 0; /* 1=recurse into directories; 2=match filenames */ + dispose = 0; /* 1=remove files after put in zip file */ + pathput = 1; /* 1=store path with name */ + method = BEST; /* one of BEST, DEFLATE (only), or STORE (only) */ + dosify = 0; /* 1=make new entries look like MSDOS */ + verbose = 0; /* 1=report oddities in zip file structure */ + fix = 0; /* 1=fix the zip file */ + adjust = 0; /* 1=adjust offsets for sfx'd file (keep preamble) */ + level = 6; /* 0=fastest compression, 9=best compression */ + translate_eol = 0; /* Translate end-of-line LF -> CR LF */ +#if defined(OS2) || defined(WIN32) + use_longname_ea = 0; /* 1=use the .LONGNAME EA as the file's name */ +#endif +#ifdef NTSD_EAS + use_privileges = 0; /* 1=use security privileges overrides */ +#endif + no_wild = 0; /* 1 = wildcards are disabled */ +#ifdef WILD_STOP_AT_DIR + wild_stop_at_dir = 1; /* default wildcards do not include / in matches */ +#else + wild_stop_at_dir = 0; /* default wildcards do include / in matches */ +#endif + + skip_this_disk = 0; + des_good = 0; /* Good data descriptor found */ + des_crc = 0; /* Data descriptor CRC */ + des_csize = 0; /* Data descriptor csize */ + des_usize = 0; /* Data descriptor usize */ + + dot_size = 0; /* buffers processed in deflate per dot, 0 = no dots */ + dot_count = 0; /* buffers seen, recyles at dot_size */ + + display_counts = 0; /* display running file count */ + display_bytes = 0; /* display running bytes remaining */ + display_globaldots = 0; /* display dots for archive instead of each file */ + display_volume = 0; /* display current input and output volume (disk) numbers */ + display_usize = 0; /* display uncompressed bytes */ + + files_so_far = 0; /* files processed so far */ + bad_files_so_far = 0; /* bad files skipped so far */ + files_total = 0; /* files total to process */ + bytes_so_far = 0; /* bytes processed so far (from initial scan) */ + good_bytes_so_far = 0; /* good bytes read so far */ + bad_bytes_so_far = 0; /* bad bytes skipped so far */ + bytes_total = 0; /* total bytes to process (from initial scan) */ + + logall = 0; /* 0 = warnings/errors, 1 = all */ + logfile = NULL; /* pointer to open logfile or NULL */ + logfile_append = 0; /* append to existing logfile */ + logfile_path = NULL; /* pointer to path of logfile */ + + hidden_files = 0; /* process hidden and system files */ + volume_label = 0; /* add volume label */ + dirnames = 1; /* include directory entries by default */ +#if defined(WIN32) + only_archive_set = 0; /* only include if DOS archive bit set */ + clear_archive_bits = 0; /* clear DOS archive bit of included files */ +#endif + linkput = 0; /* 1=store symbolic links as such */ + noisy = 1; /* 0=quiet operation */ + extra_fields = 1; /* 0=create minimum, 1=don't copy old, 2=keep old */ + + use_descriptors = 0; /* 1=use data descriptors 12/29/04 */ + zip_to_stdout = 0; /* output zipfile to stdout 12/30/04 */ + allow_empty_archive = 0;/* if no files, create empty archive anyway 12/28/05 */ + copy_only = 0; /* 1=copying archive entries only */ + + output_seekable = 1; /* 1 = output seekable 3/13/05 EG */ + +#ifdef ZIP64_SUPPORT /* zip64 support 10/4/03 */ + force_zip64 = -1; /* if 1 force entries to be zip64 */ + /* mainly for streaming from stdin */ + zip64_entry = 0; /* current entry needs Zip64 */ + zip64_archive = 0; /* if 1 then at least 1 entry needs zip64 */ +#endif + +#ifdef UNICODE_SUPPORT + utf8_force = 0; /* 1=force storing UTF-8 as standard per AppNote bit 11 */ +#endif + + unicode_escape_all = 0; /* 1=escape all non-ASCII characters in paths */ + unicode_mismatch = 1; /* unicode mismatch is 0=error, 1=warn, 2=ignore, 3=no */ + + scan_delay = 5; /* seconds before display Scanning files message */ + scan_dot_time = 2; /* time in seconds between Scanning files dots */ + scan_start = 0; /* start of scan */ + scan_last = 0; /* time of last message */ + scan_started = 0; /* scan has started */ + scan_count = 0; /* Used for Scanning files ... message */ + + before = 0; /* 0=ignore, else exclude files before this time */ + after = 0; /* 0=ignore, else exclude files newer than this time */ + + special = ".Z:.zip:.zoo:.arc:.lzh:.arj"; /* List of special suffixes */ + key = NULL; /* Scramble password if scrambling */ + key_needed = 0; /* Need scramble password */ + tempath = NULL; /* Path for temporary files */ + patterns = NULL; /* List of patterns to be matched */ + pcount = 0; /* number of patterns */ + icount = 0; /* number of include only patterns */ + Rcount = 0; /* number of -R include patterns */ + + found = NULL; /* List of names found, or new found entry */ + fnxt = &found; + + /* used by get_option */ + argcnt = 0; /* size of args */ + argnum = 0; /* current arg number */ + optchar = 0; /* option state */ + value = NULL; /* non-option arg, option value or NULL */ + negated = 0; /* 1 = option negated */ + fna = 0; /* current first nonopt arg */ + optnum = 0; /* option index */ + + show_options = 0; /* 1 = show options */ + show_what_doing = 0; /* 1 = show what zip doing */ + show_args = 0; /* 1 = show command line */ + seen_doubledash = 0; /* seen -- argument */ + + zipfile = NULL; /* path of usual in and out zipfile */ + tempzip = NULL; /* name of temp file */ + y = NULL; /* output file now global so can change in splits */ + in_file = NULL; /* current input file for splits */ + in_split_path = NULL; /* current in split path */ + in_path = NULL; /* used by splits to track changing split locations */ + out_path = NULL; /* if set, use -O out_path as output */ + have_out = 0; /* if set, in_path and out_path not the same archive */ + + total_disks = 0; /* total disks in archive */ + current_in_disk = 0; /* current read split disk */ + current_in_offset = 0; /* current offset in current read disk */ + skip_current_disk = 0; /* if != 0 and fix then skip entries on this disk */ + + zip64_eocd_disk = 0; /* disk with Zip64 End Of Central Directory Record */ + zip64_eocd_offset = 0; /* offset for Zip64 EOCD Record */ + + current_local_disk = 0; /* disk with current local header */ + + current_disk = 0; /* current disk number */ + cd_start_disk = (ulg)-1; /* central directory start disk */ + cd_start_offset = 0; /* offset of start of cd on cd start disk */ + cd_entries_this_disk = 0; /* cd entries this disk */ + total_cd_entries = 0; /* total cd entries in new/updated archive */ + + /* for split method 1 (keep split with local header open and update) */ + current_local_tempname = NULL; /* name of temp file */ + current_local_file = NULL; /* file pointer for current local header */ + current_local_offset = 0; /* offset to start of current local header */ + + /* global */ + bytes_this_split = 0; /* bytes written to the current split */ + read_split_archive = 0; /* 1=scanzipf_reg detected spanning signature */ + split_method = 0; /* 0=no splits, 1=update LHs, 2=data descriptors */ + split_size = 0; /* how big each split should be */ + split_bell = 0; /* when pause for next split ring bell */ + bytes_prev_splits = 0; /* total bytes written to all splits before this */ + bytes_this_entry = 0; /* bytes written for this entry across all splits */ + noisy_splits = 0; /* be verbose about creating splits */ + mesg_line_started = 0; /* 1=started writing a line to mesg */ + logfile_line_started = 0; /* 1=started writing a line to logfile */ + + filelist = NULL; + filearg_count = 0; + allow_empty_archive = 0; /* if no files, allow creation of empty archive anyway */ + bad_open_is_error = 0; /* if read fails, 0=warning, 1=error */ + unicode_mismatch = 0; /* unicode mismatch is 0=error, 1=warn, 2=ignore, 3=no */ + show_files = 0; /* show files to operate on and exit */ + scan_delay = 5; /* seconds before display Scanning files message */ + scan_dot_time = 2; /* time in seconds between Scanning files dots */ + scan_started = 0; /* space at start of scan has been displayed */ + scan_last = 0; /* Time last dot displayed for Scanning files message */ + scan_start = 0; /* Time scanning started for Scanning files message */ +#ifdef UNICODE_SUPPORT + use_wide_to_mb_default = 0; +#endif + filter_match_case = 1; /* default is to match case when matching archive entries */ + allow_fifo = 0; /* 1=allow reading Unix FIFOs, waiting if pipe open */ + +#if !defined(MACOS) && !defined(USE_ZIPMAIN) + retcode = setjmp(zipdll_error_return); + if (retcode) { + return retcode; + } +#endif /* !MACOS */ +#endif /* MACOS || WINDLL */ + +#if !defined(ALLOW_REGEX) && (defined(MSDOS) || defined(WIN32)) + allow_regex = 0; /* 1 = allow [list] matching (regex) */ +#else + allow_regex = 1; +#endif + + mesg = (FILE *) stdout; /* cannot be made at link time for VMS */ + comment_stream = (FILE *)stdin; + + init_upper(); /* build case map table */ + +#ifdef LARGE_FILE_SUPPORT + /* test if we can support large files - 9/29/04 */ + if (sizeof(zoff_t) < 8) { + ZIPERR(ZE_COMPERR, "LARGE_FILE_SUPPORT enabled but OS not supporting it"); + } +#endif + /* test if sizes are the same - 12/30/04 */ + if (sizeof(uzoff_t) != sizeof(zoff_t)){ + ZIPERR(ZE_COMPERR, "uzoff_t not same size as zoff_t"); + } + +#if (defined(WIN32) && defined(USE_EF_UT_TIME)) + /* For the Win32 environment, we may have to "prepare" the environment + prior to the tzset() call, to work around tzset() implementation bugs. + */ + iz_w32_prepareTZenv(); +#endif + +#if (defined(IZ_CHECK_TZ) && defined(USE_EF_UT_TIME)) +# ifndef VALID_TIMEZONE +# define VALID_TIMEZONE(tmp) \ + (((tmp = getenv("TZ")) != NULL) && (*tmp != '\0')) +# endif + zp_tz_is_valid = VALID_TIMEZONE(p); +#if (defined(AMIGA) || defined(DOS)) + if (!zp_tz_is_valid) + extra_fields = 0; /* disable storing "UT" time stamps */ +#endif /* AMIGA || DOS */ +#endif /* IZ_CHECK_TZ && USE_EF_UT_TIME */ + +/* For systems that do not have tzset() but supply this function using another + name (_tzset() or something similar), an appropiate "#define tzset ..." + should be added to the system specifc configuration section. */ +#if (!defined(TOPS20) && !defined(VMS)) +#if (!defined(RISCOS) && !defined(MACOS) && !defined(QDOS)) +#if (!defined(BSD) && !defined(MTS) && !defined(CMS_MVS) && !defined(TANDEM)) + tzset(); +#endif +#endif +#endif + +#ifdef VMSCLI + { + ulg status = vms_zip_cmdline(&argc, &argv); + if (!(status & 1)) + return status; + } +#endif /* VMSCLI */ + + /* Substitutes the extended command line argument list produced by + * the MKS Korn Shell in place of the command line info from DOS. + */ + + /* extract extended argument list from environment */ + expand_args(&argc, &argv); + +#ifndef WINDLL + /* Process arguments */ + diag("processing arguments"); + /* First, check if just the help or version screen should be displayed */ + if (argc == 1 && isatty(1)) /* no arguments, and output screen available */ + { /* show help screen */ +# ifdef VMSCLI + VMSCLI_help(); +# else + help(); +# endif + EXIT(ZE_OK); + } + /* Check -v here as env arg can change argc. Handle --version in main switch. */ + else if (argc == 2 && strcmp(argv[1], "-v") == 0 && + /* only "-v" as argument, and */ + (isatty(1) || isatty(0))) + /* stdout or stdin is connected to console device */ + { /* show diagnostic version info */ + version_info(); + EXIT(ZE_OK); + } +# ifndef VMS +# ifndef RISCOS + envargs(&argc, &argv, "ZIPOPT", "ZIP"); /* get options from environment */ +# else /* RISCOS */ + envargs(&argc, &argv, "ZIPOPT", "Zip$Options"); /* get options from environment */ + getRISCOSexts("Zip$Exts"); /* get the extensions to swap from environment */ +# endif /* ? RISCOS */ +# else /* VMS */ + envargs(&argc, &argv, "ZIPOPT", "ZIP_OPTS"); /* 4th arg for unzip compat. */ +# endif /* ?VMS */ +#endif /* !WINDLL */ + + zipfile = tempzip = NULL; + y = NULL; + d = 0; /* disallow adding to a zip file */ +#if (!defined(MACOS) && !defined(WINDLL) && !defined(NLM)) + signal(SIGINT, handler); +#ifdef SIGTERM /* AMIGADOS and others have no SIGTERM */ + signal(SIGTERM, handler); +#endif +# if defined(SIGABRT) && !(defined(AMIGA) && defined(__SASC)) + signal(SIGABRT, handler); +# endif +# ifdef SIGBREAK + signal(SIGBREAK, handler); +# endif +# ifdef SIGBUS + signal(SIGBUS, handler); +# endif +# ifdef SIGILL + signal(SIGILL, handler); +# endif +# ifdef SIGSEGV + signal(SIGSEGV, handler); +# endif +#endif /* !MACOS && !WINDLL && !NLM */ +#ifdef NLM + NLMsignals(); +#endif + + +#if defined(UNICODE_SUPPORT) && defined(WIN32) + /* check if this Win32 OS has support for wide character calls */ + has_win32_wide(); +#endif + + /* make copy of args that can use with insert_arg() used by get_option() */ + args = copy_args(argv, 0); + + kk = 0; /* Next non-option argument type */ + s = 0; /* set by -@ */ + + /* + ------------------------------------------- + Process command line using get_option + ------------------------------------------- + + Each call to get_option() returns either a command + line option and possible value or a non-option argument. + Arguments are permuted so that all options (-r, -b temp) + are returned before non-option arguments (zipfile). + Returns 0 when nothing left to read. + */ + + /* set argnum = 0 on first call to init get_option */ + argnum = 0; + + /* get_option returns the option ID and updates parameters: + args - usually same as argv if no argument file support + argcnt - current argc for args + value - char* to value (free() when done with it) or NULL if no value + negated - option was negated with trailing - + */ + + while ((option = get_option(&args, &argcnt, &argnum, + &optchar, &value, &negated, + &fna, &optnum, 0))) + { + switch (option) + { +#ifdef EBCDIC + case 'a': + aflag = ASCII; + printf("Translating to ASCII...\n"); + break; +#endif /* EBCDIC */ +#ifdef CMS_MVS + case 'B': + bflag = 1; + printf("Using binary mode...\n"); + break; +#endif /* CMS_MVS */ +#ifdef TANDEM + case 'B': + nskformatopt(value); + free(value); + break; +#endif + + case '0': + method = STORE; level = 0; break; + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + /* Set the compression efficacy */ + level = (int)option - '0'; break; + case 'A': /* Adjust unzipsfx'd zipfile: adjust offsets only */ + adjust = 1; break; +#if defined(WIN32) + case o_AC: + clear_archive_bits = 1; break; + case o_AS: + /* Since some directories could be empty if no archive bits are + set for files in a directory, don't add directory entries (-D). + Just files with the archive bit set are added, including paths + (unless paths are excluded). All major unzips should create + directories for the paths as needed. */ + dirnames = 0; + only_archive_set = 1; break; +#endif /* MSDOS || OS2 || WIN32 */ + case 'b': /* Specify path for temporary file */ + tempdir = 1; + tempath = value; + break; + case 'c': /* Add comments for new files in zip file */ + comadd = 1; break; + + /* -C, -C2, and -C5 are with -V */ + + case 'd': /* Delete files from zip file */ + if (action != ADD) { + ZIPERR(ZE_PARMS, "specify just one action"); + } + action = DELETE; + break; +#ifdef MACOS + case o_df: + MacZip.DataForkOnly = true; + break; +#endif /* MACOS */ + case o_db: + if (negated) + display_bytes = 0; + else + display_bytes = 1; + break; + case o_dc: + if (negated) + display_counts = 0; + else + display_counts = 1; + break; + case o_dd: + /* display dots */ + display_globaldots = 0; + if (negated) { + dot_count = 0; + } else { + /* set default dot size if dot_size not set (dot_count = 0) */ + if (dot_count == 0) + /* default to 10 MB */ + dot_size = 10 * 0x100000; + dot_count = -1; + } + break; + case o_dg: + /* display dots globally for archive instead of for each file */ + if (negated) { + display_globaldots = 0; + } else { + display_globaldots = 1; + /* set default dot size if dot_size not set (dot_count = 0) */ + if (dot_count == 0) + dot_size = 10 * 0x100000; + dot_count = -1; + } + break; + case o_ds: + /* input dot_size is now actual dot size to account for + different buffer sizes */ + if (value == NULL) + dot_size = 10 * 0x100000; + else if (value[0] == '\0') { + /* default to 10 MB */ + dot_size = 10 * 0x100000; + free(value); + } else { + dot_size = ReadNumString(value); + if (dot_size == (zoff_t)-1) { + sprintf(errbuf, "option -ds (--dot-size) has bad size: '%s'", + value); + free(value); + ZIPERR(ZE_PARMS, errbuf); + } + if (dot_size < 0x400) { + /* < 1 KB so there is no multiplier, assume MB */ + dot_size *= 0x100000; + + } else if (dot_size < 0x400L * 32) { + /* 1K <= dot_size < 32K */ + sprintf(errbuf, "dot size must be at least 32 KB: '%s'", value); + free(value); + ZIPERR(ZE_PARMS, errbuf); + + } else { + /* 32K <= dot_size */ + } + free(value); + } + dot_count = -1; + break; + case o_du: + if (negated) + display_usize = 0; + else + display_usize = 1; + break; + case o_dv: + if (negated) + display_volume = 0; + else + display_volume = 1; + break; + case 'D': /* Do not add directory entries */ + dirnames = 0; break; + case o_DF: /* Create a difference archive */ + diff_mode = 1; + allow_empty_archive = 1; + break; + case 'e': /* Encrypt */ +#if !CRYPT + ZIPERR(ZE_PARMS, "encryption not supported"); +#else /* CRYPT */ + if (key) + free(key); + key_needed = 1; +#endif /* !CRYPT */ + break; + case 'F': /* fix the zip file */ + fix = 1; break; + case o_FF: /* try harder to fix file */ + fix = 2; break; + case o_FI: + if (negated) + allow_fifo = 0; + else + allow_fifo = 1; + break; + case o_FS: /* delete exiting entries in archive where there is + no matching file on file system */ + filesync = 1; break; + case 'f': /* Freshen zip file--overwrite only */ + if (action != ADD) { + ZIPERR(ZE_PARMS, "specify just one action"); + } + action = FRESHEN; + break; + case 'g': /* Allow appending to a zip file */ + d = 1; break; +#ifndef WINDLL + case 'h': case 'H': case '?': /* Help */ +#ifdef VMSCLI + VMSCLI_help(); +#else + help(); +#endif + RETURN(finish(ZE_OK)); +#endif /* !WINDLL */ + +#ifndef WINDLL + case o_h2: /* Extended Help */ + help_extended(); + RETURN(finish(ZE_OK)); +#endif /* !WINDLL */ + + /* -i is with -x */ +#if defined(VMS) || defined(WIN32) + case o_ic: /* Ignore case (case-insensitive matching of archive entries) */ + if (negated) + filter_match_case = 1; + else + filter_match_case = 0; + break; +#endif +#ifdef RISCOS + case 'I': /* Don't scan through Image files */ + scanimage = 0; + break; +#endif +#ifdef MACOS + case o_jj: /* store absolute path including volname */ + MacZip.StoreFullPath = true; + break; +#endif /* ?MACOS */ + case 'j': /* Junk directory names */ + pathput = 0; break; + case 'J': /* Junk sfx prefix */ + junk_sfx = 1; break; + case 'k': /* Make entries using DOS names (k for Katz) */ + dosify = 1; break; + case 'l': /* Translate end-of-line */ + translate_eol = 1; break; + case o_ll: + translate_eol = 2; break; + case o_lf: + /* open a logfile */ + /* allow multiple use of option but only last used */ + if (logfile_path) { + free(logfile_path); + } + logfile_path = value; + break; + case o_la: + /* append to existing logfile */ + if (negated) + logfile_append = 0; + else + logfile_append = 1; + break; + case o_li: + /* log all including informational messages */ + if (negated) + logall = 0; + else + logall = 1; + break; +#ifndef WINDLL + case 'L': /* Show license */ + license(); + RETURN(finish(ZE_OK)); +#endif + case 'm': /* Delete files added or updated in zip file */ + dispose = 1; break; + case o_mm: /* To prevent use of -mm for -MM */ + ZIPERR(ZE_PARMS, "-mm not supported, Must_Match is -MM"); + dispose = 1; break; + case o_MM: /* Exit with error if input file can't be read */ + bad_open_is_error = 1; break; + case 'n': /* Don't compress files with a special suffix */ + special = value; + /* special = NULL; */ /* will be set at next argument */ + break; + case o_nw: /* no wildcards - wildcards are handled like other characters */ + no_wild = 1; + break; +#if defined(AMIGA) || defined(MACOS) + case 'N': /* Get zipfile comments from AmigaDOS/MACOS filenotes */ + filenotes = 1; break; +#endif + case 'o': /* Set zip file time to time of latest file in it */ + latest = 1; break; + case 'O': /* Set output file different than input archive */ + out_path = ziptyp(value); + free(value); + have_out = 1; + break; + case 'p': /* Store path with name */ + break; /* (do nothing as annoyance avoidance) */ + case 'P': /* password for encryption */ + if (key != NULL) { + free(key); + } +#if CRYPT + key = value; + key_needed = 0; +#else + ZIPERR(ZE_PARMS, "encryption not supported"); +#endif /* CRYPT */ + break; +#if defined(QDOS) || defined(QLZIP) + case 'Q': + qlflag = strtol(value, NULL, 10); + /* qlflag = strtol((p+1), &p, 10); */ + /* p--; */ + if (qlflag == 0) qlflag = 4; + free(value); + break; +#endif + case 'q': /* Quiet operation */ + noisy = 0; +#ifdef MACOS + MacZip.MacZip_Noisy = false; +#endif /* MACOS */ + if (verbose) verbose--; + break; + case 'r': /* Recurse into subdirectories, match full path */ + if (recurse == 2) { + ZIPERR(ZE_PARMS, "do not specify both -r and -R"); + } + recurse = 1; break; + case 'R': /* Recurse into subdirectories, match filename */ + if (recurse == 1) { + ZIPERR(ZE_PARMS, "do not specify both -r and -R"); + } + recurse = 2; break; + + case o_RE: /* Allow [list] matching (regex) */ + allow_regex = 1; break; + + case o_sc: /* show command line args */ + show_args = 1; break; +#ifdef UNICODE_TEST + case o_sC: /* create empty files from archive names */ + create_files = 1; + show_files = 1; break; +#endif + case o_sd: /* show debugging */ + show_what_doing = 1; break; + case o_sf: /* show files to operate on */ + if (!negated) + show_files = 1; + else + show_files = 2; + break; + case o_so: /* show all options */ + show_options = 1; break; +#ifdef UNICODE_SUPPORT + case o_su: /* -sf but also show Unicode if exists */ + if (!negated) + show_files = 3; + else + show_files = 4; + break; + case o_sU: /* -sf but only show Unicode if exists or normal if not */ + if (!negated) + show_files = 5; + else + show_files = 6; + break; +#endif + + case 's': /* enable split archives */ + /* get the split size from value */ + if (strcmp(value, "-") == 0) { + /* -s- do not allow splits */ + split_method = -1; + } else { + split_size = ReadNumString(value); + if (split_size == (uzoff_t)-1) { + sprintf(errbuf, "bad split size: '%s'", value); + ZIPERR(ZE_PARMS, errbuf); + } + if (split_size == 0) { + /* do not allow splits */ + split_method = -1; + } else { + if (split_method == 0) { + split_method = 1; + } + if (split_size < 0x400) { + /* < 1 KB there is no multiplier, assume MB */ + split_size *= 0x100000; + } + /* By setting the minimum split size to 64 KB we avoid + not having enough room to write a header unsplit + which is required */ + if (split_size < 0x400L * 64) { + /* split_size < 64K */ + sprintf(errbuf, "minimum split size is 64 KB: '%s'", value); + free(value); + ZIPERR(ZE_PARMS, errbuf); + } + } + } + free(value); + break; + case o_sb: /* when pause for next split ring bell */ + split_bell = 1; + break; + case o_sp: /* enable split select - pause splitting between splits */ + use_descriptors = 1; + split_method = 2; + break; + case o_sv: /* be verbose about creating splits */ + noisy_splits = 1; + break; + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(ATARI) + case 'S': + hidden_files = 1; break; +#endif /* MSDOS || OS2 || WIN32 || ATARI */ +#ifdef MACOS + case 'S': + MacZip.IncludeInvisible = true; break; +#endif /* MACOS */ + case 't': /* Exclude files earlier than specified date */ + { + int yyyy, mm, dd; /* results of sscanf() */ + + /* Support ISO 8601 & American dates */ + if ((sscanf(value, "%4d-%2d-%2d", &yyyy, &mm, &dd) != 3 && + sscanf(value, "%2d%2d%4d", &mm, &dd, &yyyy) != 3) || + mm < 1 || mm > 12 || dd < 1 || dd > 31) { + ZIPERR(ZE_PARMS, + "invalid date entered for -t option - use mmddyyyy or yyyy-mm-dd"); + } + before = dostime(yyyy, mm, dd, 0, 0, 0); + } + free(value); + break; + case o_tt: /* Exclude files at or after specified date */ + { + int yyyy, mm, dd; /* results of sscanf() */ + + /* Support ISO 8601 & American dates */ + if ((sscanf(value, "%4d-%2d-%2d", &yyyy, &mm, &dd) != 3 && + sscanf(value, "%2d%2d%4d", &mm, &dd, &yyyy) != 3) || + mm < 1 || mm > 12 || dd < 1 || dd > 31) { + ZIPERR(ZE_PARMS, + "invalid date entered for -tt option - use mmddyyyy or yyyy-mm-dd"); + } + after = dostime(yyyy, mm, dd, 0, 0, 0); + } + free(value); + break; + case 'T': /* test zip file */ + test = 1; break; + case o_TT: /* command path to use instead of 'unzip -t ' */ + if (unzip_path) + free(unzip_path); + unzip_path = value; + break; + case 'U': /* Select archive entries to keep or operate on */ + if (action != ADD) { + ZIPERR(ZE_PARMS, "specify just one action"); + } + action = ARCHIVE; + break; +#ifdef UNICODE_SUPPORT + case o_UN: /* Unicode */ + if (abbrevmatch("quit", value, 0, 1)) { + /* Unicode path mismatch is error */ + unicode_mismatch = 0; + } else if (abbrevmatch("warn", value, 0, 1)) { + /* warn of mismatches and continue */ + unicode_mismatch = 1; + } else if (abbrevmatch("ignore", value, 0, 1)) { + /* ignore mismatches and continue */ + unicode_mismatch = 2; + } else if (abbrevmatch("no", value, 0, 1)) { + /* no use Unicode path */ + unicode_mismatch = 3; + } else if (abbrevmatch("escape", value, 0, 1)) { + /* escape all non-ASCII characters */ + unicode_escape_all = 1; + + } else if (abbrevmatch("UTF8", value, 0, 1)) { + /* force storing UTF-8 as standard per AppNote bit 11 */ + utf8_force = 1; + + } else { + zipwarn("-UN must be Quit, Warn, Ignore, No, Escape, or UTF8: ", value); + + free(value); + ZIPERR(ZE_PARMS, "-UN (unicode) bad value"); + } + free(value); + break; +#endif + case 'u': /* Update zip file--overwrite only if newer */ + if (action != ADD) { + ZIPERR(ZE_PARMS, "specify just one action"); + } + action = UPDATE; + break; + case 'v': /* Either display version information or */ + case o_ve: /* Mention oddities in zip file structure */ + if (option == o_ve || /* --version */ + (argcnt == 2 && strlen(args[1]) == 2)) { /* -v only */ + /* display version */ +#ifndef WINDLL + version_info(); +#else + zipwarn("version information not supported for dll", ""); +#endif + RETURN(finish(ZE_OK)); + } else { + noisy = 1; + verbose++; + } + break; +#ifdef VMS + case 'C': /* Preserve case (- = down-case) all. */ + if (negated) + { /* Down-case all. */ + if ((vms_case_2 > 0) || (vms_case_5 > 0)) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C-)"); + } + vms_case_2 = -1; + vms_case_5 = -1; + } + else + { /* Not negated. Preserve all. */ + if ((vms_case_2 < 0) || (vms_case_5 < 0)) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C)"); + } + vms_case_2 = 1; + vms_case_5 = 1; + } + break; + case o_C2: /* Preserve case (- = down-case) ODS2. */ + if (negated) + { /* Down-case ODS2. */ + if (vms_case_2 > 0) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C2-)"); + } + vms_case_2 = -1; + } + else + { /* Not negated. Preserve ODS2. */ + if (vms_case_2 < 0) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C2)"); + } + vms_case_2 = 1; + } + break; + case o_C5: /* Preserve case (- = down-case) ODS5. */ + if (negated) + { /* Down-case ODS5. */ + if (vms_case_5 > 0) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C5-)"); + } + vms_case_5 = -1; + } + else + { /* Not negated. Preserve ODS5. */ + if (vms_case_5 < 0) + { + ZIPERR( ZE_PARMS, "Conflicting case directives (-C5)"); + } + vms_case_5 = 1; + } + break; + case 'V': /* Store in VMS format. (Record multiples.) */ + vms_native = 1; break; + /* below does work with new parser but doesn't allow tracking + -VV separately, like adding a separate description */ + /* vms_native++; break; */ + case o_VV: /* Store in VMS specific format */ + vms_native = 2; break; + case 'w': /* Append the VMS version number */ + vmsver |= 1; break; + case o_ww: /* Append the VMS version number as ".nnn". */ + vmsver |= 3; break; +#endif /* VMS */ + case o_ws: /* Wildcards do not include directory boundaries in matches */ + wild_stop_at_dir = 1; + break; + + case 'i': /* Include only the following files */ + /* if nothing matches include list then still create an empty archive */ + allow_empty_archive = 1; + case 'x': /* Exclude following files */ + add_filter((int) option, value); + free(value); + break; +#ifdef S_IFLNK + case 'y': /* Store symbolic links as such */ + linkput = 1; break; +#endif /* S_IFLNK */ + case 'z': /* Edit zip file comment */ + zipedit = 1; break; + case 'Z': /* Compression method */ + if (abbrevmatch("deflate", value, 0, 1)) { + /* deflate */ + method = DEFLATE; + } else if (abbrevmatch("store", value, 0, 1)) { + /* store */ + method = STORE; + } else if (abbrevmatch("bzip2", value, 0, 1)) { + /* bzip2 */ +#ifdef BZIP2_SUPPORT + method = BZIP2; +#else + ZIPERR(ZE_COMPERR, "Compression method bzip2 not enabled"); +#endif + } else { +#ifdef BZIP2_SUPPORT + zipwarn("valid compression methods are: store, deflate, bzip2", ""); +#else + zipwarn("valid compression methods are: store, deflate)", ""); +#endif + zipwarn("unknown compression method found: ", value); + free(value); + ZIPERR(ZE_PARMS, "Option -Z (--compression-method): unknown method"); + } + free(value); + break; +#if defined(MSDOS) || defined(OS2) + case '$': /* Include volume label */ + volume_label = 1; break; +#endif +#ifndef MACOS + case '@': /* read file names from stdin */ + comment_stream = NULL; + s = 1; /* defer -@ until have zipfile name */ + break; +#endif /* !MACOS */ + case 'X': + if (negated) + extra_fields = 2; + else + extra_fields = 0; + break; +#ifdef OS2 + case 'E': + /* use the .LONGNAME EA (if any) as the file's name. */ + use_longname_ea = 1; + break; +#endif +#ifdef NTSD_EAS + case '!': + /* use security privilege overrides */ + use_privileges = 1; + break; +#endif +#ifdef RISCOS + case '/': + exts2swap = value; /* override Zip$Exts */ + break; +#endif + case o_des: + use_descriptors = 1; + break; + +#ifdef ZIP64_SUPPORT + case o_z64: /* Force creation of Zip64 entries */ + if (negated) { + force_zip64 = 0; + } else { + force_zip64 = 1; + } + break; +#endif + + case o_NON_OPTION_ARG: + /* not an option */ + /* no more options as permuting */ + /* just dash also ends up here */ + + if (recurse != 2 && kk == 0 && patterns == NULL) { + /* have all filters so convert filterlist to patterns array + as PROCNAME needs patterns array */ + filterlist_to_patterns(); + } + + /* "--" stops arg processing for remaining args */ + /* ignore only first -- */ + if (strcmp(value, "--") == 0 && seen_doubledash == 0) { + /* -- */ + seen_doubledash = 1; + if (kk == 0) { + ZIPERR(ZE_PARMS, "can't use -- before archive name"); + } + + /* just ignore as just marks what follows as non-option arguments */ + + } else if (kk == 6) { + /* value is R pattern */ + add_filter((int)'R', value); + free(value); + if (first_listarg == 0) { + first_listarg = argnum; + } + } else switch (kk) + { + case 0: + /* first non-option arg is zipfile name */ +#if (!defined(MACOS) && !defined(WINDLL)) + if (strcmp(value, "-") == 0) { /* output zipfile is dash */ + /* just a dash */ + zipstdout(); + } else +#endif /* !MACOS && !WINDLL */ + { + /* name of zipfile */ + if ((zipfile = ziptyp(value)) == NULL) { + ZIPERR(ZE_MEM, "was processing arguments"); + } + /* read zipfile if exists */ + /* + if ((r = readzipfile()) != ZE_OK) { + ZIPERR(r, zipfile); + } + */ + free(value); + } + if (show_what_doing) { + fprintf(mesg, "sd: Zipfile name '%s'\n", zipfile); + fflush(mesg); + } + /* if in_path not set, use zipfile path as usual for input */ + /* in_path is used as the base path to find splits */ + if (in_path == NULL) { + if ((in_path = malloc(strlen(zipfile) + 1)) == NULL) { + ZIPERR(ZE_MEM, "was processing arguments"); + } + strcpy(in_path, zipfile); + } + /* if out_path not set, use zipfile path as usual for output */ + /* out_path is where the output archive is written */ + if (out_path == NULL) { + if ((out_path = malloc(strlen(zipfile) + 1)) == NULL) { + ZIPERR(ZE_MEM, "was processing arguments"); + } + strcpy(out_path, zipfile); + } + kk = 3; + if (s) + { + /* do -@ and get names from stdin */ + /* should be able to read names from + stdin and output to stdout, but + this was not allowed in old code. + This check moved to kk = 3 case to fix. */ + /* if (strcmp(zipfile, "-") == 0) { + ZIPERR(ZE_PARMS, "can't use - and -@ together"); + } + */ + while ((pp = getnam(stdin)) != NULL) + { + kk = 4; + if (recurse == 2) { + /* reading patterns from stdin */ + add_filter((int)'R', pp); + } else { + /* file argument now processed later */ + add_name(pp); + } + /* + if ((r = PROCNAME(pp)) != ZE_OK) { + if (r == ZE_MISS) + zipwarn("name not matched: ", pp); + else { + ZIPERR(r, pp); + } + } + */ + free(pp); + } + s = 0; + } + if (recurse == 2) { + /* rest are -R patterns */ + kk = 6; + } + break; + + case 3: case 4: + /* no recurse and -r file names */ + /* can't read filenames -@ and input - from stdin at + same time */ + if (s == 1 && strcmp(value, "-") == 0) { + ZIPERR(ZE_PARMS, "can't read input (-) and filenames (-@) both from stdin"); + } + /* add name to list for later processing */ + add_name(value); + /* + if ((r = PROCNAME(value)) != ZE_OK) { + if (r == ZE_MISS) + zipwarn("name not matched: ", value); + else { + ZIPERR(r, value); + } + } + */ + if (kk == 3) { + first_listarg = argnum; + kk = 4; + } + break; + + } /* switch kk */ + break; + + default: + /* should never get here as get_option will exit if not in table */ + sprintf(errbuf, "no such option ID: %ld", option); + ZIPERR(ZE_PARMS, errbuf); + + } /* switch */ + } + + + /* do processing of command line and one-time tasks */ + + /* Key not yet specified. If needed, get/verify it now. */ + if (key_needed) { + if ((key = malloc(IZ_PWLEN+1)) == NULL) { + ZIPERR(ZE_MEM, "was getting encryption password"); + } + r = encr_passwd(ZP_PW_ENTER, key, IZ_PWLEN+1, zipfile); + if (r != IZ_PW_ENTERED) { + if (r < IZ_PW_ENTERED) + r = ZE_PARMS; + ZIPERR(r, "was getting encryption password"); + } + if (*key == '\0') { + ZIPERR(ZE_PARMS, "zero length password not allowed"); + } + if ((e = malloc(IZ_PWLEN+1)) == NULL) { + ZIPERR(ZE_MEM, "was verifying encryption password"); + } + r = encr_passwd(ZP_PW_VERIFY, e, IZ_PWLEN+1, zipfile); + if (r != IZ_PW_ENTERED && r != IZ_PW_SKIPVERIFY) { + free((zvoid *)e); + if (r < ZE_OK) r = ZE_PARMS; + ZIPERR(r, "was verifying encryption password"); + } + r = ((r == IZ_PW_SKIPVERIFY) ? 0 : strcmp(key, e)); + free((zvoid *)e); + if (r) { + ZIPERR(ZE_PARMS, "password verification failed"); + } + } + if (key) { + /* if -P "" could get here */ + if (*key == '\0') { + ZIPERR(ZE_PARMS, "zero length password not allowed"); + } + } + + if (show_what_doing) { + fprintf(mesg, "sd: Command line read\n"); + fflush(mesg); + } + + /* show command line args */ + if (show_args) { + fprintf(mesg, "command line:\n"); + for (i = 0; args[i]; i++) { + fprintf(mesg, "'%s' ", args[i]); + } + fprintf(mesg, "\n"); + ZIPERR(ZE_ABORT, "show command line"); + } + + /* show all options */ + if (show_options) { + printf("available options:\n"); + printf(" %-2s %-18s %-4s %-3s %-30s\n", "sh", "long", "val", "neg", "description"); + printf(" %-2s %-18s %-4s %-3s %-30s\n", "--", "----", "---", "---", "-----------"); + for (i = 0; options[i].option_ID; i++) { + printf(" %-2s %-18s ", options[i].shortopt, options[i].longopt); + switch (options[i].value_type) { + case o_NO_VALUE: + printf("%-4s ", ""); + break; + case o_REQUIRED_VALUE: + printf("%-4s ", "req"); + break; + case o_OPTIONAL_VALUE: + printf("%-4s ", "opt"); + break; + case o_VALUE_LIST: + printf("%-4s ", "list"); + break; + case o_ONE_CHAR_VALUE: + printf("%-4s ", "char"); + break; + case o_NUMBER_VALUE: + printf("%-4s ", "num"); + break; + default: + printf("%-4s ", "unk"); + } + switch (options[i].negatable) { + case o_NEGATABLE: + printf("%-3s ", "neg"); + break; + case o_NOT_NEGATABLE: + printf("%-3s ", ""); + break; + default: + printf("%-3s ", "unk"); + } + if (options[i].name) + printf("%-30s\n", options[i].name); + else + printf("\n"); + } + RETURN(finish(ZE_OK)); + } + + + /* open log file */ + if (logfile_path) { + char mode[10]; + char *p; + char *lastp; + + /* if no extension add .log */ + p = logfile_path; + /* find last / */ + lastp = NULL; + for (p = logfile_path; (p = MBSRCHR(p, '/')) != NULL; p++) { + lastp = p; + } + if (lastp == NULL) + lastp = logfile_path; + if (MBSRCHR(lastp, '.') == NULL) { + /* add .log */ + if ((p = malloc(strlen(logfile_path) + 5)) == NULL) { + ZIPERR(ZE_MEM, "logpath"); + } + strcpy(p, logfile_path); + strcat(p, ".log"); + free(logfile_path); + logfile_path = p; + } + + if (logfile_append) { + sprintf(mode, "a"); + } else { + sprintf(mode, "w"); + } + if ((logfile = zfopen(logfile_path, mode)) == NULL) { + sprintf(errbuf, "could not open logfile '%s'", logfile_path); + ZIPERR(ZE_PARMS, errbuf); + } + { + /* At top put start time and command line */ + + /* get current time */ + struct tm *now; + time_t clocktime; + + time(&clocktime); + now = localtime(&clocktime); + + fprintf(logfile, "---------\n"); + fprintf(logfile, "Zip log opened %s", asctime(now)); + fprintf(logfile, "command line arguments:\n "); + for (i = 1; args[i]; i++) { + size_t j; + int has_space = 0; + + for (j = 0; j < strlen(args[i]); j++) { + if (isspace(args[i][j])) { + has_space = 1; + break; + } + } + if (has_space) + fprintf(logfile, "\"%s\" ", args[i]); + else + fprintf(logfile, "%s ", args[i]); + } + fprintf(logfile, "\n\n"); + fflush(logfile); + } + } else { + /* only set logall if logfile open */ + logall = 0; + } + + + if (split_method && out_path) { + /* if splitting, the archive name must have .zip extension */ + int plen = strlen(out_path); + char *out_path_ext; + +#ifdef VMS + /* On VMS, adjust plen (and out_path_ext) to avoid the file version. */ + plen -= strlen( vms_file_version( out_path)); +#endif /* def VMS */ + out_path_ext = out_path+ plen- 4; + + if (plen < 4 || + out_path_ext[0] != '.' || + toupper(out_path_ext[1]) != 'Z' || + toupper(out_path_ext[2]) != 'I' || + toupper(out_path_ext[3]) != 'P') { + ZIPERR(ZE_PARMS, "archive name must end in .zip for splits"); + } + } + + + if (verbose && (dot_size == 0) && (dot_count == 0)) { + /* now default to default 10 MB dot size */ + dot_size = 10 * 0x100000; + /* show all dots as before if verbose set and dot_size not set (dot_count = 0) */ + /* maybe should turn off dots in default verbose mode */ + /* dot_size = -1; */ + } + + /* done getting -R filters so convert filterlist if not done */ + if (pcount && patterns == NULL) { + filterlist_to_patterns(); + } + +#if (defined(MSDOS) || defined(OS2)) && !defined(WIN32) + if ((kk == 3 || kk == 4) && volume_label == 1) { + /* read volume label */ + PROCNAME(NULL); + kk = 4; + } +#endif + + if (have_out && kk == 3) { + copy_only = 1; + action = ARCHIVE; + } + + if (have_out && namecmp(in_path, out_path) == 0) { + sprintf(errbuf, "--out path must be different than in path: %s", out_path); + ZIPERR(ZE_PARMS, errbuf); + } + + if (fix && diff_mode) { + ZIPERR(ZE_PARMS, "can't use --diff (-DF) with fix (-F or -FF)"); + } + + if (action == ARCHIVE && !have_out && !show_files) { + ZIPERR(ZE_PARMS, "-U (--copy) requires -O (--out)"); + } + + if (fix && !have_out) { + zipwarn("fix options -F and -FF require --out:\n", + " zip -F indamagedarchive --out outfixedarchive"); + ZIPERR(ZE_PARMS, "fix options require --out"); + } + + if (fix && !copy_only) { + ZIPERR(ZE_PARMS, "no other actions allowed when fixing archive (-F or -FF)"); + } + + if (!have_out && diff_mode) { + ZIPERR(ZE_PARMS, "-DF (--diff) requires -O (--out)"); + } + + if (diff_mode && (action == ARCHIVE || action == DELETE)) { + ZIPERR(ZE_PARMS, "can't use --diff (-DF) with -d or -U"); + } + + if (action != ARCHIVE && (recurse == 2 || pcount) && first_listarg == 0 && + !filelist && (kk < 3 || (action != UPDATE && action != FRESHEN))) { + ZIPERR(ZE_PARMS, "nothing to select from"); + } + +/* + ------------------------------------- + end of new command line code + ------------------------------------- +*/ + +#if (!defined(MACOS) && !defined(WINDLL)) + if (kk < 3) { /* zip used as filter */ + zipstdout(); + comment_stream = NULL; + if ((r = procname("-", 0)) != ZE_OK) { + if (r == ZE_MISS) { + if (bad_open_is_error) { + zipwarn("name not matched: ", "-"); + ZIPERR(ZE_OPEN, "-"); + } else { + zipwarn("name not matched: ", "-"); + } + } else { + ZIPERR(r, "-"); + } + } + kk = 4; + if (s) { + ZIPERR(ZE_PARMS, "can't use - and -@ together"); + } + } +#endif /* !MACOS && !WINDLL */ + + if (zipfile && !strcmp(zipfile, "-")) { + if (show_what_doing) { + fprintf(mesg, "sd: Zipping to stdout\n"); + fflush(mesg); + } + zip_to_stdout = 1; + } + + /* Check option combinations */ + if (special == NULL) { + ZIPERR(ZE_PARMS, "missing suffix list"); + } + if (level == 9 || !strcmp(special, ";") || !strcmp(special, ":")) + special = NULL; /* compress everything */ + + if (action == DELETE && (method != BEST || dispose || recurse || + key != NULL || comadd || zipedit)) { + zipwarn("invalid option(s) used with -d; ignored.",""); + /* reset flags - needed? */ + method = BEST; + dispose = 0; + recurse = 0; + if (key != NULL) { + free((zvoid *)key); + key = NULL; + } + comadd = 0; + zipedit = 0; + } + if (action == ARCHIVE && (method != BEST || dispose || recurse || + comadd || zipedit)) { + zipwarn("can't set method, move, recurse, or comments with copy mode.",""); + /* reset flags - needed? */ + method = BEST; + dispose = 0; + recurse = 0; + comadd = 0; + zipedit = 0; + } + if (linkput && dosify) + { + zipwarn("can't use -y with -k, -y ignored", ""); + linkput = 0; + } + if (fix == 1 && adjust) + { + zipwarn("can't use -F with -A, -F ignored", ""); + fix = 0; + } + if (fix == 2 && adjust) + { + zipwarn("can't use -FF with -A, -FF ignored", ""); + fix = 0; + } + if (test && zip_to_stdout) { + test = 0; + zipwarn("can't use -T on stdout, -T ignored", ""); + } + if (split_method && (fix || adjust)) { + ZIPERR(ZE_PARMS, "can't create split archive while fixing or adjusting\n"); + } + if (split_method && (d || zip_to_stdout)) { + ZIPERR(ZE_PARMS, "can't create split archive with -d or -g or on stdout\n"); + } + if ((action != ADD || d) && filesync) { + ZIPERR(ZE_PARMS, "can't use -d, -f, -u, -U, or -g with filesync -FS\n"); + } + if ((action != ADD || d) && zip_to_stdout) { + ZIPERR(ZE_PARMS, "can't use -d, -f, -u, -U, or -g on stdout\n"); + } +#if defined(EBCDIC) && !defined(OS390) + if (aflag==ASCII && !translate_eol) { + /* Translation to ASCII implies EOL translation! + * (on OS390, consistent EOL translation is controlled separately) + * The default translation mode is "UNIX" mode (single LF terminators). + */ + translate_eol = 2; + } +#endif +#ifdef CMS_MVS + if (aflag==ASCII && bflag) + ZIPERR(ZE_PARMS, "can't use -a with -B"); +#endif +#ifdef VMS + if (!extra_fields && vms_native) + { + zipwarn("can't use -V with -X, -V ignored", ""); + vms_native = 0; + } + if (vms_native && translate_eol) + ZIPERR(ZE_PARMS, "can't use -V with -l or -ll"); +#endif + + if (noisy) { + if (fix == 1) + zipmessage("Fix archive (-F) - assume mostly intact archive", ""); + else if (fix == 2) + zipmessage("Fix archive (-FF) - salvage what can", ""); + } + + /* Read old archive */ + + /* Now read the zip file here instead of when doing args above */ + /* Only read the central directory and build zlist */ + if (show_what_doing) { + fprintf(mesg, "sd: Reading archive\n"); + fflush(mesg); + } + + + + + /* If -FF we do it all here */ + if (fix == 2) { + + /* Open zip file and temporary output file */ + if (show_what_doing) { + fprintf(mesg, "sd: Open zip file and create temp file (-FF)\n"); + fflush(mesg); + } + diag("opening zip file and creating temporary zip file"); + x = NULL; + tempzn = 0; + if (show_what_doing) { + fprintf(mesg, "sd: Creating new zip file (-FF)\n"); + fflush(mesg); + } +#if defined(UNIX) && !defined(NO_MKSTEMP) + { + int yd; + int i; + + /* use mkstemp to avoid race condition and compiler warning */ + + if (tempath != NULL) + { + /* if -b used to set temp file dir use that for split temp */ + if ((tempzip = malloc(strlen(tempath) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, tempath); + if (lastchar(tempzip) != '/') + strcat(tempzip, "/"); + } + else + { + /* create path by stripping name and appending template */ + if ((tempzip = malloc(strlen(zipfile) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, zipfile); + for(i = strlen(tempzip); i > 0; i--) { + if (tempzip[i - 1] == '/') + break; + } + tempzip[i] = '\0'; + } + strcat(tempzip, "ziXXXXXX"); + + if ((yd = mkstemp(tempzip)) == EOF) { + ZIPERR(ZE_TEMP, tempzip); + } + if ((y = fdopen(yd, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } + } +#else + if ((tempzip = tempname(zipfile)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + if ((y = zfopen(tempzip, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } +#endif + +#if (!defined(VMS) && !defined(CMS_MVS)) + /* Use large buffer to speed up stdio: */ +#if (defined(_IOFBF) || !defined(BUFSIZ)) + zipbuf = (char *)malloc(ZBSZ); +#else + zipbuf = (char *)malloc(BUFSIZ); +#endif + if (zipbuf == NULL) { + ZIPERR(ZE_MEM, tempzip); + } +# ifdef _IOFBF + setvbuf(y, zipbuf, _IOFBF, ZBSZ); +# else + setbuf(y, zipbuf); +# endif /* _IOBUF */ +#endif /* !VMS && !CMS_MVS */ + + + if ((r = readzipfile()) != ZE_OK) { + ZIPERR(r, zipfile); + } + + /* Write central directory and end header to temporary zip */ + if (show_what_doing) { + fprintf(mesg, "sd: Writing central directory (-FF)\n"); + fflush(mesg); + } + diag("writing central directory"); + k = 0; /* keep count for end header */ + c = tempzn; /* get start of central */ + n = t = 0; + for (z = zfiles; z != NULL; z = z->nxt) + { + if ((r = putcentral(z)) != ZE_OK) { + ZIPERR(r, tempzip); + } + tempzn += 4 + CENHEAD + z->nam + z->cext + z->com; + n += z->len; + t += z->siz; + k++; + } + if (zcount == 0) + zipwarn("zip file empty", ""); + t = tempzn - c; /* compute length of central */ + diag("writing end of central directory"); + if ((r = putend(k, t, c, zcomlen, zcomment)) != ZE_OK) { + ZIPERR(r, tempzip); + } + if (fclose(y)) { + ZIPERR(d ? ZE_WRITE : ZE_TEMP, tempzip); + } + if (in_file != NULL) { + fclose(in_file); + in_file = NULL; + } + + /* Replace old zip file with new zip file, leaving only the new one */ + if (strcmp(zipfile, "-") && !d) + { + diag("replacing old zip file with new zip file"); + if ((r = replace(out_path, tempzip)) != ZE_OK) + { + zipwarn("new zip file left as: ", tempzip); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing the original zip file"); + } + free((zvoid *)tempzip); + } + tempzip = NULL; + if (zip_attributes && strcmp(zipfile, "-")) { + setfileattr(out_path, zip_attributes); +#ifdef VMS + /* If the zip file existed previously, restore its record format: */ + if (x != NULL) + (void)VMSmunch(out_path, RESTORE_RTYPE, NULL); +#endif + } + + set_filetype(out_path); + + /* finish logfile (it gets closed in freeup() called by finish()) */ + if (logfile) { + struct tm *now; + time_t clocktime; + + fprintf(logfile, "\nTotal %ld entries (", files_total); + DisplayNumString(logfile, bytes_total); + fprintf(logfile, " bytes)"); + + /* get current time */ + time(&clocktime); + now = localtime(&clocktime); + fprintf(logfile, "\nDone %s", asctime(now)); + fflush(logfile); + } + + RETURN(finish(ZE_OK)); + } + + + + /* read zipfile if exists */ + if ((r = readzipfile()) != ZE_OK) { + ZIPERR(r, zipfile); + } + +#ifndef UTIL + if (split_method == -1) { + split_method = 0; + } else if (!fix && split_method == 0 && total_disks > 1) { + /* if input archive is multi-disk and splitting has not been + enabled or disabled (split_method == -1), then automatically + set split size to same as first input split */ + zoff_t size = 0; + + in_split_path = get_in_split_path(in_path, 0); + + if (filetime(in_split_path, NULL, &size, NULL) == 0) { + zipwarn("Could not get info for input split: ", in_split_path); + return ZE_OPEN; + } + split_method = 1; + split_size = (uzoff_t) size; + + free(in_split_path); + in_split_path = NULL; + } + + if (noisy_splits && split_size > 0) + zipmessage("splitsize = ", zip_fuzofft(split_size, NULL, NULL)); +#endif + + /* so disk display starts at 1, will be updated when entries are read */ + current_in_disk = 0; + + /* no input zipfile and showing contents */ + if (!zipfile_exists && show_files && (kk == 3 || action == ARCHIVE)) { + ZIPERR(ZE_OPEN, zipfile); + } + + if (zcount == 0 && (action != ADD || d)) { + zipwarn(zipfile, " not found or empty"); + } + + if (have_out && kk == 3) { + /* no input paths so assume copy mode and match everything if --out */ + for (z = zfiles; z != NULL; z = z->nxt) { + z->mark = pcount ? filter(z->zname, filter_match_case) : 1; + } + } + + /* Scan for new files */ + + /* Process file arguments from command line */ + if (filelist) { + if (action == ARCHIVE) { + /* find in archive */ + if (show_what_doing) { + fprintf(mesg, "sd: Scanning archive entries\n"); + fflush(mesg); + } + for (; filelist; ) { + if ((r = proc_archive_name(filelist->name, filter_match_case)) != ZE_OK) { + if (r == ZE_MISS) { + char *n = NULL; +#ifdef WIN32 + /* Win9x console always uses OEM character coding, and + WinNT console is set to OEM charset by default, too */ + if ((n = malloc(strlen(filelist->name) + 1)) == NULL) + ZIPERR(ZE_MEM, "name not matched error"); + INTERN_TO_OEM(filelist->name, n); +#else + n = filelist->name; +#endif + zipwarn("not in archive: ", n); +#ifdef WIN32 + free(n); +#endif + } + else { + ZIPERR(r, filelist->name); + } + } + free(filelist->name); + filearg = filelist; + filelist = filelist->next; + free(filearg); + } + } else { + /* try find matching files on OS first then try find entries in archive */ + if (show_what_doing) { + fprintf(mesg, "sd: Scanning files\n"); + fflush(mesg); + } + for (; filelist; ) { + if ((r = PROCNAME(filelist->name)) != ZE_OK) { + if (r == ZE_MISS) { + if (bad_open_is_error) { + zipwarn("name not matched: ", filelist->name); + ZIPERR(ZE_OPEN, filelist->name); + } else { + zipwarn("name not matched: ", filelist->name); + } + } else { + ZIPERR(r, filelist->name); + } + } + free(filelist->name); + filearg = filelist; + filelist = filelist->next; + free(filearg); + } + } + } + + /* recurse from current directory for -R */ + if (recurse == 2) { +#ifdef AMIGA + if ((r = PROCNAME("")) != ZE_OK) +#else + if ((r = PROCNAME(".")) != ZE_OK) +#endif + { + if (r == ZE_MISS) { + if (bad_open_is_error) { + zipwarn("name not matched: ", "current directory for -R"); + ZIPERR(ZE_OPEN, "-R"); + } else { + zipwarn("name not matched: ", "current directory for -R"); + } + } else { + ZIPERR(r, "-R"); + } + } + } + + + if (show_what_doing) { + fprintf(mesg, "sd: Applying filters\n"); + fflush(mesg); + } + /* Clean up selections ("3 <= kk <= 5" now) */ + if (kk != 4 && first_listarg == 0 && + (action == UPDATE || action == FRESHEN)) { + /* if -u or -f with no args, do all, but, when present, apply filters */ + for (z = zfiles; z != NULL; z = z->nxt) { + z->mark = pcount ? filter(z->zname, filter_match_case) : 1; +#ifdef DOS + if (z->mark) z->dosflag = 1; /* force DOS attribs for incl. names */ +#endif + } + } + if (show_what_doing) { + fprintf(mesg, "sd: Checking dups\n"); + fflush(mesg); + } + if ((r = check_dup()) != ZE_OK) { /* remove duplicates in found list */ + if (r == ZE_PARMS) { + ZIPERR(r, "cannot repeat names in zip file"); + } + else { + ZIPERR(r, "was processing list of files"); + } + } + + if (zcount) + free((zvoid *)zsort); + + +/* + * XXX make some kind of mktemppath() function for each OS. + */ + +#ifndef VM_CMS +/* For CMS, leave tempath NULL. A-disk will be used as default. */ + /* If -b not specified, make temporary path the same as the zip file */ +#if defined(MSDOS) || defined(__human68k__) || defined(AMIGA) + if (tempath == NULL && ((p = MBSRCHR(zipfile, '/')) != NULL || +# ifdef MSDOS + (p = MBSRCHR(zipfile, '\\')) != NULL || +# endif /* MSDOS */ + (p = MBSRCHR(zipfile, ':')) != NULL)) + { + if (*p == ':') + p++; +#else +#ifdef RISCOS + if (tempath == NULL && (p = MBSRCHR(zipfile, '.')) != NULL) + { +#else +#ifdef QDOS + if (tempath == NULL && (p = LastDir(zipfile)) != NULL) + { +#else + if (tempath == NULL && (p = MBSRCHR(zipfile, '/')) != NULL) + { +#endif /* QDOS */ +#endif /* RISCOS */ +#endif /* MSDOS || __human68k__ || AMIGA */ + if ((tempath = (char *)malloc((int)(p - zipfile) + 1)) == NULL) { + ZIPERR(ZE_MEM, "was processing arguments"); + } + r = *p; *p = 0; + strcpy(tempath, zipfile); + *p = (char)r; + } +#endif /* VM_CMS */ + +#if (defined(IZ_CHECK_TZ) && defined(USE_EF_UT_TIME)) + if (!zp_tz_is_valid) { + zipwarn("TZ environment variable not found, cannot use UTC times!!",""); + } +#endif /* IZ_CHECK_TZ && USE_EF_UT_TIME */ + + /* For each marked entry, if not deleting, check if it exists, and if + updating or freshening, compare date with entry in old zip file. + Unmark if it doesn't exist or is too old, else update marked count. */ + if (show_what_doing) { + fprintf(mesg, "sd: Scanning files to update\n"); + fflush(mesg); + } +#ifdef MACOS + PrintStatProgress("Getting file information ..."); +#endif + diag("stating marked entries"); + k = 0; /* Initialize marked count */ + scan_started = 0; + scan_count = 0; + all_current = 1; + for (z = zfiles; z != NULL; z = z->nxt) { + /* if already displayed Scanning files in newname() then continue dots */ + if (noisy && scan_last) { + scan_count++; + if (scan_count % 100 == 0) { + time_t current = time(NULL); + + if (current - scan_last > scan_dot_time) { + if (scan_started == 0) { + scan_started = 1; + fprintf(mesg, " "); + fflush(mesg); + } + scan_last = current; + fprintf(mesg, "."); + fflush(mesg); + } + } + } + z->current = 0; + if (!(z->mark)) { + /* if something excluded run through the list to catch deletions */ + all_current = 0; + } + if (z->mark) { +#ifdef USE_EF_UT_TIME + iztimes f_utim, z_utim; + ulg z_tim; +#endif /* USE_EF_UT_TIME */ + Trace((stderr, "zip diagnostics: marked file=%s\n", z->oname)); + + csize = z->siz; + usize = z->len; + if (action == DELETE) { + /* only delete files in date range */ +#ifdef USE_EF_UT_TIME + z_tim = (get_ef_ut_ztime(z, &z_utim) & EB_UT_FL_MTIME) ? + unix2dostime(&z_utim.mtime) : z->tim; +#else /* !USE_EF_UT_TIME */ +# define z_tim z->tim +#endif /* ?USE_EF_UT_TIME */ + if (z_tim < before || (after && z_tim >= after)) { + /* include in archive */ + z->mark = 0; + } else { + /* delete file */ + files_total++; + /* ignore len in old archive and update to current size */ + z->len = usize; + if (csize != (uzoff_t) -1 && csize != (uzoff_t) -2) + bytes_total += csize; + k++; + } + } else if (action == ARCHIVE) { + /* only keep files in date range */ +#ifdef USE_EF_UT_TIME + z_tim = (get_ef_ut_ztime(z, &z_utim) & EB_UT_FL_MTIME) ? + unix2dostime(&z_utim.mtime) : z->tim; +#else /* !USE_EF_UT_TIME */ +# define z_tim z->tim +#endif /* ?USE_EF_UT_TIME */ + if (z_tim < before || (after && z_tim >= after)) { + /* exclude from archive */ + z->mark = 0; + } else { + /* keep file */ + files_total++; + /* ignore len in old archive and update to current size */ + z->len = usize; + if (csize != (uzoff_t) -1 && csize != (uzoff_t) -2) + bytes_total += csize; + k++; + } + } else { + int isdirname = 0; + + if (z->name && (z->name)[strlen(z->name) - 1] == '/') { + isdirname = 1; + } + +# if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) { + if (z->namew == NULL) { + if (z->uname != NULL) + z->namew = utf8_to_wchar_string(z->uname); + else + z->namew = local_to_wchar_string(z->name); + } + } +# endif + +#ifdef USE_EF_UT_TIME +# if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) + tf = filetimew(z->namew, (ulg *)NULL, (zoff_t *)&usize, &f_utim); + else + tf = filetime(z->name, (ulg *)NULL, (zoff_t *)&usize, &f_utim); +# else + tf = filetime(z->name, (ulg *)NULL, (zoff_t *)&usize, &f_utim); +# endif +#else /* !USE_EF_UT_TIME */ +# if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) + tf = filetimew(z->namew, (ulg *)NULL, (zoff_t *)&usize, NULL); + else + tf = filetime(z->name, (ulg *)NULL, (zoff_t *)&usize, NULL); +# else + tf = filetime(z->name, (ulg *)NULL, (zoff_t *)&usize, NULL); +# endif +#endif /* ?USE_EF_UT_TIME */ + if (tf == 0) + /* entry that is not on OS */ + all_current = 0; + if (tf == 0 || + tf < before || (after && tf >= after) || + ((action == UPDATE || action == FRESHEN) && +#ifdef USE_EF_UT_TIME + ((get_ef_ut_ztime(z, &z_utim) & EB_UT_FL_MTIME) ? + f_utim.mtime <= ROUNDED_TIME(z_utim.mtime) : tf <= z->tim) +#else /* !USE_EF_UT_TIME */ + tf <= z->tim +#endif /* ?USE_EF_UT_TIME */ + )) + { + z->mark = comadd ? 2 : 0; + z->trash = tf && tf >= before && + (after ==0 || tf < after); /* delete if -um or -fm */ + if (verbose) + fprintf(mesg, "zip diagnostic: %s %s\n", z->oname, + z->trash ? "up to date" : "missing or early"); + if (logfile) + fprintf(logfile, "zip diagnostic: %s %s\n", z->oname, + z->trash ? "up to date" : "missing or early"); + } + else if (diff_mode && tf == z->tim && + ((isdirname && (zoff_t)usize == -1) || (usize == z->len))) { + /* if in diff mode only include if file time or size changed */ + /* usize is -1 for directories */ + z->mark = 0; + } + else { + /* usize is -1 for directories and -2 for devices */ + if (tf == z->tim && + ((z->len == 0 && (zoff_t)usize == -1) + || usize == z->len)) { + /* FileSync uses the current flag */ + /* Consider an entry current if file time is the same + and entry size is 0 and a directory on the OS + or the entry size matches the OS size */ + z->current = 1; + } else { + all_current = 0; + } + files_total++; + if (usize != (uzoff_t) -1 && usize != (uzoff_t) -2) + /* ignore len in old archive and update to current size */ + z->len = usize; + else + z->len = 0; + if (usize != (uzoff_t) -1 && usize != (uzoff_t) -2) + bytes_total += usize; + k++; + } + } + } + } + + /* Remove entries from found list that do not exist or are too old */ + if (show_what_doing) { + fprintf(mesg, "sd: fcount = %u\n", (unsigned)fcount); + fflush(mesg); + } + + diag("stating new entries"); + scan_count = 0; + scan_started = 0; + Trace((stderr, "zip diagnostic: fcount=%u\n", (unsigned)fcount)); + for (f = found; f != NULL;) { + Trace((stderr, "zip diagnostic: new file=%s\n", f->oname)); + + if (noisy) { + /* if updating archive and update was quick, scanning for new files + can still take a long time */ + if (!zip_to_stdout && scan_last == 0 && scan_count % 100 == 0) { + time_t current = time(NULL); + + if (current - scan_start > scan_delay) { + fprintf(mesg, "Scanning files "); + fflush(mesg); + mesg_line_started = 1; + scan_last = current; + } + } + /* if already displayed Scanning files in newname() or above then continue dots */ + if (scan_last) { + scan_count++; + if (scan_count % 100 == 0) { + time_t current = time(NULL); + + if (current - scan_last > scan_dot_time) { + if (scan_started == 0) { + scan_started = 1; + fprintf(mesg, " "); + fflush(mesg); + } + scan_last = current; + fprintf(mesg, "."); + fflush(mesg); + } + } + } + } + tf = 0; + if (action != DELETE && action != FRESHEN) { +#if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) + tf = filetimew(f->namew, (ulg *)NULL, (zoff_t *)&usize, NULL); + else + tf = filetime(f->name, (ulg *)NULL, (zoff_t *)&usize, NULL); +#else + tf = filetime(f->name, (ulg *)NULL, (zoff_t *)&usize, NULL); +#endif + } + + if (action == DELETE || action == FRESHEN || + tf == 0 || + tf < before || (after && tf >= after) || + (namecmp(f->zname, zipfile) == 0 && !zip_to_stdout) + ) + f = fexpel(f); + else { + /* ??? */ + files_total++; + f->usize = 0; + if (usize != (uzoff_t) -1 && usize != (uzoff_t) -2) { + bytes_total += usize; + f->usize = usize; + } + f = f->nxt; + } + } + if (mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } +#ifdef MACOS + PrintStatProgress("done"); +#endif + + if (show_files) { + uzoff_t count = 0; + uzoff_t bytes = 0; + + if (noisy) { + fflush(mesg); + } + + if (noisy && (show_files == 1 || show_files == 3 || show_files == 5)) { + /* sf, su, sU */ + if (mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } + if (kk == 3) + /* -sf alone */ + fprintf(mesg, "Archive contains:\n"); + else if (action == DELETE) + fprintf(mesg, "Would Delete:\n"); + else if (action == FRESHEN) + fprintf(mesg, "Would Freshen:\n"); + else if (action == ARCHIVE) + fprintf(mesg, "Would Copy:\n"); + else + fprintf(mesg, "Would Add/Update:\n"); + fflush(mesg); + } + + if (logfile) { + if (logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + } + if (kk == 3) + /* -sf alone */ + fprintf(logfile, "Archive contains:\n"); + else if (action == DELETE) + fprintf(logfile, "Would Delete:\n"); + else if (action == FRESHEN) + fprintf(logfile, "Would Freshen:\n"); + else if (action == ARCHIVE) + fprintf(logfile, "Would Copy:\n"); + else + fprintf(logfile, "Would Add/Update:\n"); + fflush(logfile); + } + + for (z = zfiles; z != NULL; z = z->nxt) { + if (z->mark || kk == 3) { + count++; + if ((zoff_t)z->len > 0) + bytes += z->len; + if (noisy && (show_files == 1 || show_files == 3)) + /* sf, su */ + fprintf(mesg, " %s\n", z->oname); + if (logfile && !(show_files == 5 || show_files == 6)) + /* not sU or sU- show normal name in log */ + fprintf(logfile, " %s\n", z->oname); + +#ifdef UNICODE_TEST + if (create_files) { + int r; + int dir = 0; + FILE *f; + +#if defined(UNICODE_SUPPORT) && defined(WIN32) + char *fn = NULL; + wchar_t *fnw = NULL; + + if (!no_win32_wide) { + if ((fnw = malloc((wcslen(z->znamew) + 120) * sizeof(wchar_t))) == NULL) + ZIPERR(ZE_MEM, "sC"); + wcscpy(fnw, L"testdir/"); + wcscat(fnw, z->znamew); + if (fnw[wcslen(fnw) - 1] == '/') + dir = 1; + if (dir) + r = _wmkdir(fnw); + else + f = _wfopen(fnw, L"w"); + } else { + if ((fn = malloc(strlen(z->zname) + 120)) == NULL) + ZIPERR(ZE_MEM, "sC"); + strcpy(fn, "testdir/"); + strcat(fn, z->zname); + if (fn[strlen(fn) - 1] == '/') + dir = 1; + if (dir) + r = mkdir(fn); + else + f = fopen(fn, "w"); + } +#else + char *fn = NULL; + if ((fn = malloc(strlen(z->zname) + 120)) == NULL) + ZIPERR(ZE_MEM, "sC"); + strcpy(fn, "testdir/"); + if (z->uname) + strcat(fn, z->uname); + else + strcat(fn, z->zname); + + if (fn[strlen(fn) - 1] == '/') + dir = 1; + if (dir) + r = mkdir(fn, 0777); + else + f = fopen(fn, "w"); +#endif + if (dir) { + if (r) { + if (errno != 17) { + printf(" - could not create directory testdir/%s\n", z->oname); + perror(" dir"); + } + } else { + printf(" - created directory testdir/%s\n", z->oname); + } + } else { + if (f == NULL) { + printf(" - could not open testdir/%s\n", z->oname); + perror(" file"); + } else { + fclose(f); + printf(" - created testdir/%s\n", z->oname); + if (z->uname) + printf(" u - created testdir/%s\n", z->uname); + } + } + } +#endif +#ifdef UNICODE_SUPPORT + if (show_files == 3 || show_files == 4) { + /* su, su- */ + /* Include escaped Unicode name if exists under standard name */ + if (z->ouname) { + if (noisy && show_files == 3) + fprintf(mesg, " Escaped Unicode: %s\n", z->ouname); + if (logfile) + fprintf(logfile, " Escaped Unicode: %s\n", z->ouname); + } + } + if (show_files == 5 || show_files == 6) { + /* sU, sU- */ + /* Display only escaped Unicode name if exists or standard name */ + if (z->ouname) { + /* Unicode name */ + if (noisy && show_files == 5) { + fprintf(mesg, " %s\n", z->ouname); + } + if (logfile) { + fprintf(logfile, " %s\n", z->ouname); + } + } else { + /* No Unicode name so use standard name */ + if (noisy && show_files == 5) { + fprintf(mesg, " %s\n", z->oname); + } + if (logfile) { + fprintf(logfile, " %s\n", z->oname); + } + } + } +#endif + } + } + for (f = found; f != NULL; f = f->nxt) { + count++; + if ((zoff_t)f->usize > 0) + bytes += f->usize; +#ifdef UNICODE_SUPPORT + if (unicode_escape_all) { + char *escaped_unicode; + escaped_unicode = local_to_escape_string(f->zname); + if (noisy && (show_files == 1 || show_files == 3 || show_files == 5)) + /* sf, su, sU */ + fprintf(mesg, " %s\n", escaped_unicode); + if (logfile) + fprintf(logfile, " %s\n", escaped_unicode); + free(escaped_unicode); + } else { +#endif + if (noisy && (show_files == 1 || show_files == 3 || show_files == 5)) + /* sf, su, sU */ + fprintf(mesg, " %s\n", f->oname); + if (logfile) + fprintf(logfile, " %s\n", f->oname); +#ifdef UNICODE_SUPPORT + } +#endif + } + if (noisy || logfile == NULL) + fprintf(mesg, "Total %s entries (%s bytes)\n", + zip_fuzofft(count, NULL, NULL), + zip_fuzofft(bytes, NULL, NULL)); + if (logfile) + fprintf(logfile, "Total %s entries (%s bytes)\n", + zip_fuzofft(count, NULL, NULL), + zip_fuzofft(bytes, NULL, NULL)); + RETURN(finish(ZE_OK)); + } + + /* Make sure there's something left to do */ + if (k == 0 && found == NULL && !diff_mode && + !(zfiles == NULL && allow_empty_archive) && + !(zfiles != NULL && + (latest || fix || adjust || junk_sfx || comadd || zipedit))) { + if (test && (zfiles != NULL || zipbeg != 0)) { +#ifndef WINDLL + check_zipfile(zipfile, argv[0]); +#endif + RETURN(finish(ZE_OK)); + } + if (action == UPDATE || action == FRESHEN) { + RETURN(finish(ZE_NONE)); + } + else if (zfiles == NULL && (latest || fix || adjust || junk_sfx)) { + ZIPERR(ZE_NAME, zipfile); + } +#ifndef WINDLL + else if (recurse && (pcount == 0) && (first_listarg > 0)) { +#ifdef VMS + strcpy(errbuf, "try: zip \""); + for (i = 1; i < (first_listarg - 1); i++) + strcat(strcat(errbuf, args[i]), "\" "); + strcat(strcat(errbuf, args[i]), " *.* -i"); +#else /* !VMS */ + strcpy(errbuf, "try: zip"); + for (i = 1; i < first_listarg; i++) + strcat(strcat(errbuf, " "), args[i]); +# ifdef AMIGA + strcat(errbuf, " \"\" -i"); +# else + strcat(errbuf, " . -i"); +# endif +#endif /* ?VMS */ + for (i = first_listarg; i < argc; i++) + strcat(strcat(errbuf, " "), args[i]); + ZIPERR(ZE_NONE, errbuf); + } + else { + ZIPERR(ZE_NONE, zipfile); + } +#endif /* !WINDLL */ + } + + if (filesync && all_current && fcount == 0) { + zipmessage("Archive is current", ""); + RETURN(finish(ZE_OK)); + } + + d = (d && k == 0 && (zipbeg || zfiles != NULL)); /* d true if appending */ + +#if CRYPT + /* Initialize the crc_32_tab pointer, when encryption was requested. */ + if (key != NULL) { + crc_32_tab = get_crc_table(); +#ifdef EBCDIC + /* convert encryption key to ASCII (ISO variant for 8-bit ASCII chars) */ + strtoasc(key, key); +#endif /* EBCDIC */ + } +#endif /* CRYPT */ + + /* Just ignore the spanning signature if a multi-disk archive */ + if (zfiles && total_disks != 1 && zipbeg == 4) { + zipbeg = 0; + } + + /* Before we get carried away, make sure zip file is writeable. This + * has the undesired side effect of leaving one empty junk file on a WORM, + * so when the zipfile does not exist already and when -b is specified, + * the writability check is made in replace(). + */ + if (strcmp(zipfile, "-")) + { + if (tempdir && zfiles == NULL && zipbeg == 0) { + zip_attributes = 0; + } else { + x = (have_out || (zfiles == NULL && zipbeg == 0)) ? zfopen(out_path, FOPW) : + zfopen(out_path, FOPM); + /* Note: FOPW and FOPM expand to several parameters for VMS */ + if (x == NULL) { + ZIPERR(ZE_CREAT, out_path); + } + fclose(x); + zip_attributes = getfileattr(out_path); + if (zfiles == NULL && zipbeg == 0) + destroy(out_path); + } + } + else + zip_attributes = 0; + + /* Throw away the garbage in front of the zip file for -J */ + if (junk_sfx) zipbeg = 0; + + /* Open zip file and temporary output file */ + if (show_what_doing) { + fprintf(mesg, "sd: Open zip file and create temp file\n"); + fflush(mesg); + } + diag("opening zip file and creating temporary zip file"); + x = NULL; + tempzn = 0; + if (strcmp(zipfile, "-") == 0) + { +#ifdef MSDOS + /* It is nonsense to emit the binary data stream of a zipfile to + * the (text mode) console. This case should already have been caught + * in a call to zipstdout() far above. Therefore, if the following + * failsafe check detects a console attached to stdout, zip is stopped + * with an "internal logic error"! */ + if (isatty(fileno(stdout))) + ZIPERR(ZE_LOGIC, "tried to write binary zipfile data to console!"); + /* Set stdout mode to binary for MSDOS systems */ +# ifdef __HIGHC__ + setmode(stdout, _BINARY); +# else + setmode(fileno(stdout), O_BINARY); +# endif + y = zfdopen(fileno(stdout), FOPW); +#else + y = stdout; +#endif + /* tempzip must be malloced so a later free won't barf */ + tempzip = malloc(4); + if (tempzip == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, "-"); + } + else if (d) /* d true if just appending (-g) */ + { + if (total_disks > 1) { + ZIPERR(ZE_PARMS, "cannot grow split archive"); + } + if ((y = zfopen(zipfile, FOPM)) == NULL) { + ZIPERR(ZE_NAME, zipfile); + } + tempzip = zipfile; + /* + tempzf = y; + */ + + if (zfseeko(y, cenbeg, SEEK_SET)) { + ZIPERR(ferror(y) ? ZE_READ : ZE_EOF, zipfile); + } + bytes_this_split = cenbeg; + tempzn = cenbeg; + } + else + { + if (show_what_doing) { + fprintf(mesg, "sd: Creating new zip file\n"); + fflush(mesg); + } + /* See if there is something at beginning of disk 1 to copy. + If not, do nothing as zipcopy() will open files to read + as needed. */ + if (zipbeg) { + in_split_path = get_in_split_path(in_path, 0); + + while ((in_file = zfopen(in_split_path, FOPR_EX)) == NULL) { + /* could not open split */ + + /* Ask for directory with split. Updates in_path */ + if (ask_for_split_read_path(0) != ZE_OK) { + ZIPERR(ZE_ABORT, "could not open archive to read"); + } + free(in_split_path); + in_split_path = get_in_split_path(in_path, 1); + } + } +#if defined(UNIX) && !defined(NO_MKSTEMP) + { + int yd; + int i; + + /* use mkstemp to avoid race condition and compiler warning */ + + if (tempath != NULL) + { + /* if -b used to set temp file dir use that for split temp */ + if ((tempzip = malloc(strlen(tempath) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, tempath); + if (lastchar(tempzip) != '/') + strcat(tempzip, "/"); + } + else + { + /* create path by stripping name and appending template */ + if ((tempzip = malloc(strlen(zipfile) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, zipfile); + for(i = strlen(tempzip); i > 0; i--) { + if (tempzip[i - 1] == '/') + break; + } + tempzip[i] = '\0'; + } + strcat(tempzip, "ziXXXXXX"); + + if ((yd = mkstemp(tempzip)) == EOF) { + ZIPERR(ZE_TEMP, tempzip); + } + if ((y = fdopen(yd, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } + } +#else + if ((tempzip = tempname(zipfile)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + if ((y = zfopen(tempzip, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } +#endif + } + +#if (!defined(VMS) && !defined(CMS_MVS)) + /* Use large buffer to speed up stdio: */ +#if (defined(_IOFBF) || !defined(BUFSIZ)) + zipbuf = (char *)malloc(ZBSZ); +#else + zipbuf = (char *)malloc(BUFSIZ); +#endif + if (zipbuf == NULL) { + ZIPERR(ZE_MEM, tempzip); + } +# ifdef _IOFBF + setvbuf(y, zipbuf, _IOFBF, ZBSZ); +# else + setbuf(y, zipbuf); +# endif /* _IOBUF */ +#endif /* !VMS && !CMS_MVS */ + + /* If not seekable set some flags 3/14/05 EG */ + output_seekable = 1; + if (!is_seekable(y)) { + output_seekable = 0; + use_descriptors = 1; + } + + /* Not needed. Only need Zip64 when input file is larger than 2 GB or reading + stdin and writing stdout. This is set in putlocal() for each file. */ +#if 0 + /* If using descriptors and Zip64 enabled force Zip64 3/13/05 EG */ +# ifdef ZIP64_SUPPORT + if (use_descriptors && force_zip64 != 0) { + force_zip64 = 1; + } +# endif +#endif + + /* if archive exists, not streaming and not deleting or growing, copy + any bytes at beginning */ + if (strcmp(zipfile, "-") != 0 && !d) /* this must go *after* set[v]buf */ + { + /* copy anything before archive */ + if (in_file && zipbeg && (r = bfcopy(zipbeg)) != ZE_OK) { + ZIPERR(r, r == ZE_TEMP ? tempzip : zipfile); + } + if (in_file) { + fclose(in_file); + in_file = NULL; + free(in_split_path); + } + tempzn = zipbeg; + if (split_method) { + /* add spanning signature */ + if (show_what_doing) { + fprintf(mesg, "sd: Adding spanning/splitting signature at top of archive\n"); + fflush(mesg); + } + /* write the spanning signature at the top of the archive */ + errbuf[0] = 0x50 /*'P' except for EBCDIC*/; + errbuf[1] = 0x4b /*'K' except for EBCDIC*/; + errbuf[2] = 7; + errbuf[3] = 8; + bfwrite(errbuf, 1, 4, BFWRITE_DATA); + /* tempzn updated below */ + tempzn += 4; + } + } + + o = 0; /* no ZE_OPEN errors yet */ + + + /* Process zip file, updating marked files */ +#ifdef DEBUG + if (zfiles != NULL) + diag("going through old zip file"); +#endif + if (zfiles != NULL && show_what_doing) { + fprintf(mesg, "sd: Going through old zip file\n"); + fflush(mesg); + } + w = &zfiles; + while ((z = *w) != NULL) { + if (z->mark == 1) + { + uzoff_t len; + if ((zoff_t)z->len == -1) + /* device */ + len = 0; + else + len = z->len; + + /* if not deleting, zip it up */ + if (action != ARCHIVE && action != DELETE) + { + struct zlist far *localz; /* local header */ + + if (verbose || !(filesync && z->current)) + DisplayRunningStats(); + if (noisy) + { + if (action == FRESHEN) { + fprintf(mesg, "freshening: %s", z->oname); + mesg_line_started = 1; + fflush(mesg); + } else if (filesync && z->current) { + if (verbose) { + fprintf(mesg, " ok: %s", z->oname); + mesg_line_started = 1; + fflush(mesg); + } + } else if (!(filesync && z->current)) { + fprintf(mesg, "updating: %s", z->oname); + mesg_line_started = 1; + fflush(mesg); + } + } + if (logall) + { + if (action == FRESHEN) { + fprintf(logfile, "freshening: %s", z->oname); + logfile_line_started = 1; + fflush(logfile); + } else if (filesync && z->current) { + if (verbose) { + fprintf(logfile, " current: %s", z->oname); + logfile_line_started = 1; + fflush(logfile); + } + } else { + fprintf(logfile, "updating: %s", z->oname); + logfile_line_started = 1; + fflush(logfile); + } + } + + /* Get local header flags and extra fields */ + if (readlocal(&localz, z) != ZE_OK) { + zipwarn("could not read local entry information: ", z->oname); + z->lflg = z->flg; + z->ext = 0; + } else { + z->lflg = localz->lflg; + z->ext = localz->ext; + z->extra = localz->extra; + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + } + + if (!(filesync && z->current) && + (r = zipup(z)) != ZE_OK && r != ZE_OPEN && r != ZE_MISS) + { + zipmessage_nl("", 1); + /* + if (noisy) + { + if (mesg_line_started) { +#if (!defined(MACOS) && !defined(WINDLL)) + putc('\n', mesg); + fflush(mesg); +#else + fprintf(stdout, "\n"); + fflush(stdout); +#endif + mesg_line_started = 0; + } + } + if (logall) { + if (logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + } + */ + sprintf(errbuf, "was zipping %s", z->name); + ZIPERR(r, errbuf); + } + if (filesync && z->current) + { + /* if filesync if entry matches OS just copy */ + if ((r = zipcopy(z)) != ZE_OK) + { + sprintf(errbuf, "was copying %s", z->oname); + ZIPERR(r, errbuf); + } + zipmessage_nl("", 1); + /* + if (noisy) + { + if (mesg_line_started) { +#if (!defined(MACOS) && !defined(WINDLL)) + putc('\n', mesg); + fflush(mesg); +#else + fprintf(stdout, "\n"); + fflush(stdout); +#endif + mesg_line_started = 0; + } + } + if (logall) { + if (logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + } + */ + } + if (r == ZE_OPEN || r == ZE_MISS) + { + o = 1; + zipmessage_nl("", 1); + /* + if (noisy) + { +#if (!defined(MACOS) && !defined(WINDLL)) + putc('\n', mesg); + fflush(mesg); +#else + fprintf(stdout, "\n"); +#endif + mesg_line_started = 0; + } + if (logall) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + */ + if (r == ZE_OPEN) { + perror(z->oname); + zipwarn("could not open for reading: ", z->oname); + if (bad_open_is_error) { + sprintf(errbuf, "was zipping %s", z->name); + ZIPERR(r, errbuf); + } + } else { + zipwarn("file and directory with the same name: ", z->oname); + } + zipwarn("will just copy entry over: ", z->oname); + if ((r = zipcopy(z)) != ZE_OK) + { + sprintf(errbuf, "was copying %s", z->oname); + ZIPERR(r, errbuf); + } + z->mark = 0; + } + files_so_far++; + good_bytes_so_far += z->len; + bytes_so_far += len; + w = &z->nxt; + } + else if (action == ARCHIVE) + { +#ifdef DEBUG + zoff_t here = zftello(y); +#endif + + DisplayRunningStats(); + if (skip_this_disk - 1 != z->dsk) + /* moved to another disk so start copying again */ + skip_this_disk = 0; + if (skip_this_disk - 1 == z->dsk) { + /* skipping this disk */ + if (noisy) { + fprintf(mesg, " skipping: %s", z->oname); + mesg_line_started = 1; + fflush(mesg); + } + if (logall) { + fprintf(logfile, " skipping: %s", z->oname); + logfile_line_started = 1; + fflush(logfile); + } + } else { + /* copying this entry */ + if (noisy) { + fprintf(mesg, " copying: %s", z->oname); + if (display_usize) { + fprintf(mesg, " ("); + DisplayNumString(mesg, z->len ); + fprintf(mesg, ")"); + } + mesg_line_started = 1; + fflush(mesg); + } + if (logall) + { + fprintf(logfile, " copying: %s", z->oname); + if (display_usize) { + fprintf(logfile, " ("); + DisplayNumString(logfile, z->len ); + fprintf(logfile, ")"); + } + logfile_line_started = 1; + fflush(logfile); + } + } + + if (skip_this_disk - 1 == z->dsk) + /* skip entries on this disk */ + z->mark = 0; + else if ((r = zipcopy(z)) != ZE_OK) + { + if (r == ZE_ABORT) { + ZIPERR(r, "user requested abort"); + } else if (fix != 1) { + /* exit */ + sprintf(errbuf, "was copying %s", z->oname); + zipwarn("(try -F to attempt to fix)", ""); + ZIPERR(r, errbuf); + } + else /* if (r == ZE_FORM) */ { +#ifdef DEBUG + zoff_t here = zftello(y); +#endif + + /* seek back in output to start of this entry so can overwrite */ + if (zfseeko(y, current_local_offset, SEEK_SET) != 0){ + ZIPERR(r, "could not seek in output file"); + } + zipwarn("bad - skipping: ", z->oname); +#ifdef DEBUG + here = zftello(y); +#endif + tempzn = current_local_offset; + bytes_this_split = current_local_offset; + } + } + if (skip_this_disk || !(fix == 1 && r != ZE_OK)) + { + if (noisy && mesg_line_started) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + fflush(mesg); + } + if (logall && logfile_line_started) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + } + /* input counts */ + files_so_far++; + if (r != ZE_OK) + bad_bytes_so_far += z->siz; + else + good_bytes_so_far += z->siz; + bytes_so_far += z->siz; + + if (r != ZE_OK && fix == 1) { + /* remove bad entry from list */ + v = z->nxt; /* delete entry from list */ + free((zvoid *)(z->iname)); + free((zvoid *)(z->zname)); + free(z->oname); +#ifdef UNICODE_SUPPORT + if (z->uname) free(z->uname); +#endif /* def UNICODE_SUPPORT */ + if (z->ext) + /* don't have local extra until zipcopy reads it */ + if (z->extra) free((zvoid *)(z->extra)); + if (z->cext && z->cextra != z->extra) + free((zvoid *)(z->cextra)); + if (z->com) + free((zvoid *)(z->comment)); + farfree((zvoid far *)z); + *w = v; + zcount--; + } else { + w = &z->nxt; + } + +#ifdef WINDLL +#ifdef ZIP64_SUPPORT + /* int64 support in caller */ + if (lpZipUserFunctions->ServiceApplication64 != NULL) + { + if ((*lpZipUserFunctions->ServiceApplication64)(z->zname, z->siz)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + else + { + /* no int64 support in caller */ + filesize64 = z->siz; + low = (unsigned long)(filesize64 & 0x00000000FFFFFFFF); + high = (unsigned long)((filesize64 >> 32) & 0x00000000FFFFFFFF); + if (lpZipUserFunctions->ServiceApplication64_No_Int64 != NULL) { + if ((*lpZipUserFunctions->ServiceApplication64_No_Int64)(z->zname, low, high)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + } +#else + if (lpZipUserFunctions->ServiceApplication != NULL) { + if ((*lpZipUserFunctions->ServiceApplication)(z->zname, z->siz)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } +#endif /* ZIP64_SUPPORT - I added comments around // comments - does that help below? EG */ +/* strange but true: if I delete this and put these two endifs adjacent to + each other, the Aztec Amiga compiler never sees the second endif! WTF?? PK */ +#endif /* WINDLL */ + } + else + { + DisplayRunningStats(); + if (noisy) + { + fprintf(mesg, "deleting: %s", z->oname); + if (display_usize) { + fprintf(mesg, " ("); + DisplayNumString(mesg, z->len ); + fprintf(mesg, ")"); + } + fflush(mesg); + fprintf(mesg, "\n"); + } + if (logall) + { + fprintf(logfile, "deleting: %s", z->oname); + if (display_usize) { + fprintf(logfile, " ("); + DisplayNumString(logfile, z->len ); + fprintf(logfile, ")"); + } + fprintf(logfile, "\n"); + fflush(logfile); + } + files_so_far++; + good_bytes_so_far += z->siz; + bytes_so_far += z->siz; +#ifdef WINDLL +#ifdef ZIP64_SUPPORT + /* int64 support in caller */ + if (lpZipUserFunctions->ServiceApplication64 != NULL) + { + if ((*lpZipUserFunctions->ServiceApplication64)(z->zname, z->siz)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + else + { + /* no int64 support in caller */ + filesize64 = z->siz; + low = (unsigned long)(filesize64 & 0x00000000FFFFFFFF); + high = (unsigned long)((filesize64 >> 32) & 0x00000000FFFFFFFF); + if (lpZipUserFunctions->ServiceApplication64_No_Int64 != NULL) { + if ((*lpZipUserFunctions->ServiceApplication64_No_Int64)(z->zname, low, high)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + } +#else + if (lpZipUserFunctions->ServiceApplication != NULL) { + if ((*lpZipUserFunctions->ServiceApplication)(z->zname, z->siz)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } +#endif /* ZIP64_SUPPORT - I added comments around // comments - does that help below? EG */ +/* strange but true: if I delete this and put these two endifs adjacent to + each other, the Aztec Amiga compiler never sees the second endif! WTF?? PK */ +#endif /* WINDLL */ + + v = z->nxt; /* delete entry from list */ + free((zvoid *)(z->iname)); + free((zvoid *)(z->zname)); + free(z->oname); +#ifdef UNICODE_SUPPORT + if (z->uname) free(z->uname); +#endif /* def UNICODE_SUPPORT */ + if (z->ext) + /* don't have local extra until zipcopy reads it */ + if (z->extra) free((zvoid *)(z->extra)); + if (z->cext && z->cextra != z->extra) + free((zvoid *)(z->cextra)); + if (z->com) + free((zvoid *)(z->comment)); + farfree((zvoid far *)z); + *w = v; + zcount--; + } + } + else + { + if (action == ARCHIVE) { + v = z->nxt; /* delete entry from list */ + free((zvoid *)(z->iname)); + free((zvoid *)(z->zname)); + free(z->oname); +#ifdef UNICODE_SUPPORT + if (z->uname) free(z->uname); +#endif /* def UNICODE_SUPPORT */ + if (z->ext) + /* don't have local extra until zipcopy reads it */ + if (z->extra) free((zvoid *)(z->extra)); + if (z->cext && z->cextra != z->extra) + free((zvoid *)(z->cextra)); + if (z->com) + free((zvoid *)(z->comment)); + farfree((zvoid far *)z); + *w = v; + zcount--; + } + else + { + if (filesync) { + /* Delete entries if don't match a file on OS */ + BlankRunningStats(); + if (noisy) + { + fprintf(mesg, "deleting: %s", z->oname); + if (display_usize) { + fprintf(mesg, " ("); + DisplayNumString(mesg, z->len ); + fprintf(mesg, ")"); + } + fflush(mesg); + fprintf(mesg, "\n"); + mesg_line_started = 0; + } + if (logall) + { + fprintf(logfile, "deleting: %s", z->oname); + if (display_usize) { + fprintf(logfile, " ("); + DisplayNumString(logfile, z->len ); + fprintf(logfile, ")"); + } + fprintf(logfile, "\n"); + fflush(logfile); + logfile_line_started = 0; + } + } + /* copy the original entry */ + else if (!d && !diff_mode && (r = zipcopy(z)) != ZE_OK) + { + sprintf(errbuf, "was copying %s", z->oname); + ZIPERR(r, errbuf); + } + w = &z->nxt; + } + } + } + + + /* Process the edited found list, adding them to the zip file */ + if (show_what_doing) { + fprintf(mesg, "sd: Zipping up new entries\n"); + fflush(mesg); + } + diag("zipping up new entries, if any"); + Trace((stderr, "zip diagnostic: fcount=%u\n", (unsigned)fcount)); + for (f = found; f != NULL; f = fexpel(f)) + { + uzoff_t len; + /* add a new zfiles entry and set the name */ + if ((z = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + ZIPERR(ZE_MEM, "was adding files to zip file"); + } + z->nxt = NULL; + z->name = f->name; + f->name = NULL; +#ifdef UNICODE_SUPPORT + z->uname = NULL; /* UTF-8 name for extra field */ + z->zuname = NULL; /* externalized UTF-8 name for matching */ + z->ouname = NULL; /* display version of UTF-8 name with OEM */ + +#if 0 + /* New AppNote bit 11 allowing storing UTF-8 in path */ + if (utf8_force && f->uname) { + if (f->iname) + free(f->iname); + if ((f->iname = malloc(strlen(f->uname) + 1)) == NULL) + ZIPERR(ZE_MEM, "Unicode bit 11"); + strcpy(f->iname, f->uname); +# ifdef WIN32 + if (f->inamew) + free(f->inamew); + f->inamew = utf8_to_wchar_string(f->iname); +# endif + } +#endif + + /* Only set z->uname if have a non-ASCII Unicode name */ + /* The Unicode path extra field is created if z->uname is not NULL, + unless on a UTF-8 system, then instead of creating the extra field + set bit 11 in the General Purpose Bit Flag */ + { + int is_ascii = 0; + +# ifdef WIN32 + if (!no_win32_wide) + is_ascii = is_ascii_stringw(f->inamew); + else + is_ascii = is_ascii_string(f->uname); +# else + is_ascii = is_ascii_string(f->uname); +# endif + + if (z->uname == NULL) { + if (!is_ascii) + z->uname = f->uname; + else + free(f->uname); + } else { + free(f->uname); + } + } + f->uname = NULL; + +#endif + z->iname = f->iname; + f->iname = NULL; + z->zname = f->zname; + f->zname = NULL; + z->oname = f->oname; + f->oname = NULL; +#if defined(UNICODE_SUPPORT) && defined(WIN32) + z->namew = f->namew; + f->namew = NULL; + z->inamew = f->inamew; + f->inamew = NULL; + z->znamew = f->znamew; + f->znamew = NULL; +#endif + z->ext = z->cext = z->com = 0; + z->extra = z->cextra = NULL; + z->mark = 1; + z->dosflag = f->dosflag; + /* zip it up */ + DisplayRunningStats(); + if (noisy) + { + fprintf(mesg, " adding: %s", z->oname); + mesg_line_started = 1; + fflush(mesg); + } + if (logall) + { + fprintf(logfile, " adding: %s", z->oname); + logfile_line_started = 1; + fflush(logfile); + } + /* initial scan */ + len = f->usize; + if ((r = zipup(z)) != ZE_OK && r != ZE_OPEN && r != ZE_MISS) + { + zipmessage_nl("", 1); + /* + if (noisy) + { +#if (!defined(MACOS) && !defined(WINDLL)) + putc('\n', mesg); + fflush(mesg); +#else + fprintf(stdout, "\n"); +#endif + mesg_line_started = 0; + fflush(mesg); + } + if (logall) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + */ + sprintf(errbuf, "was zipping %s", z->oname); + ZIPERR(r, errbuf); + } + if (r == ZE_OPEN || r == ZE_MISS) + { + o = 1; + zipmessage_nl("", 1); + /* + if (noisy) + { +#if (!defined(MACOS) && !defined(WINDLL)) + putc('\n', mesg); + fflush(mesg); +#else + fprintf(stdout, "\n"); +#endif + mesg_line_started = 0; + fflush(mesg); + } + if (logall) { + fprintf(logfile, "\n"); + logfile_line_started = 0; + fflush(logfile); + } + */ + if (r == ZE_OPEN) { + perror("zip warning"); + if (logfile) + fprintf(logfile, "zip warning: %s\n", strerror(errno)); + zipwarn("could not open for reading: ", z->oname); + if (bad_open_is_error) { + sprintf(errbuf, "was zipping %s", z->name); + ZIPERR(r, errbuf); + } + } else { + zipwarn("file and directory with the same name: ", z->oname); + } + files_so_far++; + bytes_so_far += len; + bad_files_so_far++; + bad_bytes_so_far += len; + free((zvoid *)(z->name)); + free((zvoid *)(z->iname)); + free((zvoid *)(z->zname)); + free(z->oname); +#ifdef UNICODE_SUPPORT + if (z->uname) + free(z->uname); +# ifdef WIN32 + if (z->namew) + free((zvoid *)(z->namew)); + if (z->inamew) + free((zvoid *)(z->inamew)); + if (z->znamew) + free((zvoid *)(z->znamew)); +# endif +#endif + farfree((zvoid far *)z); + } + else + { + files_so_far++; + /* current size of file (just before reading) */ + good_bytes_so_far += z->len; + /* size of file on initial scan */ + bytes_so_far += len; + *w = z; + w = &z->nxt; + zcount++; + } + } + if (key != NULL) + { + free((zvoid *)key); + key = NULL; + } + + /* final status 3/17/05 EG */ + if (noisy && bad_files_so_far) + { + char tempstrg[100]; + + fprintf(mesg, "\nzip warning: Not all files were readable\n"); + fprintf(mesg, " files/entries read: %lu", files_total - bad_files_so_far); + WriteNumString(good_bytes_so_far, tempstrg); + fprintf(mesg, " (%s bytes)", tempstrg); + fprintf(mesg, " skipped: %lu", bad_files_so_far); + WriteNumString(bad_bytes_so_far, tempstrg); + fprintf(mesg, " (%s bytes)\n", tempstrg); + fflush(mesg); + } + if (logfile && bad_files_so_far) + { + char tempstrg[100]; + + fprintf(logfile, "\nzip warning: Not all files were readable\n"); + fprintf(logfile, " files/entries read: %lu", files_total - bad_files_so_far); + WriteNumString(good_bytes_so_far, tempstrg); + fprintf(logfile, " (%s bytes)", tempstrg); + fprintf(logfile, " skipped: %lu", bad_files_so_far); + WriteNumString(bad_bytes_so_far, tempstrg); + fprintf(logfile, " (%s bytes)", tempstrg); + } + + /* Get one line comment for each new entry */ + if (show_what_doing) { + fprintf(mesg, "sd: Get comment if any\n"); + fflush(mesg); + } +#if defined(AMIGA) || defined(MACOS) + if (comadd || filenotes) + { + if (comadd) +#else + if (comadd) + { +#endif + { + if (comment_stream == NULL) { +#ifndef RISCOS + comment_stream = (FILE*)fdopen(fileno(stderr), "r"); +#else + comment_stream = stderr; +#endif + } + if ((e = malloc(MAXCOM + 1)) == NULL) { + ZIPERR(ZE_MEM, "was reading comment lines"); + } + } +#ifdef __human68k__ + setmode(fileno(comment_stream), O_TEXT); +#endif +#ifdef MACOS + if (noisy) fprintf(mesg, "\nStart commenting files ...\n"); +#endif + for (z = zfiles; z != NULL; z = z->nxt) + if (z->mark) +#if defined(AMIGA) || defined(MACOS) + if (filenotes && (p = GetComment(z->zname))) + { + if (z->comment = malloc(k = strlen(p)+1)) + { + z->com = k; + strcpy(z->comment, p); + } + else + { + free((zvoid *)e); + ZIPERR(ZE_MEM, "was reading filenotes"); + } + } + else if (comadd) +#endif /* AMIGA || MACOS */ + { + if (noisy) + fprintf(mesg, "Enter comment for %s:\n", z->oname); + if (fgets(e, MAXCOM+1, comment_stream) != NULL) + { + if ((p = malloc((extent)(k = strlen(e))+1)) == NULL) + { + free((zvoid *)e); + ZIPERR(ZE_MEM, "was reading comment lines"); + } + strcpy(p, e); + if (p[k-1] == '\n') + p[--k] = 0; + z->comment = p; + /* zip64 support 09/05/2003 R.Nausedat */ + z->com = (extent)k; + } + } +#ifdef MACOS + if (noisy) fprintf(mesg, "\n...done"); +#endif +#if defined(AMIGA) || defined(MACOS) + if (comadd) + free((zvoid *)e); + GetComment(NULL); /* makes it free its internal storage */ +#else + free((zvoid *)e); +#endif + } + + /* Get multi-line comment for the zip file */ + if (zipedit) + { +#ifndef WINDLL + if (comment_stream == NULL) { +#ifndef RISCOS + comment_stream = (FILE*)fdopen(fileno(stderr), "r"); +#else + comment_stream = stderr; +#endif + } + if ((e = malloc(MAXCOM + 1)) == NULL) { + ZIPERR(ZE_MEM, "was reading comment lines"); + } + if (noisy && zcomlen) + { + fputs("current zip file comment is:\n", mesg); + fwrite(zcomment, 1, zcomlen, mesg); + if (zcomment[zcomlen-1] != '\n') + putc('\n', mesg); + free((zvoid *)zcomment); + } + if ((zcomment = malloc(1)) == NULL) + ZIPERR(ZE_MEM, "was setting comments to null"); + zcomment[0] = '\0'; + if (noisy) + fputs("enter new zip file comment (end with .):\n", mesg); +#if (defined(AMIGA) && (defined(LATTICE)||defined(__SASC))) + flushall(); /* tty input/output is out of sync here */ +#endif +#ifdef __human68k__ + setmode(fileno(comment_stream), O_TEXT); +#endif +#ifdef MACOS + printf("\n enter new zip file comment \n"); + if (fgets(e, MAXCOM+1, comment_stream) != NULL) { + if ((p = malloc((k = strlen(e))+1)) == NULL) { + free((zvoid *)e); + ZIPERR(ZE_MEM, "was reading comment lines"); + } + strcpy(p, e); + if (p[k-1] == '\n') p[--k] = 0; + zcomment = p; + } +#else /* !MACOS */ + while (fgets(e, MAXCOM+1, comment_stream) != NULL && strcmp(e, ".\n")) + { + if (e[(r = strlen(e)) - 1] == '\n') + e[--r] = 0; + if ((p = malloc((*zcomment ? strlen(zcomment) + 3 : 1) + r)) == NULL) + { + free((zvoid *)e); + ZIPERR(ZE_MEM, "was reading comment lines"); + } + if (*zcomment) + strcat(strcat(strcpy(p, zcomment), "\r\n"), e); + else + strcpy(p, *e ? e : "\r\n"); + free((zvoid *)zcomment); + zcomment = p; + } +#endif /* ?MACOS */ + free((zvoid *)e); +#else /* WINDLL */ + comment(zcomlen); + if ((p = malloc(strlen(szCommentBuf)+1)) == NULL) { + ZIPERR(ZE_MEM, "was setting comments to null"); + } + if (szCommentBuf[0] != '\0') + lstrcpy(p, szCommentBuf); + else + p[0] = '\0'; + free((zvoid *)zcomment); + GlobalUnlock(hStr); + GlobalFree(hStr); + zcomment = p; +#endif /* WINDLL */ + zcomlen = strlen(zcomment); + } + + if (display_globaldots) { +#ifndef WINDLL + putc('\n', mesg); +#else + fprintf(stdout,"%c",'\n'); +#endif + mesg_line_started = 0; + } + + /* Write central directory and end header to temporary zip */ + if (show_what_doing) { + fprintf(mesg, "sd: Writing central directory\n"); + fflush(mesg); + } + diag("writing central directory"); + k = 0; /* keep count for end header */ + c = tempzn; /* get start of central */ + n = t = 0; + for (z = zfiles; z != NULL; z = z->nxt) + { + if (z->mark || !(diff_mode || filesync)) { + if ((r = putcentral(z)) != ZE_OK) { + ZIPERR(r, tempzip); + } + tempzn += 4 + CENHEAD + z->nam + z->cext + z->com; + n += z->len; + t += z->siz; + k++; + } + } + + if (k == 0) + zipwarn("zip file empty", ""); + if (verbose) { + fprintf(mesg, "total bytes=%s, compressed=%s -> %d%% savings\n", + zip_fzofft(n, NULL, "u"), zip_fzofft(t, NULL, "u"), percent(n, t)); + fflush(mesg); + } + if (logall) { + fprintf(logfile, "total bytes=%s, compressed=%s -> %d%% savings\n", + zip_fzofft(n, NULL, "u"), zip_fzofft(t, NULL, "u"), percent(n, t)); + fflush(logfile); + } + t = tempzn - c; /* compute length of central */ + diag("writing end of central directory"); + if (show_what_doing) { + fprintf(mesg, "sd: Writing end of central directory\n"); + fflush(mesg); + } + + if ((r = putend(k, t, c, zcomlen, zcomment)) != ZE_OK) { + ZIPERR(r, tempzip); + } + + /* + tempzf = NULL; + */ + if (fclose(y)) { + ZIPERR(d ? ZE_WRITE : ZE_TEMP, tempzip); + } + y = NULL; + if (in_file != NULL) { + fclose(in_file); + in_file = NULL; + } + /* + if (x != NULL) + fclose(x); + */ + + /* Free some memory before spawning unzip */ +#ifdef USE_ZLIB + zl_deflate_free(); +#else + lm_free(); +#endif +#ifdef BZIP2_SUPPORT + bz_compress_free(); +#endif + +#ifndef WINDLL + /* Test new zip file before overwriting old one or removing input files */ + if (test) + check_zipfile(tempzip, argv[0]); +#endif + /* Replace old zip file with new zip file, leaving only the new one */ + if (strcmp(zipfile, "-") && !d) + { + diag("replacing old zip file with new zip file"); + if (show_what_doing) { + fprintf(mesg, "sd: Replacing old zip file\n"); + fflush(mesg); + } + if ((r = replace(out_path, tempzip)) != ZE_OK) + { + zipwarn("new zip file left as: ", tempzip); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing the original zip file"); + } + free((zvoid *)tempzip); + } + tempzip = NULL; + if (zip_attributes && strcmp(zipfile, "-")) { + setfileattr(out_path, zip_attributes); +#ifdef VMS + /* If the zip file existed previously, restore its record format: */ + if (x != NULL) + (void)VMSmunch(out_path, RESTORE_RTYPE, NULL); +#endif + } + if (strcmp(zipfile, "-")) { + if (show_what_doing) { + fprintf(mesg, "sd: Setting file type\n"); + fflush(mesg); + } + + set_filetype(out_path); + } + +#if defined(WIN32) + /* All looks good so, if requested, clear the DOS archive bits */ + if (clear_archive_bits) { + if (noisy) + zipmessage("Clearing archive bits...", ""); + for (z = zfiles; z != NULL; z = z->nxt) + { +# ifdef UNICODE_SUPPORT + if (z->mark) { + if (!no_win32_wide) { + if (!ClearArchiveBitW(z->namew)){ + zipwarn("Could not clear archive bit for: ", z->oname); + } + } else { + if (!ClearArchiveBit(z->name)){ + zipwarn("Could not clear archive bit for: ", z->oname); + } + } + } +# else + if (!ClearArchiveBit(z->name)){ + zipwarn("Could not clear archive bit for: ", z->oname); + } +# endif + } + } +#endif + + /* finish logfile (it gets closed in freeup() called by finish()) */ + if (logfile) { + struct tm *now; + time_t clocktime; + + fprintf(logfile, "\nTotal %ld entries (", files_total); + if (good_bytes_so_far != bytes_total) { + fprintf(logfile, "planned "); + DisplayNumString(logfile, bytes_total); + fprintf(logfile, " bytes, actual "); + DisplayNumString(logfile, good_bytes_so_far); + fprintf(logfile, " bytes)"); + } else { + DisplayNumString(logfile, bytes_total); + fprintf(logfile, " bytes)"); + } + + /* get current time */ + + time(&clocktime); + now = localtime(&clocktime); + fprintf(logfile, "\nDone %s", asctime(now)); + } + + /* Finish up (process -o, -m, clean up). Exit code depends on o. */ +#if (!defined(VMS) && !defined(CMS_MVS)) + free((zvoid *) zipbuf); +#endif /* !VMS && !CMS_MVS */ + RETURN(finish(o ? ZE_OPEN : ZE_OK)); +} diff --git a/third_party/infozip/zip/zip.h b/third_party/infozip/zip/zip.h new file mode 100644 index 000000000..ba7743b1c --- /dev/null +++ b/third_party/infozip/zip/zip.h @@ -0,0 +1,1082 @@ +/* clang-format off */ +/* + zip.h - Zip 3 + +/---------------------------------------------------------------------/ + +Info-ZIP Licence + +This is version 2007-Mar-4 of the Info-ZIP license. +The definitive version of this document should be available at +ftp://ftp.info-zip.org/pub/infozip/license.html indefinitely and +a copy at http://www.info-zip.org/pub/infozip/license.html. + + +Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + +For the purposes of this copyright and license, "Info-ZIP" is defined as +the following set of individuals: + + Mark Adler, John Bush, Karl Davis, Harald Denker, Jean-Michel Dubois, + Jean-loup Gailly, Hunter Goatley, Ed Gordon, Ian Gorman, Chris Herborth, + Dirk Haase, Greg Hartwig, Robert Heath, Jonathan Hudson, Paul Kienitz, + David Kirschbaum, Johnny Lee, Onno van der Linden, Igor Mandrichenko, + Steve P. Miller, Sergio Monesi, Keith Owens, George Petrov, Greg Roelofs, + Kai Uwe Rommel, Steve Salisbury, Dave Smith, Steven M. Schweda, + Christian Spieler, Cosmin Truta, Antoine Verheijen, Paul von Behren, + Rich Wales, Mike White. + +This software is provided "as is," without warranty of any kind, express +or implied. In no event shall Info-ZIP or its contributors be held liable +for any direct, indirect, incidental, special or consequential damages +arising out of the use of or inability to use this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the above disclaimer and the following restrictions: + + 1. Redistributions of source code (in whole or in part) must retain + the above copyright notice, definition, disclaimer, and this list + of conditions. + + 2. Redistributions in binary form (compiled executables and libraries) + must reproduce the above copyright notice, definition, disclaimer, + and this list of conditions in documentation and/or other materials + provided with the distribution. The sole exception to this condition + is redistribution of a standard UnZipSFX binary (including SFXWiz) as + part of a self-extracting archive; that is permitted without inclusion + of this license, as long as the normal SFX banner has not been removed + from the binary or disabled. + + 3. Altered versions--including, but not limited to, ports to new operating + systems, existing ports with new graphical interfaces, versions with + modified or added functionality, and dynamic, shared, or static library + versions not from Info-ZIP--must be plainly marked as such and must not + be misrepresented as being the original source or, if binaries, + compiled from the original source. Such altered versions also must not + be misrepresented as being Info-ZIP releases--including, but not + limited to, labeling of the altered versions with the names "Info-ZIP" + (or any variation thereof, including, but not limited to, different + capitalizations), "Pocket UnZip," "WiZ" or "MacZip" without the + explicit permission of Info-ZIP. Such altered versions are further + prohibited from misrepresentative use of the Zip-Bugs or Info-ZIP + e-mail addresses or the Info-ZIP URL(s), such as to imply Info-ZIP + will provide support for the altered versions. + + 4. Info-ZIP retains the right to use the names "Info-ZIP," "Zip," "UnZip," + "UnZipSFX," "WiZ," "Pocket UnZip," "Pocket Zip," and "MacZip" for its + own source and binary releases. + +/---------------------------------------------------------------------/ + +*/ + +/* + * zip.h by Mark Adler + */ +#ifndef __zip_h +#define __zip_h 1 + +#define ZIP /* for crypt.c: include zip password functions, not unzip */ + +/* Types centralized here for easy modification */ +#define local static /* More meaningful outside functions */ +typedef unsigned char uch; /* unsigned 8-bit value */ +typedef unsigned short ush; /* unsigned 16-bit value */ +typedef unsigned long ulg; /* unsigned 32-bit value */ + +/* Set up portability */ +#include "third_party/infozip/zip/tailor.h" + +#ifdef USE_ZLIB +# include "third_party/zlib/zlib.h" +#endif + +/* In the utilities, the crc32() function is only used for UNICODE_SUPPORT. */ +#if defined(UTIL) && !defined(UNICODE_SUPPORT) +# define CRC_TABLE_ONLY +#endif + +#define MIN_MATCH 3 +#define MAX_MATCH 258 +/* The minimum and maximum match lengths */ + +#ifndef WSIZE +# define WSIZE (0x8000) +#endif +/* Maximum window size = 32K. If you are really short of memory, compile + * with a smaller WSIZE but this reduces the compression ratio for files + * of size > WSIZE. WSIZE must be a power of two in the current implementation. + */ + +#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) +/* Minimum amount of lookahead, except at the end of the input file. + * See deflate.c for comments about the MIN_MATCH+1. + */ + +#define MAX_DIST (WSIZE-MIN_LOOKAHEAD) +/* In order to simplify the code, particularly on 16 bit machines, match + * distances are limited to MAX_DIST instead of WSIZE. + */ + +/* Forget FILENAME_MAX (incorrectly = 14 on some System V) */ +#ifdef DOS +# define FNMAX 256 +#else +# define FNMAX 1024 +#endif + +#ifndef MATCH +# define MATCH shmatch /* Default for pattern matching: UNIX style */ +#endif + +/* Structure carrying extended timestamp information */ +typedef struct iztimes { + time_t atime; /* new access time */ + time_t mtime; /* new modification time */ + time_t ctime; /* new creation time (!= Unix st.ctime) */ +} iztimes; + +/* Lengths of headers after signatures in bytes */ +#define LOCHEAD 26 +#define CENHEAD 42 +#define ENDHEAD 18 +#define EC64LOC 16 +#define EC64REC 52 + +/* Structures for in-memory file information */ +struct zlist { + /* See central header in zipfile.c for what vem..off are */ + /* Do not rearrange these as less than smart coding in zipfile.c + in scanzipf_reg() depends on u being set to ver and then stepping + through as a byte array. Ack. Should be fixed. 5/25/2005 EG */ + /* All the new read code does not rely on this order. */ + ush vem, ver, flg, how; + ulg tim, crc; + uzoff_t siz, len; /* zip64 support 08/29/2003 R.Nausedat */ + /* changed from extent to ush 3/10/2005 EG */ + ush nam, ext, cext, com; /* offset of ext must be >= LOCHEAD */ + ulg dsk; /* disk number was ush but now ulg */ + ush att, lflg; /* offset of lflg must be >= LOCHEAD */ + uzoff_t off; + ulg atx; + char *name; /* File name in zip file */ + char *extra; /* Extra field (set only if ext != 0) */ + char *cextra; /* Extra in central (set only if cext != 0) */ + char *comment; /* Comment (set only if com != 0) */ + char *iname; /* Internal file name after cleanup (stored in archive) */ + char *zname; /* External version of internal name */ + char *oname; /* Display version of name used in messages */ +#ifdef UNICODE_SUPPORT + /* Unicode support */ + char *uname; /* UTF-8 version of iname */ + /* if uname has chars not in local char set, zuname can be different than zname */ + char *zuname; /* Escaped Unicode zname from uname */ + char *ouname; /* Display version of zuname */ +# ifdef WIN32 + char *wuname; /* Converted back ouname for Win32 */ + wchar_t *namew; /* Windows wide character version of name */ + wchar_t *inamew; /* Windows wide character version of iname */ + wchar_t *znamew; /* Windows wide character version of zname */ +# endif +#endif + int mark; /* Marker for files to operate on */ + int trash; /* Marker for files to delete */ + int current; /* Marker for files that are current to what is on OS (filesync) */ + int dosflag; /* Set to force MSDOS file attributes */ + struct zlist far *nxt; /* Pointer to next header in list */ +}; +struct flist { + char *name; /* Raw internal file name */ + char *iname; /* Internal file name after cleanup */ + char *zname; /* External version of internal name */ + char *oname; /* Display version of internal name */ +#ifdef UNICODE_SUPPORT + char *uname; /* UTF-8 name */ +# ifdef WIN32 + wchar_t *namew; /* Windows wide character version of name */ + wchar_t *inamew; /* Windows wide character version of iname */ + wchar_t *znamew; /* Windows wide character version of zname */ +# endif +#endif + int dosflag; /* Set to force MSDOS file attributes */ + uzoff_t usize; /* usize from initial scan */ + struct flist far *far *lst; /* Pointer to link pointing here */ + struct flist far *nxt; /* Link to next name */ +}; +struct plist { + char *zname; /* External version of internal name */ + int select; /* Selection flag ('i' or 'x') */ +}; + +/* internal file attribute */ +#define UNKNOWN (-1) +#define BINARY 0 +#define ASCII 1 +#define __EBCDIC 2 + +/* extra field definitions */ +#define EF_VMCMS 0x4704 /* VM/CMS Extra Field ID ("G")*/ +#define EF_MVS 0x470f /* MVS Extra Field ID ("G") */ +#define EF_IZUNIX 0x5855 /* UNIX Extra Field ID ("UX") */ +#define EF_IZUNIX2 0x7855 /* Info-ZIP's new Unix ("Ux") */ +#define EF_TIME 0x5455 /* universal timestamp ("UT") */ +#define EF_OS2EA 0x0009 /* OS/2 Extra Field ID (extended attributes) */ +#define EF_ACL 0x4C41 /* ACL Extra Field ID (access control list, "AL") */ +#define EF_NTSD 0x4453 /* NT Security Descriptor Extra Field ID, ("SD") */ +#define EF_BEOS 0x6542 /* BeOS Extra Field ID ("Be") */ +#define EF_ATHEOS 0x7441 /* AtheOS Extra Field ID ("At") */ +#define EF_QDOS 0xfb4a /* SMS/QDOS ("J\373") */ +#define EF_AOSVS 0x5356 /* AOS/VS ("VS") */ +#define EF_SPARK 0x4341 /* David Pilling's Acorn/SparkFS ("AC") */ +#define EF_THEOS 0x6854 /* THEOS ("Th") */ +#define EF_TANDEM 0x4154 /* Tandem NSK ("TA") */ + +/* Definitions for extra field handling: */ +#define EF_SIZE_MAX ((unsigned)0xFFFF) /* hard limit of total e.f. length */ +#define EB_HEADSIZE 4 /* length of a extra field block header */ +#define EB_ID 0 /* offset of block ID in header */ +#define EB_LEN 2 /* offset of data length field in header */ +#define EB_MEMCMPR_HSIZ 6 /* header length for memcompressed data */ +#define EB_DEFLAT_EXTRA 10 /* overhead for 64kByte "undeflatable" data */ + +#define EB_UX_MINLEN 8 /* minimal "UX" field contains atime, mtime */ +#define EB_UX_ATIME 0 /* offset of atime in "UX" extra field data */ +#define EB_UX_MTIME 4 /* offset of mtime in "UX" extra field data */ + +#define EB_UX_FULLSIZE 12 /* full "UX" field (atime, mtime, uid, gid) */ +#define EB_UX_UID 8 /* byte offset of UID in "UX" field data */ +#define EB_UX_GID 10 /* byte offset of GID in "UX" field data */ + +#define EB_UT_MINLEN 1 /* minimal UT field contains Flags byte */ +#define EB_UT_FLAGS 0 /* byte offset of Flags field */ +#define EB_UT_TIME1 1 /* byte offset of 1st time value */ +#define EB_UT_FL_MTIME (1 << 0) /* mtime present */ +#define EB_UT_FL_ATIME (1 << 1) /* atime present */ +#define EB_UT_FL_CTIME (1 << 2) /* ctime present */ +#define EB_UT_LEN(n) (EB_UT_MINLEN + 4 * (n)) + +#define EB_UX2_MINLEN 4 /* minimal Ux field contains UID/GID */ +#define EB_UX2_UID 0 /* byte offset of UID in "Ux" field data */ +#define EB_UX2_GID 2 /* byte offset of GID in "Ux" field data */ +#define EB_UX2_VALID (1 << 8) /* UID/GID present */ + +/* ASCII definitions for line terminators in text files: */ +#define LF 10 /* '\n' on ASCII machines; must be 10 due to EBCDIC */ +#define CR 13 /* '\r' on ASCII machines; must be 13 due to EBCDIC */ +#define CTRLZ 26 /* DOS & OS/2 EOF marker (used in fileio.c, vms.c) */ + +/* return codes of password fetches (negative: user abort; positive: error) */ +#define IZ_PW_ENTERED 0 /* got some PWD string, use/try it */ +#define IZ_PW_CANCEL -1 /* no password available (for this entry) */ +#define IZ_PW_CANCELALL -2 /* no password, skip any further PWD request */ +#define IZ_PW_ERROR 5 /* = PK_MEM2 : failure (no mem, no tty, ...) */ +#define IZ_PW_SKIPVERIFY IZ_PW_CANCEL /* skip encrypt. passwd verification */ + +/* mode flag values of password prompting function */ +#define ZP_PW_ENTER 0 /* request for encryption password */ +#define ZP_PW_VERIFY 1 /* request for reentering password */ + +/* Error return codes and PERR macro */ +#include "third_party/infozip/zip/ziperr.h" + +#if 0 /* Optimization: use the (const) result of crc32(0L,NULL,0) */ +# define CRCVAL_INITIAL crc32(0L, (uch *)NULL, 0) +# if 00 /* not used, should be removed !! */ +# define ADLERVAL_INITIAL adler16(0U, (uch *)NULL, 0) +# endif /* 00 */ +#else +# define CRCVAL_INITIAL 0L +# if 00 /* not used, should be removed !! */ +# define ADLERVAL_INITIAL 1 +# endif /* 00 */ +#endif + +#define DOSTIME_MINIMUM ((ulg)0x00210000L) +#define DOSTIME_2038_01_18 ((ulg)0x74320000L) + + +/* Public globals */ +extern uch upper[256]; /* Country dependent case map table */ +extern uch lower[256]; +#ifdef EBCDIC +extern ZCONST uch ascii[256]; /* EBCDIC <--> ASCII translation tables */ +extern ZCONST uch ebcdic[256]; +#endif /* EBCDIC */ +#if (!defined(USE_ZLIB) || defined(USE_OWN_CRCTAB)) + extern ZCONST ulg near *crc_32_tab; +#else + extern ZCONST ulg Far *crc_32_tab; +#endif + +/* Are these ever used? 6/12/05 EG */ +#ifdef IZ_ISO2OEM_ARRAY /* ISO 8859-1 (Win CP 1252) --> OEM CP 850 */ +extern ZCONST uch Far iso2oem[128]; +#endif +#ifdef IZ_OEM2ISO_ARRAY /* OEM CP 850 --> ISO 8859-1 (Win CP 1252) */ +extern ZCONST uch Far oem2iso[128]; +#endif + +extern char errbuf[FNMAX+4081]; /* Handy place to build error messages */ +extern int recurse; /* Recurse into directories encountered */ +extern int dispose; /* Remove files after put in zip file */ +extern int pathput; /* Store path with name */ + +#ifdef RISCOS +extern int scanimage; /* Scan through image files */ +#endif + +#define BEST -1 /* Use best method (deflation or store) */ +#define STORE 0 /* Store method */ +#define DEFLATE 8 /* Deflation method*/ +#define BZIP2 12 /* BZIP2 method */ +#ifdef BZIP2_SUPPORT +#define LAST_KNOWN_COMPMETHOD BZIP2 +#else +#define LAST_KNOWN_COMPMETHOD DEFLATE +#endif + +extern int method; /* Restriction on compression method */ + +extern ulg skip_this_disk; +extern int des_good; /* Good data descriptor found */ +extern ulg des_crc; /* Data descriptor CRC */ +extern uzoff_t des_csize; /* Data descriptor csize */ +extern uzoff_t des_usize; /* Data descriptor usize */ +extern int dosify; /* Make new entries look like MSDOS */ +extern char *special; /* Don't compress special suffixes */ +extern int verbose; /* Report oddities in zip file structure */ +extern int fix; /* Fix the zip file */ +extern int filesync; /* 1=file sync, delete entries not on file system */ +extern int adjust; /* Adjust the unzipsfx'd zip file */ +extern int level; /* Compression level */ +extern int translate_eol; /* Translate end-of-line LF -> CR LF */ +#ifdef VMS + extern int vmsver; /* Append VMS version number to file names */ + extern int vms_native; /* Store in VMS format */ + extern int vms_case_2; /* ODS2 file name case in VMS. -1: down. */ + extern int vms_case_5; /* ODS5 file name case in VMS. +1: preserve. */ + +/* Accomodation for /NAMES = AS_IS with old header files. */ +# define cma$tis_errno_get_addr CMA$TIS_ERRNO_GET_ADDR +# define lib$establish LIB$ESTABLISH +# define lib$get_foreign LIB$GET_FOREIGN +# define lib$get_input LIB$GET_INPUT +# define lib$sig_to_ret LIB$SIG_TO_RET +# define ots$cvt_tu_l OTS$CVT_TU_L +# define str$concat STR$CONCAT +# define str$find_first_substring STR$FIND_FIRST_SUBSTRING +# define str$free1_dx STR$FREE1_DX +# define sys$asctim SYS$ASCTIM +# define sys$assign SYS$ASSIGN +# define sys$bintim SYS$BINTIM +# define sys$close SYS$CLOSE +# define sys$connect SYS$CONNECT +# define sys$dassgn SYS$DASSGN +# define sys$display SYS$DISPLAY +# define sys$getjpiw SYS$GETJPIW +# define sys$open SYS$OPEN +# define sys$parse SYS$PARSE +# define sys$qiow SYS$QIOW +# define sys$read SYS$READ +# define sys$search SYS$SEARCH +#endif /* VMS */ +#if defined(OS2) || defined(WIN32) + extern int use_longname_ea; /* use the .LONGNAME EA as the file's name */ +#endif +#if defined (QDOS) || defined(QLZIP) +extern short qlflag; +#endif +/* 9/26/04 EG */ +extern int no_wild; /* wildcards are disabled */ +extern int allow_regex; /* 1 = allow [list] matching (regex) */ +extern int wild_stop_at_dir; /* wildcards do not include / in matches */ +#ifdef UNICODE_SUPPORT + extern int using_utf8; /* 1 if current character set is UTF-8 */ +# ifdef WIN32 + extern int no_win32_wide; /* 1 = no wide functions, like GetFileAttributesW() */ +# endif +#endif +/* 10/20/04 */ +extern zoff_t dot_size; /* if not 0 then display dots every size buffers */ +extern zoff_t dot_count; /* if dot_size not 0 counts buffers */ +/* status 10/30/04 */ +extern int display_counts; /* display running file count */ +extern int display_bytes; /* display running bytes remaining */ +extern int display_globaldots; /* display dots for archive instead of for each file */ +extern int display_volume; /* display current input and output volume (disk) numbers */ +extern int display_usize; /* display uncompressed bytes */ +extern ulg files_so_far; /* files processed so far */ +extern ulg bad_files_so_far; /* files skipped so far */ +extern ulg files_total; /* files total to process */ +extern uzoff_t bytes_so_far; /* bytes processed so far (from initial scan) */ +extern uzoff_t good_bytes_so_far;/* good bytes read so far */ +extern uzoff_t bad_bytes_so_far;/* bad bytes skipped so far */ +extern uzoff_t bytes_total; /* total bytes to process (from initial scan) */ +/* logfile 6/5/05 */ +extern int logall; /* 0 = warnings/errors, 1 = all */ +extern FILE *logfile; /* pointer to open logfile or NULL */ +extern int logfile_append; /* append to existing logfile */ +extern char *logfile_path; /* pointer to path of logfile */ +#ifdef WIN32 +extern int nonlocal_name; /* Name has non-local characters */ +extern int nonlocal_path; /* Path has non-local characters */ +#endif +#ifdef UNICODE_SUPPORT +/* Unicode 10/12/05 */ +extern int use_wide_to_mb_default;/* use the default MB char instead of escape */ +#endif + +extern int hidden_files; /* process hidden and system files */ +extern int volume_label; /* add volume label */ +extern int dirnames; /* include directory names */ +extern int filter_match_case; /* 1=match case when filter() */ +extern int diff_mode; /* 1=require --out and only store changed and add */ +#if defined(WIN32) +extern int only_archive_set; /* only include if DOS archive bit set */ +extern int clear_archive_bits; /* clear DOS archive bit of included files */ +#endif +extern int linkput; /* Store symbolic links as such */ +extern int noisy; /* False for quiet operation */ +extern int extra_fields; /* 0=create minimum, 1=don't copy old, 2=keep old */ +#ifdef NTSD_EAS + extern int use_privileges; /* use security privilege overrides */ +#endif +extern int use_descriptors; /* use data descriptors (extended headings) */ +extern int allow_empty_archive; /* if no files, create empty archive anyway */ +extern int copy_only; /* 1 = copy archive with no changes */ +extern int zip_to_stdout; /* output to stdout */ +extern int output_seekable; /* 1 = output seekable 3/13/05 EG */ +#ifdef ZIP64_SUPPORT /* zip64 globals 10/4/03 E. Gordon */ + extern int force_zip64; /* force use of zip64 when streaming from stdin */ + extern int zip64_entry; /* current entry needs Zip64 */ + extern int zip64_archive; /* at least 1 entry needs zip64 */ +#endif +extern int allow_fifo; /* Allow reading Unix FIFOs, waiting if pipe open */ +extern int show_files; /* show files to operate on and exit (=2 log only) */ + +extern char *tempzip; /* temp file name */ +extern FILE *y; /* output file now global for splits */ + +#ifdef UNICODE_SUPPORT + extern int utf8_force; /* 1=store UTF-8 as standard per AppNote bit 11 */ +#endif +extern int unicode_escape_all; /* 1=escape all non-ASCII characters in paths */ +extern int unicode_mismatch; /* unicode mismatch is 0=error, 1=warn, 2=ignore, 3=no */ + +extern time_t scan_delay; /* seconds before display Scanning files message */ +extern time_t scan_dot_time; /* time in seconds between Scanning files dots */ +extern time_t scan_start; /* start of file scan */ +extern time_t scan_last; /* time of last message */ +extern int scan_started; /* scan has started */ +extern uzoff_t scan_count; /* Used for "Scanning files..." message */ + +extern ulg before; /* 0=ignore, else exclude files before this time */ +extern ulg after; /* 0=ignore, else exclude files newer than this time */ + +/* in split globals */ + +extern ulg total_disks; + +extern ulg current_in_disk; +extern uzoff_t current_in_offset; +extern ulg skip_current_disk; + + +/* out split globals */ + +extern ulg current_local_disk; /* disk with current local header */ + +extern ulg current_disk; /* current disk number */ +extern ulg cd_start_disk; /* central directory start disk */ +extern uzoff_t cd_start_offset; /* offset of start of cd on cd start disk */ +extern uzoff_t cd_entries_this_disk; /* cd entries this disk */ +extern uzoff_t total_cd_entries; /* total cd entries in new/updated archive */ +extern ulg zip64_eocd_disk; /* disk with Zip64 EOCD Record */ +extern uzoff_t zip64_eocd_offset; /* offset of Zip64 EOCD Record */ +/* for split method 1 (keep split with local header open and update) */ +extern char *current_local_tempname; /* name of temp file */ +extern FILE *current_local_file; /* file pointer for current local header */ +extern uzoff_t current_local_offset; /* offset to start of current local header */ +/* global */ +extern uzoff_t bytes_this_split; /* bytes written to current split */ +extern int read_split_archive; /* 1=scanzipf_reg detected spanning signature */ +extern int split_method; /* 0=no splits, 1=seekable, 2=data descs, -1=no */ +extern uzoff_t split_size; /* how big each split should be */ +extern int split_bell; /* when pause for next split ring bell */ +extern uzoff_t bytes_prev_splits; /* total bytes written to all splits before this */ +extern uzoff_t bytes_this_entry; /* bytes written for this entry across all splits */ +extern int noisy_splits; /* note when splits are being created */ +extern int mesg_line_started; /* 1=started writing a line to mesg */ +extern int logfile_line_started; /* 1=started writing a line to logfile */ +extern char *key; /* Scramble password or NULL */ +extern char *tempath; /* Path for temporary files */ +extern FILE *mesg; /* Where informational output goes */ +extern char *zipfile; /* New or existing zip archive (zip file) */ +extern FILE *in_file; /* Current input file for spits */ +extern char *in_path; /* Name of input archive, used to track reading splits */ +extern char *in_split_path; /* in split path */ +extern char *out_path; /* Name of output file, usually same as zipfile */ +extern int zip_attributes; + +/* zip64 support 08/31/2003 R.Nausedat */ +extern uzoff_t zipbeg; /* Starting offset of zip structures */ +extern uzoff_t cenbeg; /* Starting offset of central directory */ +extern uzoff_t tempzn; /* Count of bytes written to output zip file */ + +/* NOTE: zcount and fcount cannot exceed "size_t" (resp. "extent") range. + This is an internal limitation built into Zip's action handling: + Zip keeps "{z|f}count * struct {z|f}list" arrays in (flat) memory, + for sorting, file matching, and building the central-dir structures. + */ + +extern struct zlist far *zfiles;/* Pointer to list of files in zip file */ +extern extent zcount; /* Number of files in zip file */ +extern int zipfile_exists; /* 1 if zipfile exists */ +extern ush zcomlen; /* Length of zip file comment */ +extern char *zcomment; /* Zip file comment (not zero-terminated) */ +extern struct flist far **fsort;/* List of files sorted by name */ +extern struct zlist far **zsort;/* List of files sorted by name */ +#ifdef UNICODE_SUPPORT +extern struct zlist far **zusort;/* List of files sorted by zuname */ +#endif +extern struct flist far *found; /* List of names found */ +extern struct flist far *far *fnxt; /* Where to put next in found list */ +extern extent fcount; /* Count of names in found list */ + +extern struct plist *patterns; /* List of patterns to be matched */ +extern unsigned pcount; /* number of patterns */ +extern unsigned icount; /* number of include only patterns */ +extern unsigned Rcount; /* number of -R include patterns */ + +#ifdef IZ_CHECK_TZ +extern int zp_tz_is_valid; /* signals "timezone info is available" */ +#endif +#if (defined(MACOS) || defined(WINDLL)) +extern int zipstate; /* flag "zipfile has been stat()'ed */ +#endif + +/* Diagnostic functions */ +#ifdef DEBUG +# ifdef MSDOS +# undef stderr +# define stderr stdout +# endif +# define diag(where) fprintf(stderr, "zip diagnostic: %s\n", where) +# define Assert(cond,msg) {if(!(cond)) error(msg);} +# ifdef THEOS +# define Trace(x) _fprintf x +# define Tracev(x) {if (verbose) _fprintf x ;} +# define Tracevv(x) {if (verbose>1) _fprintf x ;} +# define Tracec(c,x) {if (verbose && (c)) _fprintf x ;} +# define Tracecv(c,x) {if (verbose>1 && (c)) _fprintf x ;} +# else +# define Trace(x) fprintf x +# define Tracev(x) {if (verbose) fprintf x ;} +# define Tracevv(x) {if (verbose>1) fprintf x ;} +# define Tracec(c,x) {if (verbose && (c)) fprintf x ;} +# define Tracecv(c,x) {if (verbose>1 && (c)) fprintf x ;} +# endif +#else +# define diag(where) +# define Assert(cond,msg) +# define Trace(x) +# define Tracev(x) +# define Tracevv(x) +# define Tracec(c,x) +# define Tracecv(c,x) +#endif + +#ifdef DEBUGNAMES +# define free(x) { int *v;Free(x); v=x;*v=0xdeadbeef;x=(void *)0xdeadbeef; } +#endif + +/* Public function prototypes */ + +#ifndef UTIL +#ifdef USE_ZIPMAIN +int zipmain OF((int, char **)); +#else +int main OF((int, char **)); +#endif /* USE_ZIPMAIN */ +#endif + +#ifdef EBCDIC +extern int aflag; +#endif /* EBCDIC */ +#ifdef CMS_MVS +extern int bflag; +#endif /* CMS_MVS */ +void zipmessage_nl OF((ZCONST char *, int)); +void zipmessage OF((ZCONST char *, ZCONST char *)); +void zipwarn OF((ZCONST char *, ZCONST char *)); +void ziperr OF((int, ZCONST char *)); +#ifdef UTIL +# define error(msg) ziperr(ZE_LOGIC, msg) +#else + void error OF((ZCONST char *)); +# ifdef VMSCLI + void help OF((void)); +# endif + int encr_passwd OF((int, char *, int, ZCONST char *)); +#endif + + /* in zipup.c */ +#ifndef UTIL + /* zip64 support 08/31/2003 R.Nausedat */ + int percent OF((uzoff_t, uzoff_t)); + + int zipup OF((struct zlist far *)); +# ifdef USE_ZLIB + void zl_deflate_free OF((void)); +# else + void flush_outbuf OF((char *, unsigned *)); + int seekable OF((void)); + extern unsigned (*read_buf) OF((char *, unsigned int)); +# endif /* !USE_ZLIB */ +# ifdef ZP_NEED_MEMCOMPR + ulg memcompress OF((char *, ulg, char *, ulg)); +# endif +# ifdef BZIP2_SUPPORT + void bz_compress_free OF((void)); +# endif +#endif /* !UTIL */ + + /* in zipfile.c */ +#ifndef UTIL + struct zlist far *zsearch OF((ZCONST char *)); +# ifdef USE_EF_UT_TIME + int get_ef_ut_ztime OF((struct zlist far *, iztimes *)); +# endif /* USE_EF_UT_TIME */ + int trash OF((void)); +#endif /* !UTIL */ +char *ziptyp OF((char *)); +int readzipfile OF((void)); +int putlocal OF((struct zlist far *, int)); +int putextended OF((struct zlist far *)); +int putcentral OF((struct zlist far *)); +/* zip64 support 09/05/2003 R.Nausedat */ +int putend OF((uzoff_t, uzoff_t, uzoff_t, extent, char *)); +/* moved seekable to separate function 3/14/05 EG */ +int is_seekable OF((FILE *)); +int zipcopy OF((struct zlist far *)); +int readlocal OF((struct zlist far **, struct zlist far *)); +/* made global for handling extra fields */ +char *get_extra_field OF((ush, char *, unsigned)); +char *copy_nondup_extra_fields OF((char *, unsigned, char *, unsigned, unsigned *)); + + /* in fileio.c */ +#ifndef UTIL + char *getnam OF((FILE *)); + struct flist far *fexpel OF((struct flist far *)); + char *last OF((char *, int)); +# ifdef UNICODE_SUPPORT + wchar_t *lastw OF((wchar_t *, wchar_t)); +# endif + char *msname OF((char *)); +# ifdef UNICODE_SUPPORT + wchar_t *msnamew OF((wchar_t *)); +# endif + int check_dup OF((void)); + int filter OF((char *, int)); + int newname OF((char *, int, int)); +# ifdef UNICODE_SUPPORT +# ifdef WIN32 + int newnamew OF((wchar_t *, int, int)); +# endif +# endif + /* used by copy mode */ + int proc_archive_name OF((char *, int)); +#endif /* !UTIL */ +#if (!defined(UTIL) || defined(W32_STATROOT_FIX)) + time_t dos2unixtime OF((ulg)); +#endif +#ifndef UTIL + ulg dostime OF((int, int, int, int, int, int)); + ulg unix2dostime OF((time_t *)); + int issymlnk OF((ulg a)); +# ifdef S_IFLNK +# define rdsymlnk(p,b,n) readlink(p,b,n) +/* extern int readlink OF((char *, char *, int)); */ +# else /* !S_IFLNK */ +# define rdsymlnk(p,b,n) (0) +# endif /* !S_IFLNK */ +#endif /* !UTIL */ + +int destroy OF((char *)); +int replace OF((char *, char *)); +int getfileattr OF((char *)); +int setfileattr OF((char *, int)); +char *tempname OF((char *)); + +/* for splits */ +int close_split OF((ulg, FILE *, char *)); +int ask_for_split_read_path OF((ulg)); +int ask_for_split_write_path OF((ulg)); +char *get_in_split_path OF((char *, ulg)); +char *find_in_split_path OF((char *, ulg)); +char *get_out_split_path OF((char *, ulg)); +int rename_split OF((char *, char *)); +int set_filetype OF((char *)); + +int bfcopy OF((uzoff_t)); + +int fcopy OF((FILE *, FILE *, uzoff_t)); + +#ifdef ZMEM + char *memset OF((char *, int, unsigned int)); + char *memcpy OF((char *, char *, unsigned int)); + int memcmp OF((char *, char *, unsigned int)); +#endif /* ZMEM */ + + /* in system dependent fileio code (.c) */ +#ifndef UTIL +# ifdef PROCNAME + int wild OF((char *)); +# endif + char *in2ex OF((char *)); + char *ex2in OF((char *, int, int *)); +#if defined(UNICODE_SUPPORT) && defined(WIN32) + int has_win32_wide OF((void)); + wchar_t *in2exw OF((wchar_t *)); + wchar_t *ex2inw OF((wchar_t *, int, int *)); + int procnamew OF((wchar_t *, int)); +#endif + int procname OF((char *, int)); + void stamp OF((char *, ulg)); + + ulg filetime OF((char *, ulg *, zoff_t *, iztimes *)); + /* Windows Unicode */ +# ifdef UNICODE_SUPPORT +# ifdef WIN32 + ulg filetimew OF((wchar_t *, ulg *, zoff_t *, iztimes *)); + char *get_win32_utf8path OF((char *)); + wchar_t *local_to_wchar_string OF ((char *)); +# endif +# endif + +# if !(defined(VMS) && defined(VMS_PK_EXTRA)) + int set_extra_field OF((struct zlist far *, iztimes *)); +# endif /* ?(VMS && VMS_PK_EXTRA) */ + int deletedir OF((char *)); +# ifdef MY_ZCALLOC + zvoid far *zcalloc OF((unsigned int, unsigned int)); + zvoid zcfree OF((zvoid far *)); +# endif /* MY_ZCALLOC */ +#endif /* !UTIL */ +void version_local OF((void)); + + /* in util.c */ +#ifndef UTIL +int fseekable OF((FILE *)); +char *isshexp OF((char *)); +#ifdef UNICODE_SUPPORT +# ifdef WIN32 + wchar_t *isshexpw OF((wchar_t *)); + int dosmatchw OF((ZCONST wchar_t *, ZCONST wchar_t *, int)); +# endif +#endif +int shmatch OF((ZCONST char *, ZCONST char *, int)); +# if defined(DOS) || defined(WIN32) + int dosmatch OF((ZCONST char *, ZCONST char *, int)); +# endif /* DOS || WIN32 */ +#endif /* !UTIL */ + +/* functions to convert zoff_t to a string */ +char *zip_fuzofft OF((uzoff_t, char *, char*)); +char *zip_fzofft OF((zoff_t, char *, char*)); + +/* read and write number strings like 10M */ +int DisplayNumString OF ((FILE *file, uzoff_t i)); +int WriteNumString OF((uzoff_t num, char *outstring)); +uzoff_t ReadNumString OF((char *numstring)); + +/* returns true if abbrev is abbreviation for string */ +int abbrevmatch OF((char *, char *, int, int)); + +void init_upper OF((void)); +int namecmp OF((ZCONST char *string1, ZCONST char *string2)); + +#ifdef EBCDIC + char *strtoasc OF((char *str1, ZCONST char *str2)); + char *strtoebc OF((char *str1, ZCONST char *str2)); + char *memtoasc OF((char *mem1, ZCONST char *mem2, unsigned len)); + char *memtoebc OF((char *mem1, ZCONST char *mem2, unsigned len)); +#endif /* EBCDIC */ +#ifdef IZ_ISO2OEM_ARRAY + char *str_iso_to_oem OF((char *dst, ZCONST char *src)); +#endif +#ifdef IZ_OEM2ISO_ARRAY + char *str_oem_to_iso OF((char *dst, ZCONST char *src)); +#endif + +zvoid far **search OF((ZCONST zvoid *, ZCONST zvoid far **, extent, + int (*)(ZCONST zvoid *, ZCONST zvoid far *))); +void envargs OF((int *, char ***, char *, char *)); +void expand_args OF((int *, char ***)); + +int is_text_buf OF((ZCONST char *buf_ptr, unsigned buf_size)); +/* this is no longer used ... +unsigned int adler16 OF((unsigned int, ZCONST uch *, extent)); +*/ + /* crc functions are now declared in crc32.h */ + +#ifndef UTIL +#ifndef USE_ZLIB + /* in deflate.c */ +void lm_init OF((int, ush *)); +void lm_free OF((void)); + +uzoff_t deflate OF((void)); + + /* in trees.c */ +void ct_init OF((ush *, int *)); +int ct_tally OF((int, int)); +uzoff_t flush_block OF((char far *, ulg, int)); +void bi_init OF((char *, unsigned int, int)); +#endif /* !USE_ZLIB */ +#endif /* !UTIL */ + + /* in system specific assembler code, replacing C code in trees.c */ +#if defined(ASMV) && defined(RISCOS) + void send_bits OF((int, int)); + unsigned bi_reverse OF((unsigned int, int)); +#endif /* ASMV && RISCOS */ + +/*--------------------------------------------------------------------------- + VMS-only functions: + ---------------------------------------------------------------------------*/ +#ifdef VMS + int vms_stat OF((char *, stat_t *)); /* vms.c */ + void vms_exit OF((int)); /* vms.c */ +#ifndef UTIL +#ifdef VMSCLI + ulg vms_zip_cmdline OF((int *, char ***)); /* cmdline.c */ + void VMSCLI_help OF((void)); /* cmdline.c */ +#endif /* VMSCLI */ +#endif /* !UTIL */ +#endif /* VMS */ + +/* +#ifdef ZIP64_SUPPORT + update_local_Zip64_extra_field OF((struct zlist far *, FILE *)); +#endif +*/ + +/*--------------------------------------------------------------------------- + WIN32-only functions: + ---------------------------------------------------------------------------*/ +#ifdef WIN32 + int ZipIsWinNT OF((void)); /* win32.c */ + int ClearArchiveBit OF((char *)); /* win32.c */ +# ifdef UNICODE_SUPPORT + int ClearArchiveBitW OF((wchar_t *)); /* win32.c */ +# endif +#endif /* WIN32 */ + +#if (defined(WINDLL) || defined(DLL_ZIPAPI)) +/*--------------------------------------------------------------------------- + Prototypes for public Zip API (DLL) functions. + ---------------------------------------------------------------------------*/ +#include "third_party/infozip/zip/api.h" +#endif /* WINDLL || DLL_ZIPAPI */ + + + /* WIN32_OEM */ +#ifdef WIN32 +/* +# if defined(UNICODE_SUPPORT) || defined(WIN32_OEM) +*/ + /* convert oem to ansi string */ + char *oem_to_local_string OF((char *, char *)); +/* +# endif +*/ +#endif + +#ifdef WIN32 +/* +# if defined(UNICODE_SUPPORT) || defined(WIN32_OEM) +*/ + /* convert local string to oem string */ + char *local_to_oem_string OF((char *, char *)); +/* +# endif +*/ +#endif + + + +/*--------------------------------------------------------------------- + Unicode Support + 28 August 2005 + ---------------------------------------------------------------------*/ +#ifdef UNICODE_SUPPORT + + /* Default character when a zwchar too big for wchar_t */ +# define zwchar_to_wchar_t_default_char '_' + + /* Default character string when wchar_t does not convert to mb */ +# define wide_to_mb_default_string "_" + + /* wide character type */ + typedef unsigned long zwchar; + + /* check if string is all ASCII */ + int is_ascii_string OF((char *)); +#ifdef WIN32 + int is_ascii_stringw OF((wchar_t *)); + zwchar *wchar_to_wide_string OF((wchar_t *)); +#endif + + /* convert UTF-8 string to multi-byte string */ + char *utf8_to_local_string OF((char *)); + char *utf8_to_escape_string OF((char *)); + + /* convert UTF-8 string to wide string */ + zwchar *utf8_to_wide_string OF((char *)); + + /* convert wide string to multi-byte string */ + char *wide_to_local_string OF((zwchar *)); + char *wide_to_escape_string OF((zwchar *)); + char *local_to_escape_string OF((char *)); +#ifdef WIN32 + /* convert UTF-8 to wchar */ + wchar_t *utf8_to_wchar_string OF ((char *)); + + char *wchar_to_local_string OF((wchar_t *)); +#endif + + /* convert local string to multi-byte display string */ + char *local_to_display_string OF((char *)); + + /* convert wide character to escape string */ + char *wide_char_to_escape_string OF((unsigned long)); + +#if 0 + /* convert escape string to wide character */ + unsigned long escape_string_to_wide OF((char *)); +#endif + + /* convert local to UTF-8 */ + char *local_to_utf8_string OF ((char *)); + + /* convert local to wide string */ + zwchar *local_to_wide_string OF ((char *)); + + /* convert wide string to UTF-8 */ + char *wide_to_utf8_string OF((zwchar *)); +#ifdef WIN32 + char *wchar_to_utf8_string OF((wchar_t *)); +#endif + +#endif /* UNICODE_SUPPORT */ + + +/*--------------------------------------------------- + * Split archives + * + * 10/20/05 EG + */ + +#define BFWRITE_DATA 0 +#define BFWRITE_LOCALHEADER 1 +#define BFWRITE_CENTRALHEADER 2 +#define BFWRITE_HEADER 3 /* data descriptor or end records */ + +size_t bfwrite OF((ZCONST void *buffer, size_t size, size_t count, + int)); + +/* for putlocal() */ +#define PUTLOCAL_WRITE 0 +#define PUTLOCAL_REWRITE 1 + + +/*-------------------------------------------------------------------- + Long option support + 23 August 2003 + See fileio.c + --------------------------------------------------------------------*/ + +/* The below is for use in the caller-provided options table */ + +/* value_type - value is always returned as a string. */ +#define o_NO_VALUE 0 /* this option does not take a value */ +#define o_REQUIRED_VALUE 1 /* this option requires a value */ +#define o_OPTIONAL_VALUE 2 /* value is optional (see get_option() for details) */ +#define o_VALUE_LIST 3 /* this option takes a list of values */ +#define o_ONE_CHAR_VALUE 4 /* next char is value (does not end short opt string) */ +#define o_NUMBER_VALUE 5 /* value is integer (does not end short opt string) */ + + +/* negatable - a dash following the option (but before any value) sets negated. */ +#define o_NOT_NEGATABLE 0 /* trailing '-' to negate either starts value or generates error */ +#define o_NEGATABLE 1 /* trailing '-' sets negated to TRUE */ + + +/* option_num can be this when option not in options table */ +#define o_NO_OPTION_MATCH -1 + +/* special values returned by get_option - do not use these as option IDs */ +#define o_NON_OPTION_ARG ((unsigned long) 0xFFFF) /* returned for non-option + args */ +#define o_ARG_FILE_ERR ((unsigned long) 0xFFFE) /* internal recursion + return (user never sees) */ + +/* options array is set in zip.c */ +struct option_struct { + char *shortopt; /* char * to sequence of char that is short option */ + char Far *longopt; /* char * to long option string */ + int value_type; /* from above */ + int negatable; /* from above */ + unsigned long option_ID; /* value returned by get_option when this option is found */ + char Far *name; /* optional string for option returned on some errors */ +}; +extern struct option_struct far options[]; + + +/* moved here from fileio.c to make global - 10/6/05 EG */ + +/* If will support wide for Unicode then need to add */ + /* multi-byte */ +#ifdef _MBCS +# ifndef MULTIBYTE_GETOPTNS +# define MULTIBYTE_GETOPTNS +# endif +#endif +#ifdef MULTIBYTE_GETOPTNS + int mb_clen OF((ZCONST char *)); +# define MB_CLEN(ptr) mb_clen(ptr) +# define MB_NEXTCHAR(ptr) ((ptr) += MB_CLEN(ptr)) +#else + /* no multi-byte */ +# define MB_CLEN(ptr) (1) +# define MB_NEXTCHAR(ptr) ((ptr)++) +#endif + + +/* function prototypes */ + +/* get the next option from args */ +unsigned long get_option OF((char ***pargs, int *argc, int *argnum, int *optchar, + char **value, int *negated, int *first_nonopt_arg, + int *option_num, int recursion_depth)); + +/* copy args - copy an args array, allocating space as needed */ +char **copy_args OF((char **args, int max_args)); + +/* free args - free args created with one of these functions */ +int free_args OF ((char **args)); + +/* insert arg - copy an arg into args */ +int insert_arg OF ((char ***args, ZCONST char *arg, int insert_at, + int free_args)); + + +/*-------------------------------------------------------------------- + End of Long option support + --------------------------------------------------------------------*/ + + +#endif /* !__zip_h */ +/* end of zip.h */ diff --git a/third_party/infozip/zip/zipcloak.c b/third_party/infozip/zip/zipcloak.c new file mode 100644 index 000000000..20e444b9a --- /dev/null +++ b/third_party/infozip/zip/zipcloak.c @@ -0,0 +1,774 @@ +/* clang-format off */ +/* + zipcloak.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + This code was originally written in Europe and could be freely distributed + from any country except the U.S.A. If this code was imported into the U.S.A, + it could not be re-exported from the U.S.A to another country. (This + restriction might seem curious but this is what US law required.) + + Now this code can be freely exported and imported. See README.CR. + */ +#define __ZIPCLOAK_C + +#ifndef UTIL +# define UTIL +#endif +#include "third_party/infozip/zip/zip.h" +#define DEFCPYRT /* main module: enable copyright string defines! */ +#include "third_party/infozip/zip/revision.h" +#include "third_party/infozip/zip/crc32.h" +#include "third_party/infozip/zip/crypt.h" +#include "third_party/infozip/zip/ttyio.h" +#include "libc/calls/calls.h" +#include "libc/log/log.h" +#include "libc/stdio/temp.h" +#ifndef NO_STDLIB_H +# include "libc/mem/mem.h" +#endif + +#if CRYPT /* defined (as TRUE or FALSE) in crypt.h */ + +int main OF((int argc, char **argv)); + +local void handler OF((int sig)); +local void license OF((void)); +local void help OF((void)); +local void version_info OF((void)); + +/* Temporary zip file pointer */ +local FILE *tempzf; + +/* Pointer to CRC-32 table (used for decryption/encryption) */ +#if (!defined(USE_ZLIB) || defined(USE_OWN_CRCTAB)) +ZCONST ulg near *crc_32_tab; +#else +ZCONST uLongf *crc_32_tab; +#endif + +int set_filetype(out_path) + char *out_path; +{ +#ifdef __BEOS__ + /* Set the filetype of the zipfile to "application/zip" */ + setfiletype( out_path, "application/zip" ); +#endif + +#ifdef __ATHEOS__ + /* Set the filetype of the zipfile to "application/x-zip" */ + setfiletype(out_path, "application/x-zip"); +#endif + +#ifdef MACOS + /* Set the Creator/Type of the zipfile to 'IZip' and 'ZIP ' */ + setfiletype(out_path, 'IZip', 'ZIP '); +#endif + +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(out_path, 0xDDC); +#endif + return ZE_OK; +} + +/* rename a split + * A split has a tempfile name until it is closed, then + * here rename it as out_path the final name for the split. + */ +int rename_split(temp_name, out_path) + char *temp_name; + char *out_path; +{ + int r; + /* Replace old zip file with new zip file, leaving only the new one */ + if ((r = replace(out_path, temp_name)) != ZE_OK) + { + zipwarn("new zip file left as: ", temp_name); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing split file"); + } + if (zip_attributes) { + setfileattr(out_path, zip_attributes); + } + return ZE_OK; +} + +void zipmessage_nl(a, nl) +ZCONST char *a; /* message string to output */ +int nl; /* 1 = add nl to end */ +/* If nl false, print a message to mesg without new line. + If nl true, print and add new line. */ +{ + if (noisy) { + fprintf(mesg, "%s", a); + if (nl) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } else { + mesg_line_started = 1; + } + fflush(mesg); + } +} + +void zipmessage(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a message to mesg and flush. Write new line first + if current line has output already. */ +{ + if (noisy) { + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "%s%s\n", a, b); + mesg_line_started = 0; + fflush(mesg); + } +} + +/*********************************************************************** + * Issue a message for the error, clean up files and memory, and exit. + */ +void ziperr(code, msg) + int code; /* error code from the ZE_ class */ + ZCONST char *msg; /* message about how it happened */ +{ + if (PERR(code)) perror("zipcloak error"); + fprintf(mesg, "zipcloak error: %s (%s)\n", ZIPERRORS(code), msg); + if (tempzf != NULL) fclose(tempzf); + if (tempzip != NULL) { + destroy(tempzip); + free((zvoid *)tempzip); + } + if (zipfile != NULL) free((zvoid *)zipfile); + EXIT(code); +} + +/*********************************************************************** + * Print a warning message to mesg (usually stderr) and return. + */ +void zipwarn(msg1, msg2) + ZCONST char *msg1, *msg2; /* message strings juxtaposed in output */ +{ + fprintf(mesg, "zipcloak warning: %s%s\n", msg1, msg2); +} + + +/*********************************************************************** + * Upon getting a user interrupt, turn echo back on for tty and abort + * cleanly using ziperr(). + */ +local void handler(sig) + int sig; /* signal number (ignored) */ +{ +#if (!defined(MSDOS) && !defined(__human68k__) && !defined(RISCOS)) + echon(); + putc('\n', mesg); +#endif + ziperr(ZE_ABORT +sig-sig, "aborting"); + /* dummy usage of sig to avoid compiler warnings */ +} + + +static ZCONST char *public[] = { +"The encryption code of this program is not copyrighted and is", +"put in the public domain. It was originally written in Europe", +"and can be freely distributed in both source and object forms", +"from any country, including the USA under License Exception", +"TSU of the U.S. Export Administration Regulations (section", +"740.13(e)) of 6 June 2002. (Prior to January 2000, re-export", +"from the US was a violation of US law.)" +}; + +/*********************************************************************** + * Print license information to stdout. + */ +local void license() +{ + extent i; /* counter for copyright array */ + + for (i = 0; i < sizeof(swlicense)/sizeof(char *); i++) { + puts(swlicense[i]); + } + putchar('\n'); + printf("Export notice:\n"); + for (i = 0; i < sizeof(public)/sizeof(char *); i++) { + puts(public[i]); + } +} + + +static ZCONST char *help_info[] = { +"", +"ZipCloak %s (%s)", +#ifdef VM_CMS +"Usage: zipcloak [-dq] [-b fm] zipfile", +#else +"Usage: zipcloak [-dq] [-b path] zipfile", +#endif +" the default action is to encrypt all unencrypted entries in the zip file", +"", +" -d --decrypt decrypt encrypted entries (copy if given wrong password)", +#ifdef VM_CMS +" -b --temp-mode use \"fm\" as the filemode for the temporary zip file", +#else +" -b --temp-path use \"path\" for the temporary zip file", +#endif +" -O --output-file write output to new zip file", +" -q --quiet quiet operation, suppress some informational messages", +" -h --help show this help", +" -v --version show version info", +" -L --license show software license" + }; + +/*********************************************************************** + * Print help (along with license info) to stdout. + */ +local void help() +{ + extent i; /* counter for help array */ + + for (i = 0; i < sizeof(help_info)/sizeof(char *); i++) { + printf(help_info[i], VERSION, REVDATE); + putchar('\n'); + } +} + + +local void version_info() +/* Print verbose info about program version and compile time options + to stdout. */ +{ + extent i; /* counter in text arrays */ + + /* Options info array */ + static ZCONST char *comp_opts[] = { +#ifdef DEBUG + "DEBUG", +#endif +#if CRYPT && defined(PASSWD_FROM_STDIN) + "PASSWD_FROM_STDIN", +#endif /* CRYPT && PASSWD_FROM_STDIN */ + NULL + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) + { + printf(copyright[i], "zipcloak"); + putchar('\n'); + } + putchar('\n'); + + for (i = 0; i < sizeof(versinfolines)/sizeof(char *); i++) + { + printf(versinfolines[i], "ZipCloak", VERSION, REVDATE); + putchar('\n'); + } + + version_local(); + + puts("ZipCloak special compilation options:"); + for (i = 0; (int)i < (int)(sizeof(comp_opts)/sizeof(char *) - 1); i++) + { + printf("\t%s\n",comp_opts[i]); + } + printf("\t[encryption, version %d.%d%s of %s]\n", + CR_MAJORVER, CR_MINORVER, CR_BETA_VER, CR_VERSION_DATE); +} + +/* options for zipcloak - 3/5/2004 EG */ +struct option_struct far options[] = { + /* short longopt value_type negatable ID name */ +#ifdef VM_CMS + {"b", "temp-mode", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'b', "temp file mode"}, +#else + {"b", "temp-path", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'b', "path for temp file"}, +#endif + {"d", "decrypt", o_NO_VALUE, o_NOT_NEGATABLE, 'd', "decrypt"}, + {"h", "help", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + {"L", "license", o_NO_VALUE, o_NOT_NEGATABLE, 'L', "license"}, + {"l", "", o_NO_VALUE, o_NOT_NEGATABLE, 'L', "license"}, + {"O", "output-file", o_REQUIRED_VALUE, o_NOT_NEGATABLE, 'O', "output to new archive"}, + {"v", "version", o_NO_VALUE, o_NOT_NEGATABLE, 'v', "version"}, + /* the end of the list */ + {NULL, NULL, o_NO_VALUE, o_NOT_NEGATABLE, 0, NULL} /* end has option_ID = 0 */ + }; + + +/*********************************************************************** + * Encrypt or decrypt all of the entries in a zip file. See the command + * help in help() above. + */ + +int main(argc, argv) + int argc; /* number of tokens in command line */ + char **argv; /* command line tokens */ +{ + int attr; /* attributes of zip file */ + zoff_t start_offset; /* start of central directory */ + int decrypt; /* decryption flag */ + int temp_path; /* 1 if next argument is path for temp files */ + char passwd[IZ_PWLEN+1]; /* password for encryption or decryption */ + char verify[IZ_PWLEN+1]; /* password for encryption or decryption */ +#if 0 + char *q; /* steps through option arguments */ + int r; /* arg counter */ +#endif + int res; /* result code */ + zoff_t length; /* length of central directory */ + FILE *inzip, *outzip; /* input and output zip files */ + struct zlist far *z; /* steps through zfiles linked list */ + /* used by get_option */ + unsigned long option; /* option ID returned by get_option */ + int argcnt = 0; /* current argcnt in args */ + int argnum = 0; /* arg number */ + int optchar = 0; /* option state */ + char *value = NULL; /* non-option arg, option value or NULL */ + int negated = 0; /* 1 = option negated */ + int fna = 0; /* current first non-opt arg */ + int optnum = 0; /* index in table */ + + char **args; /* copy of argv that can be freed */ + +#ifdef THEOS + setlocale(LC_CTYPE, "I"); +#endif + +#ifdef UNICODE_SUPPORT +# ifdef UNIX + /* For Unix, set the locale to UTF-8. Any UTF-8 locale is + OK and they should all be the same. This allows seeing, + writing, and displaying (if the fonts are loaded) all + characters in UTF-8. */ + { + char *loc; + + /* + loc = setlocale(LC_CTYPE, NULL); + printf(" Initial language locale = '%s'\n", loc); + */ + + loc = setlocale(LC_CTYPE, "en_US.UTF-8"); + + /* + printf("langinfo %s\n", nl_langinfo(CODESET)); + */ + + if (loc != NULL) { + /* using UTF-8 character set so can set UTF-8 GPBF bit 11 */ + using_utf8 = 1; + /* + printf(" Locale set to %s\n", loc); + */ + } else { + /* + printf(" Could not set Unicode UTF-8 locale\n"); + */ + } + } +# endif +#endif + + /* If no args, show help */ + if (argc == 1) { + help(); + EXIT(ZE_OK); + } + + /* Informational messages are written to stdout. */ + mesg = stdout; + + init_upper(); /* build case map table */ + + crc_32_tab = get_crc_table(); + /* initialize crc table for crypt */ + + /* Go through args */ + zipfile = tempzip = NULL; + tempzf = NULL; +#ifdef SIGINT + signal(SIGINT, handler); +#endif +#ifdef SIGTERM /* Some don't have SIGTERM */ + signal(SIGTERM, handler); +#endif +#ifdef SIGABRT + signal(SIGABRT, handler); +#endif +#ifdef SIGBREAK + signal(SIGBREAK, handler); +#endif +#ifdef SIGBUS + signal(SIGBUS, handler); +#endif +#ifdef SIGILL + signal(SIGILL, handler); +#endif +#ifdef SIGSEGV + signal(SIGSEGV, handler); +#endif + temp_path = decrypt = 0; +#if 0 + /* old command line */ + for (r = 1; r < argc; r++) { + if (*argv[r] == '-') { + if (!argv[r][1]) ziperr(ZE_PARMS, "zip file cannot be stdin"); + for (q = argv[r]+1; *q; q++) { + switch (*q) { + case 'b': /* Specify path for temporary file */ + if (temp_path) { + ziperr(ZE_PARMS, "use -b before zip file name"); + } + temp_path = 1; /* Next non-option is path */ + break; + case 'd': + decrypt = 1; break; + case 'h': /* Show help */ + help(); + EXIT(ZE_OK); + case 'l': case 'L': /* Show copyright and disclaimer */ + license(); + EXIT(ZE_OK); + case 'q': /* Quiet operation, suppress info messages */ + noisy = 0; break; + case 'v': /* Show version info */ + version_info(); + EXIT(ZE_OK); + default: + ziperr(ZE_PARMS, "unknown option"); + } /* switch */ + } /* for */ + + } else if (temp_path == 0) { + if (zipfile != NULL) { + ziperr(ZE_PARMS, "can only specify one zip file"); + + } else if ((zipfile = ziptyp(argv[r])) == NULL) { + ziperr(ZE_MEM, "was processing arguments"); + } + } else { + tempath = argv[r]; + temp_path = 0; + } /* if */ + } /* for */ + +#else + + /* new command line */ + + zipfile = NULL; + out_path = NULL; + + /* make copy of args that can use with insert_arg() */ + args = copy_args(argv, 0); + + /* + ------------------------------------------- + Process command line using get_option + ------------------------------------------- + + Each call to get_option() returns either a command + line option and possible value or a non-option argument. + Arguments are permuted so that all options (-r, -b temp) + are returned before non-option arguments (zipfile). + Returns 0 when nothing left to read. + */ + + /* set argnum = 0 on first call to init get_option */ + argnum = 0; + + /* get_option returns the option ID and updates parameters: + args - usually same as argv if no argument file support + argcnt - current argc for args + value - char* to value (free() when done with it) or NULL if no value + negated - option was negated with trailing - + */ + + while ((option = get_option(&args, &argcnt, &argnum, + &optchar, &value, &negated, + &fna, &optnum, 0))) + { + switch (option) + { + case 'b': /* Specify path for temporary file */ + if (temp_path) { + ziperr(ZE_PARMS, "more than one temp_path"); + } + temp_path = 1; + tempath = value; + break; + case 'd': + decrypt = 1; break; + case 'h': /* Show help */ + help(); + EXIT(ZE_OK); + case 'l': case 'L': /* Show copyright and disclaimer */ + license(); + EXIT(ZE_OK); + case 'O': /* Output to new zip file instead of updating original zip file */ + if ((out_path = ziptyp(value)) == NULL) { + ziperr(ZE_MEM, "was processing arguments"); + } + free(value); + break; + case 'q': /* Quiet operation, suppress info messages */ + noisy = 0; break; + case 'v': /* Show version info */ + version_info(); + EXIT(ZE_OK); + case o_NON_OPTION_ARG: + /* not an option */ + /* no more options as permuting */ + /* just dash also ends up here */ + + if (strcmp(value, "-") == 0) { + ziperr(ZE_PARMS, "zip file cannot be stdin"); + } else if (zipfile != NULL) { + ziperr(ZE_PARMS, "can only specify one zip file"); + } + + if ((zipfile = ziptyp(value)) == NULL) { + ziperr(ZE_MEM, "was processing arguments"); + } + free(value); + break; + + default: + ziperr(ZE_PARMS, "unknown option"); + } + } + + free_args(args); + +#endif + + if (zipfile == NULL) ziperr(ZE_PARMS, "need to specify zip file"); + + /* in_path is the input zip file */ + if ((in_path = malloc(strlen(zipfile) + 1)) == NULL) { + ziperr(ZE_MEM, "input"); + } + strcpy(in_path, zipfile); + + /* out_path defaults to in_path */ + if (out_path == NULL) { + if ((out_path = malloc(strlen(zipfile) + 1)) == NULL) { + ziperr(ZE_MEM, "output"); + } + strcpy(out_path, zipfile); + } + + /* Read zip file */ + if ((res = readzipfile()) != ZE_OK) ziperr(res, zipfile); + if (zfiles == NULL) ziperr(ZE_NAME, zipfile); + + /* Check for something to do */ + for (z = zfiles; z != NULL; z = z->nxt) { + if (decrypt ? z->flg & 1 : !(z->flg & 1)) break; + } + if (z == NULL) { + ziperr(ZE_NONE, decrypt ? "no encrypted files" + : "all files encrypted already"); + } + + /* Before we get carried away, make sure zip file is writeable */ + if ((inzip = fopen(zipfile, "a")) == NULL) ziperr(ZE_CREAT, zipfile); + fclose(inzip); + attr = getfileattr(zipfile); + + /* Open output zip file for writing */ +#if defined(UNIX) && !defined(NO_MKSTEMP) + { + int yd; + int i; + + /* use mkstemp to avoid race condition and compiler warning */ + + if (tempath != NULL) + { + /* if -b used to set temp file dir use that for split temp */ + if ((tempzip = malloc(strlen(tempath) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, tempath); + if (lastchar(tempzip) != '/') + strcat(tempzip, "/"); + } + else + { + /* create path by stripping name and appending template */ + if ((tempzip = malloc(strlen(zipfile) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, zipfile); + for(i = strlen(tempzip); i > 0; i--) { + if (tempzip[i - 1] == '/') + break; + } + tempzip[i] = '\0'; + } + strcat(tempzip, "ziXXXXXX"); + + if ((yd = mkstemp(tempzip)) == EOF) { + ZIPERR(ZE_TEMP, tempzip); + } + if ((y = tempzf = outzip = fdopen(yd, FOPW_TMP)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } + } +#else + if ((y = tempzf = outzip = fopen(tempzip = tempname(zipfile), FOPW)) == NULL) { + ziperr(ZE_TEMP, tempzip); + } +#endif + + /* Get password */ + if (getp("Enter password: ", passwd, IZ_PWLEN+1) == NULL) + ziperr(ZE_PARMS, + "stderr is not a tty (you may never see this message!)"); + + if (decrypt == 0) { + if (getp("Verify password: ", verify, IZ_PWLEN+1) == NULL) + ziperr(ZE_PARMS, + "stderr is not a tty (you may never see this message!)"); + + if (strcmp(passwd, verify)) + ziperr(ZE_PARMS, "password verification failed"); + + if (*passwd == '\0') + ziperr(ZE_PARMS, "zero length password not allowed"); + } + + /* Open input zip file again, copy preamble if any */ + if ((in_file = fopen(zipfile, FOPR)) == NULL) ziperr(ZE_NAME, zipfile); + + if (zipbeg && (res = bfcopy(zipbeg)) != ZE_OK) + { + ziperr(res, res == ZE_TEMP ? tempzip : zipfile); + } + tempzn = zipbeg; + + /* Go through local entries, copying, encrypting, or decrypting */ + for (z = zfiles; z != NULL; z = z->nxt) { + if (decrypt && (z->flg & 1)) { + printf("decrypting: %s", z->zname); + fflush(stdout); + if ((res = zipbare(z, passwd)) != ZE_OK) + { + if (res != ZE_MISS) ziperr(res, "was decrypting an entry"); + printf(" (wrong password--just copying)"); + fflush(stdout); + } + putchar('\n'); + + } else if ((!decrypt) && !(z->flg & 1)) { + printf("encrypting: %s\n", z->zname); + fflush(stdout); + if ((res = zipcloak(z, passwd)) != ZE_OK) + { + ziperr(res, "was encrypting an entry"); + } + } else { + printf(" copying: %s\n", z->zname); + fflush(stdout); + if ((res = zipcopy(z)) != ZE_OK) + { + ziperr(res, "was copying an entry"); + } + } /* if */ + } /* for */ + + fclose(in_file); + + + /* Write central directory and end of central directory */ + + /* get start of central */ + if ((start_offset = zftello(outzip)) == (zoff_t)-1) + ziperr(ZE_TEMP, tempzip); + + for (z = zfiles; z != NULL; z = z->nxt) { + if ((res = putcentral(z)) != ZE_OK) ziperr(res, tempzip); + } + + /* get end of central */ + if ((length = zftello(outzip)) == (zoff_t)-1) + ziperr(ZE_TEMP, tempzip); + + length -= start_offset; /* compute length of central */ + if ((res = putend((zoff_t)zcount, length, start_offset, zcomlen, + zcomment)) != ZE_OK) { + ziperr(res, tempzip); + } + tempzf = NULL; + if (fclose(outzip)) ziperr(ZE_TEMP, tempzip); + if ((res = replace(out_path, tempzip)) != ZE_OK) { + zipwarn("new zip file left as: ", tempzip); + free((zvoid *)tempzip); + tempzip = NULL; + ziperr(res, "was replacing the original zip file"); + } + free((zvoid *)tempzip); + tempzip = NULL; + setfileattr(zipfile, attr); +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(zipfile, 0xDDC); +#endif + free((zvoid *)in_path); + free((zvoid *)out_path); + + free((zvoid *)zipfile); + zipfile = NULL; + + /* Done! */ + RETURN(0); +} +#else /* !CRYPT */ + + +/* below is only used if crypt is not enabled */ + +struct option_struct far options[] = { + /* short longopt value_type negatable ID name */ + {"h", "help", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + /* the end of the list */ + {NULL, NULL, o_NO_VALUE, o_NOT_NEGATABLE, 0, NULL} /* end has option_ID = 0 */ + }; + + +int main OF((void)); + +void zipwarn(msg1, msg2) +ZCONST char *msg1, *msg2; +{ + /* Tell picky compilers to shut up about unused variables */ + msg1 = msg1; msg2 = msg2; +} + +void ziperr(c, h) +int c; +ZCONST char *h; +{ + /* Tell picky compilers to shut up about unused variables */ + c = c; h = h; +} + +int main() +{ + fprintf(mesg, "\ +This version of ZipCloak does not support encryption. Get the current Zip\n\ +source distribution and recompile ZipCloak after you have added an option to\n\ +define the symbol USE_CRYPT to the C compiler's command arguments.\n"); + RETURN(1); +} + +#endif /* ?CRYPT */ diff --git a/third_party/infozip/zip/ziperr.h b/third_party/infozip/zip/ziperr.h new file mode 100644 index 000000000..0c25c3ddc --- /dev/null +++ b/third_party/infozip/zip/ziperr.h @@ -0,0 +1,116 @@ +/* clang-format off */ +/* + ziperr.h - Zip 3 + + Copyright (c) 1990-2007 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * ziperr.h by Mark Adler + */ + + +/* + * VMS message file ident string. (The "-xxx" suffix should be + * incremented when messages are changed for a particular program + * version.) Used only when generating the VMS message source file, but + * that can be done on a non-VMS system. + */ +#define VMS_MSG_IDENT "V3.0-000" + +/* VMS-compatible "severity" values (bits 2:0): */ +#define ZE_S_WARNING 0x00 +#define ZE_S_SUCCESS 0x01 +#define ZE_S_ERROR 0x02 +#define ZE_S_INFO 0x03 +#define ZE_S_SEVERE 0x04 +#define ZE_S_UNUSED 0x07 + +/* Flags: */ +#define ZE_S_PERR 0x10 + + + /* Error return values. The values 0..4 and 12..18 follow the conventions + of PKZIP. The values 4..10 are all assigned to "insufficient memory" + by PKZIP, so the codes 5..10 are used here for other purposes. */ +#define ZE_MISS -1 /* used by procname(), zipbare() */ +#define ZE_OK 0 /* success */ +#define ZE_EOF 2 /* unexpected end of zip file */ +#define ZE_FORM 3 /* zip file structure error */ +#define ZE_MEM 4 /* out of memory */ +#define ZE_LOGIC 5 /* internal logic error */ +#define ZE_BIG 6 /* entry too large to split, read, or write */ +#define ZE_NOTE 7 /* invalid comment format */ +#define ZE_TEST 8 /* zip test (-T) failed or out of memory */ +#define ZE_ABORT 9 /* user interrupt or termination */ +#define ZE_TEMP 10 /* error using a temp file */ +#define ZE_READ 11 /* read or seek error */ +#define ZE_NONE 12 /* nothing to do */ +#define ZE_NAME 13 /* missing or empty zip file */ +#define ZE_WRITE 14 /* error writing to a file */ +#define ZE_CREAT 15 /* couldn't open to write */ +#define ZE_PARMS 16 /* bad command line */ +#define ZE_OPEN 18 /* could not open a specified file to read */ +#define ZE_COMPERR 19 /* error in compilation options */ +#define ZE_ZIP64 20 /* Zip64 not supported */ + +#define ZE_MAXERR 20 /* the highest error number */ + + +/* Error messages for the ziperr() function in the zip programs. */ + +#ifdef GLOBALS +struct +{ + char *name; + char *string; + int severity; +} ziperrors[ZE_MAXERR + 1] = { +/* 0 */ { "OK", "Normal successful completion", ZE_S_SUCCESS }, +/* 1 */ { "", "", ZE_S_UNUSED }, +/* 2 */ { "EOF", "Unexpected end of zip file", ZE_S_SEVERE }, +/* 3 */ { "FORM", "Zip file structure invalid", ZE_S_ERROR }, +/* 4 */ { "MEM", "Out of memory", ZE_S_SEVERE }, +/* 5 */ { "LOGIC", "Internal logic error", ZE_S_SEVERE }, +/* 6 */ { "BIG", "Entry too big to split, read, or write", + ZE_S_ERROR }, +/* 7 */ { "NOTE", "Invalid comment format", ZE_S_ERROR }, +/* 8 */ { "TEST", "Zip file invalid, could not spawn unzip, or wrong unzip", + ZE_S_SEVERE }, +/* 9 */ { "ABORT", "Interrupted", ZE_S_ERROR }, +/* 10 */ { "TEMP", "Temporary file failure", ZE_S_SEVERE | ZE_S_PERR }, +/* 11 */ { "READ", "Input file read failure", ZE_S_SEVERE | ZE_S_PERR }, +/* 12 */ { "NONE", "Nothing to do!", ZE_S_WARNING }, +/* 13 */ { "NAME", "Missing or empty zip file", ZE_S_ERROR }, +/* 14 */ { "WRITE", "Output file write failure", ZE_S_SEVERE | ZE_S_PERR }, +/* 15 */ { "CREAT", "Could not create output file", ZE_S_SEVERE | ZE_S_PERR }, +/* 16 */ { "PARMS", "Invalid command arguments", ZE_S_ERROR }, +/* 17 */ { "", "", ZE_S_UNUSED }, +/* 18 */ { "OPEN", "File not found or no read permission", + ZE_S_ERROR | ZE_S_PERR }, +/* 19 */ { "COMPERR", "Not supported", ZE_S_SEVERE }, +/* 20 */ { "ZIP64", "Attempt to read unsupported Zip64 archive", + ZE_S_SEVERE } +# ifdef AZTEC_C + , /* extremely lame compiler bug workaround */ +# endif +}; +#else /* !GLOBALS */ +/* Error messages for ziperr() */ +extern struct +{ + char *name; + char *string; + int severity; +} ziperrors[ZE_MAXERR + 1]; +#endif /* ?GLOBALS */ + +/* Macro to determine whether to call perror() or not. */ +#define PERR(e) (ziperrors[e].severity & ZE_S_PERR) + +/* Macro for easy access to the message string. */ +#define ZIPERRORS(e) ziperrors[e].string diff --git a/third_party/infozip/zip/zipfile.c b/third_party/infozip/zip/zipfile.c new file mode 100644 index 000000000..b4b22c825 --- /dev/null +++ b/third_party/infozip/zip/zipfile.c @@ -0,0 +1,6824 @@ +/* clang-format off */ +/* + zipfile.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * zipfile.c by Mark Adler. + */ +#define __ZIPFILE_C + +#include "third_party/infozip/zip/zip.h" +#include "third_party/infozip/zip/revision.h" +#ifdef UNICODE_SUPPORT +# include "third_party/infozip/zip/crc32.h" +#endif + +/* for realloc 2/6/2005 EG */ +#include "libc/mem/mem.h" +#include "libc/alg/alg.h" +#include "libc/errno.h" + +/* for toupper() */ +#include "libc/str/str.h" +#include "libc/fmt/fmt.h" + +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" + +#ifdef VMS +# include "vms/vms.h" +# include "vms/vmsmunch.h" +# include "vms/vmsdefs.h" +#endif + +#ifdef WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + +/* + * XXX start of zipfile.h + */ +#ifdef THEOS + /* Macros cause stack overflow in compiler */ + ush SH(uch* p) { return ((ush)(uch)((p)[0]) | ((ush)(uch)((p)[1]) << 8)); } + ulg LG(uch* p) { return ((ulg)(SH(p)) | ((ulg)(SH((p)+2)) << 16)); } +#else /* !THEOS */ + /* Macros for converting integers in little-endian to machine format */ +# define SH(a) ((ush)(((ush)(uch)(a)[0]) | (((ush)(uch)(a)[1]) << 8))) +# define LG(a) ((ulg)SH(a) | ((ulg)SH((a)+2) << 16)) +# ifdef ZIP64_SUPPORT /* zip64 support 08/31/2003 R.Nausedat */ +# define LLG(a) ((zoff_t)LG(a) | ((zoff_t)LG((a)+4) << 32)) +# endif +#endif /* ?THEOS */ + +/* Macros for writing machine integers to little-endian format */ +#define PUTSH(a,f) {putc((char)((a) & 0xff),(f)); putc((char)((a) >> 8),(f));} +#define PUTLG(a,f) {PUTSH((a) & 0xffff,(f)) PUTSH((a) >> 16,(f))} + +#ifdef ZIP64_SUPPORT /* zip64 support 08/31/2003 R.Nausedat */ +# define PUTLLG(a,f) {PUTLG((a) & 0xffffffff,(f)) PUTLG((a) >> 32,(f))} +#endif + + +/* -- Structure of a ZIP file -- */ + +/* Signatures for zip file information headers */ +#define LOCSIG 0x04034b50L +#define CENSIG 0x02014b50L +#define ENDSIG 0x06054b50L +#define EXTLOCSIG 0x08074b50L + +/* Offsets of values in headers */ +/* local header */ +#define LOCVER 0 /* version needed to extract */ +#define LOCFLG 2 /* encrypt, deflate flags */ +#define LOCHOW 4 /* compression method */ +#define LOCTIM 6 /* last modified file time, DOS format */ +#define LOCDAT 8 /* last modified file date, DOS format */ +#define LOCCRC 10 /* uncompressed crc-32 for file */ +#define LOCSIZ 14 /* compressed size in zip file */ +#define LOCLEN 18 /* uncompressed size */ +#define LOCNAM 22 /* length of filename */ +#define LOCEXT 24 /* length of extra field */ + +/* extended local header (data descriptor) following file data (if bit 3 set) */ +/* if Zip64 then all are 8 byte and not below - 11/1/03 EG */ +#define EXTCRC 0 /* uncompressed crc-32 for file */ +#define EXTSIZ 4 /* compressed size in zip file */ +#define EXTLEN 8 /* uncompressed size */ + +/* central directory header */ +#define CENVEM 0 /* version made by */ +#define CENVER 2 /* version needed to extract */ +#define CENFLG 4 /* encrypt, deflate flags */ +#define CENHOW 6 /* compression method */ +#define CENTIM 8 /* last modified file time, DOS format */ +#define CENDAT 10 /* last modified file date, DOS format */ +#define CENCRC 12 /* uncompressed crc-32 for file */ +#define CENSIZ 16 /* compressed size in zip file */ +#define CENLEN 20 /* uncompressed size */ +#define CENNAM 24 /* length of filename */ +#define CENEXT 26 /* length of extra field */ +#define CENCOM 28 /* file comment length */ +#define CENDSK 30 /* disk number start */ +#define CENATT 32 /* internal file attributes */ +#define CENATX 34 /* external file attributes */ +#define CENOFF 38 /* relative offset of local header */ + +/* end of central directory record */ +#define ENDDSK 0 /* number of this disk */ +#define ENDBEG 2 /* number of the starting disk */ +#define ENDSUB 4 /* entries on this disk */ +#define ENDTOT 6 /* total number of entries */ +#define ENDSIZ 8 /* size of entire central directory */ +#define ENDOFF 12 /* offset of central on starting disk */ +#define ENDCOM 16 /* length of zip file comment */ + +/* zip64 support 08/31/2003 R.Nausedat */ + +/* EOCDL_SIG used to detect Zip64 archive */ +#define ZIP64_EOCDL_SIG 0x07064b50 +/* EOCDL size is used in the empty archive check */ +#define ZIP64_EOCDL_OFS_SIZE 20 + +#define ZIP_UWORD16_MAX 0xFFFF /* border value */ +#define ZIP_UWORD32_MAX 0xFFFFFFFF /* border value */ +#define ZIP_EF_HEADER_SIZE 4 /* size of pre-header of extra fields */ + +#ifdef ZIP64_SUPPORT +# define ZIP64_EXTCRC 0 /* uncompressed crc-32 for file */ +# define ZIP64_EXTSIZ 4 /* compressed size in zip file */ +# define ZIP64_EXTLEN 12 /* uncompressed size */ +# define ZIP64_EOCD_SIG 0x06064b50 +# define ZIP64_EOCD_OFS_SIZE 40 +# define ZIP64_EOCD_OFS_CD_START 48 +# define ZIP64_EOCDL_OFS_SIZE 20 +# define ZIP64_EOCDL_OFS_EOCD_START 8 +# define ZIP64_EOCDL_OFS_TOTALDISKS 16 +# define ZIP64_MIN_VER 45 /* min version to set in the CD extra records */ +# define ZIP64_CENTRAL_DIR_TAIL_SIZE (56 - 8 - 4) /* size of zip64 central dir tail, minus sig and size field bytes */ +# define ZIP64_CENTRAL_DIR_TAIL_SIG 0x06064B50L /* zip64 central dir tail signature */ +# define ZIP64_CENTRAL_DIR_TAIL_END_SIG 0x07064B50L /* zip64 end of cen dir locator signature */ +# define ZIP64_LARGE_FILE_HEAD_SIZE 32 /* total size of zip64 extra field */ +# define ZIP64_EF_TAG 0x0001 /* ID for zip64 extra field */ +# define ZIP64_EFIELD_OFS_OSIZE ZIP_EF_HEADER_SIZE /* zip64 extra field: offset to original file size */ +# define ZIP64_EFIELD_OFS_CSIZE (ZIP64_EFIELD_OFS_OSIZE + 8) /* zip64 extra field: offset to compressed file size */ +# define ZIP64_EFIELD_OFS_OFS (ZIP64_EFIELD_OFS_CSIZE + 8) /* zip64 extra field: offset to offset in archive */ +# define ZIP64_EFIELD_OFS_DISK (ZIP64_EFIELD_OFS_OFS + 8) /* zip64 extra field: offset to start disk # */ +/* -------------------------------------------------------------------------------------------------------------------------- */ + local int adjust_zip_local_entry OF((struct zlist far *)); + local void adjust_zip_central_entry OF((struct zlist far *)); +#if 0 + local int remove_local_extra_field OF((struct zlist far *, ulg)); + local int remove_central_extra_field OF((struct zlist far *, ulg)); +#endif + local int add_central_zip64_extra_field OF((struct zlist far *)); + local int add_local_zip64_extra_field OF((struct zlist far *)); +#endif /* ZIP64_SUPPORT */ +#ifdef UNICODE_SUPPORT +# define UTF8_PATH_EF_TAG 0x7075 /* ID for Unicode path (up) extra field */ + local int add_Unicode_Path_local_extra_field OF((struct zlist far *)); + local int add_Unicode_Path_cen_extra_field OF((struct zlist far *)); +#endif + +/* New General Purpose Bit Flag bit 11 flags when entry path and + comment are in UTF-8 */ +#define UTF8_BIT (1 << 11) + +/* moved out of ZIP64_SUPPORT - 2/6/2005 EG */ +local void write_ushort_to_mem OF((ush, char *)); /* little endian conversions */ +local void write_ulong_to_mem OF((ulg, char *)); +#ifdef ZIP64_SUPPORT + local void write_int64_to_mem OF((uzoff_t, char *)); +#endif /* def ZIP64_SUPPORT */ +#ifdef UNICODE_SUPPORT + local void write_string_to_mem OF((char *, char *)); +#endif +#if 0 +local char *get_extra_field OF((ush, char *, unsigned)); /* zip64 */ +#endif +#ifdef UNICODE_SUPPORT +local void read_Unicode_Path_entry OF((struct zlist far *)); +local void read_Unicode_Path_local_entry OF((struct zlist far *)); +#endif + +/* added these self allocators - 2/6/2005 EG */ +local void append_ushort_to_mem OF((ush, char **, extent *, extent *)); +local void append_ulong_to_mem OF((ulg, char **, extent *, extent *)); +#ifdef ZIP64_SUPPORT + local void append_int64_to_mem OF((uzoff_t, char **, extent *, extent *)); +#endif /* def ZIP64_SUPPORT */ +local void append_string_to_mem OF((char *, int, char**, extent *, extent *)); + + +/* Local functions */ + +local int find_next_signature OF((FILE *f)); +local int find_signature OF((FILE *, ZCONST char *)); +local int is_signature OF((ZCONST char *, ZCONST char *)); +local int at_signature OF((FILE *, ZCONST char *)); + +local int zqcmp OF((ZCONST zvoid *, ZCONST zvoid *)); +#ifdef UNICODE_SUPPORT +local int zuqcmp OF((ZCONST zvoid *, ZCONST zvoid *)); +#endif +#if 0 + local int scanzipf_reg OF((FILE *f)); +#endif +local int scanzipf_regnew OF((void)); +#ifndef UTIL + local int rqcmp OF((ZCONST zvoid *, ZCONST zvoid *)); + local int zbcmp OF((ZCONST zvoid *, ZCONST zvoid far *)); +# ifdef UNICODE_SUPPORT + local int zubcmp OF((ZCONST zvoid *, ZCONST zvoid far *)); +# if 0 + local int zuebcmp OF((ZCONST zvoid *, ZCONST zvoid far *)); +# endif +# endif /* UNICODE_SUPPORT */ + local void zipoddities OF((struct zlist far *)); +# if 0 + local int scanzipf_fix OF((FILE *f)); +# endif + local int scanzipf_fixnew OF((void)); +# ifdef USE_EF_UT_TIME + local int ef_scan_ut_time OF((char *ef_buf, extent ef_len, int ef_is_cent, + iztimes *z_utim)); +# endif /* USE_EF_UT_TIME */ + local void cutpath OF((char *p, int delim)); +#endif /* !UTIL */ + +/* + * XXX end of zipfile.h + */ + +/* Local data */ + +#ifdef HANDLE_AMIGA_SFX + ulg amiga_sfx_offset; /* place where size field needs updating */ +#endif + +local int zqcmp(a, b) +ZCONST zvoid *a, *b; /* pointers to pointers to zip entries */ +/* Used by qsort() to compare entries in the zfile list. + * Compares the internal names z->iname */ +{ + char *aname = (*(struct zlist far **)a)->iname; + char *bname = (*(struct zlist far **)b)->iname; + + return namecmp(aname, bname); +} + +#ifdef UNICODE_SUPPORT +local int zuqcmp(a, b) +ZCONST zvoid *a, *b; /* pointers to pointers to zip entries */ +/* Used by qsort() to compare entries in the zfile list. + * Compares the internal names z->zuname */ +{ + char *aname = (*(struct zlist far **)a)->iname; + char *bname = (*(struct zlist far **)b)->iname; + + /* zuname could be NULL */ + if ((*(struct zlist far **)a)->zuname) + aname = (*(struct zlist far **)a)->zuname; + if ((*(struct zlist far **)b)->zuname) + bname = (*(struct zlist far **)b)->zuname; + return namecmp(aname, bname); +} +#endif + + +#ifndef UTIL + +local int rqcmp(a, b) +ZCONST zvoid *a, *b; /* pointers to pointers to zip entries */ +/* Used by qsort() to compare entries in the zfile list. + * Compare the internal names z->iname, but in reverse order. */ +{ + return namecmp((*(struct zlist far **)b)->iname, + (*(struct zlist far **)a)->iname); +} + + +local int zbcmp(n, z) +ZCONST zvoid *n; /* string to search for */ +ZCONST zvoid far *z; /* pointer to a pointer to a zip entry */ +/* Used by search() to compare a target to an entry in the zfile list. */ +{ + return namecmp((char *)n, ((struct zlist far *)z)->zname); +} + +#ifdef UNICODE_SUPPORT +/* search unicode paths */ +local int zubcmp(n, z) +ZCONST zvoid *n; /* string to search for */ +ZCONST zvoid far *z; /* pointer to a pointer to a zip entry */ +/* Used by search() to compare a target to an entry in the zfile list. */ +{ + char *zuname = ((struct zlist far *)z)->zuname; + + /* zuname is NULL if no UTF-8 name */ + if (zuname == NULL) + zuname = ((struct zlist far *)z)->zname; + + return namecmp((char *)n, zuname); +} + +#if 0 +/* search escaped unicode paths */ +local int zuebcmp(n, z) +ZCONST zvoid *n; /* string to search for */ +ZCONST zvoid far *z; /* pointer to a pointer to a zip entry */ +/* Used by search() to compare a target to an entry in the zfile list. */ +{ + char *zuname = ((struct zlist far *)z)->zuname; + char *zuename; + int k; + + /* zuname is NULL if no UTF-8 name */ + if (zuname == NULL) + zuname = ((struct zlist far *)z)->zname; + zuename = local_to_escape_string(zuname); + k = namecmp((char *)n, zuename); + free(zuename); + + return k; +} +#endif +#endif + + +struct zlist far *zsearch(n) + ZCONST char *n; /* name to find */ +/* Return a pointer to the entry in zfile with the name n, or NULL if + not found. */ +{ + zvoid far **p; /* result of search() */ + + if (zcount) { + if ((p = search(n, (ZCONST zvoid far **)zsort, zcount, zbcmp)) != NULL) + return *(struct zlist far **)p; +#ifdef UNICODE_SUPPORT + else if (unicode_mismatch != 3 && fix != 2 && + (p = search(n, (ZCONST zvoid far **)zusort, zcount, zubcmp)) != NULL) + return *(struct zlist far **)p; +#endif + else + return NULL; + } + return NULL; +} + +#endif /* !UTIL */ + +#ifndef VMS /* See [.VMS]VMS.C for VMS-specific ziptyp(). */ +# ifndef PATHCUT +# define PATHCUT '/' +# endif + +char *ziptyp(s) + char *s; /* file name to force to zip */ +/* If the file name *s has a dot (other than the first char), or if + the -A option is used (adjust self-extracting file) then return + the name, otherwise append .zip to the name. Allocate the space for + the name in either case. Return a pointer to the new name, or NULL + if malloc() fails. */ +{ + char *q; /* temporary pointer */ + char *t; /* pointer to malloc'ed string */ +# ifdef THEOS + char *r; /* temporary pointer */ + char *disk; +# endif + + if ((t = malloc(strlen(s) + 5)) == NULL) + return NULL; + strcpy(t, s); +# ifdef __human68k__ + _toslash(t); +# endif +# ifdef MSDOS + for (q = t; *q; INCSTR(q)) + if (*q == '\\') + *q = '/'; +# endif /* MSDOS */ +# if defined(__RSXNT__) || defined(WIN32_CRT_OEM) + /* RSXNT/EMX C rtl uses OEM charset */ + AnsiToOem(t, t); +# endif + if (adjust) return t; +# ifndef RISCOS +# ifndef QDOS +# ifdef AMIGA + if ((q = MBSRCHR(t, '/')) == NULL) + q = MBSRCHR(t, ':'); + if (MBSRCHR((q ? q + 1 : t), '.') == NULL) +# else /* !AMIGA */ +# ifdef THEOS + /* the argument expansion add a dot to the end of file names when + * there is no extension and at least one of a argument has wild cards. + * So check for at least one character in the extension if there is a dot + * in file name */ + if ((q = MBSRCHR((q = MBSRCHR(t, PATHCUT)) == NULL ? t : q + 1, '.')) == NULL + || q[1] == '\0') { +# else /* !THEOS */ +# ifdef TANDEM + if (MBSRCHR((q = MBSRCHR(t, '.')) == NULL ? t : q + 1, ' ') == NULL) +# else /* !TANDEM */ + if (MBSRCHR((q = MBSRCHR(t, PATHCUT)) == NULL ? t : q + 1, '.') == NULL) +# endif /* ?TANDEM */ +# endif /* ?THEOS */ +# endif /* ?AMIGA */ +# ifdef CMS_MVS + if (strncmp(t,"dd:",3) != 0 && strncmp(t,"DD:",3) != 0) +# endif /* CMS_MVS */ +# ifdef THEOS + /* insert .zip extension before disk name */ + if ((r = MBSRCHR(t, ':')) != NULL) { + /* save disk name */ + if ((disk = strdup(r)) == NULL) + return NULL; + strcpy(r[-1] == '.' ? r - 1 : r, ".zip"); + strcat(t, disk); + free(disk); + } else { + if (q != NULL && *q == '.') + strcpy(q, ".zip"); + else + strcat(t, ".zip"); + } + } +# else /* !THEOS */ +# ifdef TANDEM /* Tandem can't cope with extensions */ + strcat(t, " ZIP"); +# else /* !TANDEM */ + strcat(t, ".zip"); +# endif /* ?TANDEM */ +# endif /* ?THEOS */ +# else /* QDOS */ + q = LastDir(t); + if(MBSRCHR(q, '_') == NULL && MBSRCHR(q, '.') == NULL) + { + strcat(t, "_zip"); + } +# endif /* QDOS */ +# endif /* !RISCOS */ + return t; +} +#endif /* ndef VMS */ + +/* ---------------------------------------------------- */ + +/* moved out of ZIP64_SUPPORT - 2/6/2005 EG */ + +/* 08/31/2003 R.Nausedat */ + +local void write_ushort_to_mem( OFT( ush) usValue, + OFT( char *)pPtr) +#ifdef NO_PROTO + ush usValue; + char *pPtr; +#endif /* def NO_PROTO */ +{ + *pPtr++ = ((char)(usValue) & 0xff); + *pPtr = ((char)(usValue >> 8) & 0xff); +} + +local void write_ulong_to_mem(uValue, pPtr) +ulg uValue; +char *pPtr; +{ + write_ushort_to_mem((ush)(uValue & 0xffff), pPtr); + write_ushort_to_mem((ush)((uValue >> 16) & 0xffff), pPtr + 2); +} + +#ifdef ZIP64_SUPPORT + +local void write_int64_to_mem(l64Value,pPtr) + uzoff_t l64Value; + char *pPtr; +{ + write_ulong_to_mem((ulg)(l64Value & 0xffffffff),pPtr); + write_ulong_to_mem((ulg)((l64Value >> 32) & 0xffffffff),pPtr + 4); +} + +#endif /* def ZIP64_SUPPORT */ + +#ifdef UNICODE_SUPPORT + +/* Write a string to memory */ +local void write_string_to_mem(strValue, pPtr) + char *strValue; + char *pPtr; +{ + if (strValue != NULL) { + int ssize = strlen(strValue); + int i; + + for (i = 0; i < ssize; i++) { + *(pPtr + i) = *(strValue + i); + } + } +} + +#endif /* def UNICODE_SUPPORT */ + + + +/* same as above but allocate memory as needed and keep track of current end + using offset - 2/6/05 EG */ + +#if 0 /* ubyte version not used */ +local void append_ubyte_to_mem( OFT( unsigned char) ubValue, + OFT( char **) pPtr, + OFT( extent *) offset, + OFT( extent *) blocksize) +#ifdef NO_PROTO + unsigned char ubValue; /* byte to append */ + char **pPtr; /* start of block */ + extent *offset; /* next byte to write */ + extent *blocksize; /* current size of block */ +#endif /* def NO_PROTO */ +{ + if (*pPtr == NULL) { + /* malloc a 1K block */ + (*blocksize) = 1024; + *pPtr = (char *) malloc(*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ubyte_to_mem"); + } + } + /* if (*offset) + 1 > (*blocksize) - 1 */ + else if ((*offset) > (*blocksize) - (1 + 1)) { + /* realloc a bigger block in 1 K increments */ + (*blocksize) += 1024; + *pPtr = realloc(*pPtr, *blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ubyte_to_mem"); + } + } + *(*pPtr + *offset) = ubValue; + (*offset)++; +} +#endif + +local void append_ushort_to_mem( OFT( ush) usValue, + OFT( char **) pPtr, + OFT( extent *) offset, + OFT( extent *) blocksize) +#ifdef NO_PROTO + ush usValue; + char **pPtr; + extent *offset; + extent *blocksize; +#endif /* def NO_PROTO */ +{ + if (*pPtr == NULL) { + /* malloc a 1K block */ + (*blocksize) = 1024; + *pPtr = (char *) malloc(*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ushort_to_mem"); + } + } + /* if (*offset) + 2 > (*blocksize) - 1 */ + else if ((*offset) > (*blocksize) - (1 + 2)) { + /* realloc a bigger block in 1 K increments */ + (*blocksize) += 1024; + *pPtr = realloc(*pPtr, (extent)*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ushort_to_mem"); + } + } + write_ushort_to_mem(usValue, (*pPtr) + (*offset)); + (*offset) += 2; +} + +local void append_ulong_to_mem(uValue, pPtr, offset, blocksize) + ulg uValue; + char **pPtr; + extent *offset; + extent *blocksize; +{ + if (*pPtr == NULL) { + /* malloc a 1K block */ + (*blocksize) = 1024; + *pPtr = (char *) malloc(*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ulong_to_mem"); + } + } + else if ((*offset) > (*blocksize) - (1 + 4)) { + /* realloc a bigger block in 1 K increments */ + (*blocksize) += 1024; + *pPtr = realloc(*pPtr, *blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_ulong_to_mem"); + } + } + write_ulong_to_mem(uValue, (*pPtr) + (*offset)); + (*offset) += 4; +} + +#ifdef ZIP64_SUPPORT + +local void append_int64_to_mem(l64Value, pPtr, offset, blocksize) + uzoff_t l64Value; + char **pPtr; + extent *offset; + extent *blocksize; +{ + if (*pPtr == NULL) { + /* malloc a 1K block */ + (*blocksize) = 1024; + *pPtr = (char *) malloc(*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_int64_to_mem"); + } + } + else if ((*offset) > (*blocksize) - (1 + 8)) { + /* realloc a bigger block in 1 K increments */ + (*blocksize) += 1024; + *pPtr = realloc(*pPtr, *blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_int64_to_mem"); + } + } + write_int64_to_mem(l64Value, (*pPtr) + (*offset)); + (*offset) += 8; +} + +#endif /* def ZIP64_SUPPORT */ + +/* Append a string to the memory block. */ +local void append_string_to_mem(strValue, strLength, pPtr, offset, blocksize) + char *strValue; + int strLength; + char **pPtr; + extent *offset; + extent *blocksize; +{ + if (strValue != NULL) { + unsigned bsize = 1024; + unsigned ssize = strLength; + unsigned i; + + if (ssize > bsize) { + bsize = ssize; + } + if (*pPtr == NULL) { + /* malloc a 1K block */ + (*blocksize) = bsize; + *pPtr = (char *) malloc(*blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_string_to_mem"); + } + } + else if ((*offset) + ssize > (*blocksize) - 1) { + /* realloc a bigger block in 1 K increments */ + (*blocksize) += bsize; + *pPtr = realloc(*pPtr, *blocksize); + if (*pPtr == NULL) { + ziperr(ZE_MEM, "append_string_to_mem"); + } + } + for (i = 0; i < ssize; i++) { + *(*pPtr + *offset + i) = *(strValue + i); + } + (*offset) += ssize; + } +} + +/* ---------------------------------------------------- */ + +/* zip64 support 08/31/2003 R.Nausedat */ +/* moved out of zip64 support 10/22/05 */ + +/* Searches pExtra for extra field with specified tag. + * If it finds one it returns a pointer to it, else NULL. + * Renamed and made generic. 10/3/03 + */ +char *get_extra_field( OFT( ush) tag, + OFT( char *) pExtra, + OFT( unsigned) iExtraLen) +#ifdef NO_PROTO + ush tag; /* tag to look for */ + char *pExtra; /* pointer to extra field in memory */ + unsigned iExtraLen; /* length of extra field */ +#endif /* def NO_PROTO */ +{ + char *pTemp; + ush usBlockTag; + ush usBlockSize; + + if( pExtra == NULL ) + return NULL; + + for (pTemp = pExtra; pTemp < pExtra + iExtraLen - ZIP_EF_HEADER_SIZE;) + { + usBlockTag = SH(pTemp); /* get tag */ + usBlockSize = SH(pTemp + 2); /* get field data size */ + if (usBlockTag == tag) + return pTemp; + pTemp += (usBlockSize + ZIP_EF_HEADER_SIZE); + } + return NULL; +} + +/* copy_nondup_extra_fields + * + * Copy any extra fields in old that are not in new to new. + * Returns the new extra fields block and newLen is new length. + */ +char *copy_nondup_extra_fields(oldExtra, oldExtraLen, newExtra, newExtraLen, newLen) + char *oldExtra; /* pointer to old extra fields */ + unsigned oldExtraLen; /* length of old extra fields */ + char *newExtra; /* pointer to new extra fields */ + unsigned newExtraLen; /* length of new extra fields */ + unsigned *newLen; /* length of new extra fields after copy */ +{ + char *returnExtra = NULL; + ush returnExtraLen = 0; + char *tempExtra; + char *pTemp; + ush tag; + ush blocksize; + + if( oldExtra == NULL ) { + /* no old extra fields so return copy of newExtra */ + if (newExtra == NULL || newExtraLen == 0) { + *newLen = 0; + return NULL; + } else { + if ((returnExtra = malloc(newExtraLen)) == NULL) + ZIPERR(ZE_MEM, "extra field copy"); + memcpy(returnExtra, newExtra, newExtraLen); + returnExtraLen = newExtraLen; + *newLen = returnExtraLen; + return returnExtra; + } + } + + /* allocate block large enough for all extra fields */ + if ((tempExtra = malloc(0xFFFF)) == NULL) + ZIPERR(ZE_MEM, "extra field copy"); + + /* look for each old extra field in new block */ + for (pTemp = oldExtra; pTemp < oldExtra + oldExtraLen;) + { + tag = SH(pTemp); /* get tag */ + blocksize = SH(pTemp + 2); /* get field data size */ + if (get_extra_field(tag, newExtra, newExtraLen) == NULL) { + /* tag not in new block so add it */ + memcpy(tempExtra + returnExtraLen, pTemp, blocksize + 4); + returnExtraLen += blocksize + 4; + } + pTemp += blocksize + 4; + } + + /* copy all extra fields from new block */ + memcpy(tempExtra + returnExtraLen, newExtra, newExtraLen); + returnExtraLen += newExtraLen; + + /* copy tempExtra to returnExtra */ + if ((returnExtra = malloc(returnExtraLen)) == NULL) + ZIPERR(ZE_MEM, "extra field copy"); + memcpy(returnExtra, tempExtra, returnExtraLen); + free(tempExtra); + + *newLen = returnExtraLen; + return returnExtra; +} + +#ifdef UNICODE_SUPPORT + +/* The latest format is + 1 byte Version of Unicode Path Extra Field + 4 bytes Name Field CRC32 Checksum + variable UTF-8 Version Of Name + */ + +local void read_Unicode_Path_entry(pZipListEntry) + struct zlist far *pZipListEntry; +{ + char *pTemp; + char *UPath; + char *iname; + ush ELen; + uch Version; + ush ULen; + ulg chksum = CRCVAL_INITIAL; + ulg iname_chksum; + + /* check if we have a Unicode Path extra field ... */ + pTemp = get_extra_field( UTF8_PATH_EF_TAG, pZipListEntry->cextra, pZipListEntry->cext ); + pZipListEntry->uname = NULL; + if( pTemp == NULL ) { + return; + } + + /* ... if so, update corresponding entries in struct zlist */ + + pTemp += 2; + + /* length of this extra field */ + ELen = SH(pTemp); + pTemp += 2; + + /* version */ + Version = (uch) *pTemp; + pTemp += 1; + if (Version > 1) { + zipwarn("Unicode Path Extra Field version > 1 - skipping", pZipListEntry->oname); + return; + } + + /* iname CRC */ + iname_chksum = LG(pTemp); + pTemp += 4; + + /* + * Compute the CRC-32 checksum of iname + */ +/* + crc_16 = crc16f((uch *)(pZipListEntry->iname), strlen(pZipListEntry->iname)); + */ + + if ((iname = malloc(strlen(pZipListEntry->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "write Unicode"); + } + strcpy(iname, pZipListEntry->iname); + + chksum = crc32(chksum, (uch *)(iname), strlen(iname)); + + free(iname); + +/* chksum = adler16(ADLERVAL_INITIAL, + (uch *)(pZipListEntry->iname), strlen(pZipListEntry->iname)); +*/ + + /* If the checksums's don't match then likely iname has been modified and + * the Unicode Path is no longer valid + */ + if (chksum != iname_chksum) { + printf("unicode_mismatch = %d\n", unicode_mismatch); + if (unicode_mismatch == 1) { + /* warn and continue */ + zipwarn("Unicode does not match path - ignoring Unicode: ", pZipListEntry->oname); + } else if (unicode_mismatch == 2) { + /* ignore and continue */ + } else if (unicode_mismatch == 0) { + /* error */ + sprintf(errbuf, "Unicode does not match path: %s\n", pZipListEntry->oname); + strcat(errbuf, + " Likely entry name changed but Unicode not updated\n"); + strcat(errbuf, + " Use -UN=i to ignore errors or n for no Unicode paths"); + zipwarn(errbuf, ""); + ZIPERR(ZE_FORM, "Unicode path error"); + } + return; + } + + ULen = ELen - 5; + + /* UTF-8 Path */ + if (ULen == 0) { + /* standard path is UTF-8 so use that */ + ULen = pZipListEntry->nam; + if ((UPath = malloc(ULen + 1)) == NULL) { + return; + } + strcpy(UPath, pZipListEntry->name); + } else { + /* use Unicode path */ + if ((UPath = malloc(ULen + 1)) == NULL) { + return; + } + strncpy(UPath, pTemp, ULen); + UPath[ULen] = '\0'; + } + pZipListEntry->uname = UPath; + return; +} + +local void read_Unicode_Path_local_entry(pZipListEntry) + struct zlist far *pZipListEntry; +{ + char *pTemp; + char *UPath; + char *iname; + ush ELen; + uch Version; + ush ULen; + ulg chksum = CRCVAL_INITIAL; + ulg iname_chksum; + + /* check if we have a Unicode Path extra field ... */ + pTemp = get_extra_field( UTF8_PATH_EF_TAG, pZipListEntry->extra, pZipListEntry->ext ); + pZipListEntry->uname = NULL; + if( pTemp == NULL ) { + return; + } + + /* ... if so, update corresponding entries in struct zlist */ + + pTemp += 2; + + /* length of this extra field */ + ELen = SH(pTemp); + pTemp += 2; + + /* version */ + Version = (uch) *pTemp; + pTemp += 1; + if (Version > 1) { + zipwarn("Unicode Path Extra Field version > 1 - skipping", pZipListEntry->oname); + return; + } + + /* iname CRC */ + iname_chksum = LG(pTemp); + pTemp += 4; + + /* + * Compute 32-bit crc of iname and AND halves to make 16-bit version + */ + /* + chksum = adler16(ADLERVAL_INITIAL, + (uch *)(pZipListEntry->iname), strlen(pZipListEntry->iname)); + */ + + if ((iname = malloc(strlen(pZipListEntry->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "write Unicode"); + } + strcpy(iname, pZipListEntry->iname); + + chksum = crc32(chksum, (uch *)(iname), strlen(iname)); + + free(iname); + + /* If the checksums's don't match then likely iname has been modified and + * the Unicode Path is no longer valid + */ + if (chksum != iname_chksum) { + if (unicode_mismatch == 1) { + /* warn and continue */ + zipwarn("Unicode does not match path - ignoring Unicode: ", pZipListEntry->oname); + } else if (unicode_mismatch == 2) { + /* ignore and continue */ + } else if (unicode_mismatch == 0) { + /* error */ + sprintf(errbuf, "Unicode does not match path: %s\n", pZipListEntry->oname); + strcat(errbuf, + " Likely entry name changed but Unicode not updated\n"); + strcat(errbuf, + " Use -UN=i to ignore errors or n for no Unicode paths"); + zipwarn(errbuf, ""); + ZIPERR(ZE_FORM, "Unicode path error"); + } + return; + } + + ULen = ELen - 5; + + /* UTF-8 Path */ + if (ULen == 0) { + /* standard path is UTF-8 so use that */ + ULen = pZipListEntry->nam; + if ((UPath = malloc(ULen + 1)) == NULL) { + return; + } + strcpy(UPath, pZipListEntry->name); + } else { + /* use Unicode path */ + if ((UPath = malloc(ULen + 1)) == NULL) { + return; + } + strncpy(UPath, pTemp, ULen); + UPath[ULen] = '\0'; + } + pZipListEntry->uname = UPath; + return; +} + +#endif /* def UNICODE_SUPPORT */ + +#ifdef ZIP64_SUPPORT /* zip64 support 08/31/2003 R.Nausedat */ + +/* searches the cextra member of zlist for a zip64 extra field. if it finds one it */ +/* updates the len, siz and off members of zlist with the corresponding values of */ +/* the zip64 extra field, that is if either the len, siz or off member of zlist is */ +/* set to its max value we have to use the corresponding value from the zip64 extra */ +/* field. as of now the dsk member of zlist is not much of interest since we should */ +/* not modify multi volume archives at all. */ +local void adjust_zip_central_entry(pZipListEntry) + struct zlist far *pZipListEntry; +{ + char *pTemp; + + /* assume not using zip64 fields */ + zip64_entry = 0; + + /* check if we have a "large file" Zip64 extra field ... */ + pTemp = get_extra_field( ZIP64_EF_TAG, pZipListEntry->cextra, pZipListEntry->cext ); + if( pTemp == NULL ) + return; + + /* using zip64 field */ + zip64_entry = 1; + pTemp += ZIP_EF_HEADER_SIZE; + + /* ... if so, update corresponding entries in struct zlist */ + if (pZipListEntry->len == ZIP_UWORD32_MAX) + { + pZipListEntry->len = LLG(pTemp); + pTemp += 8; + } + + if (pZipListEntry->siz == ZIP_UWORD32_MAX) + { + pZipListEntry->siz = LLG(pTemp); + pTemp += 8; + } + + if (pZipListEntry->off == ZIP_UWORD32_MAX) + { + pZipListEntry->off = LLG(pTemp); + pTemp += 8; + } + + if (pZipListEntry->dsk == ZIP_UWORD16_MAX) + { + pZipListEntry->dsk = LG(pTemp); + } + +} + + +/* adjust_zip_local_entry + * + * Return 1 if there is a Zip64 extra field and 0 if not + */ +local int adjust_zip_local_entry(pZipListEntry) + struct zlist far *pZipListEntry; +{ + char *pTemp; + + /* assume not using zip64 fields */ + zip64_entry = 0; + + /* check if we have a "large file" Zip64 extra field ... */ + pTemp = get_extra_field(ZIP64_EF_TAG, pZipListEntry->extra, pZipListEntry->ext ); + if( pTemp == NULL ) + return zip64_entry; + + /* using zip64 field */ + zip64_entry = 1; + pTemp += ZIP_EF_HEADER_SIZE; + + /* ... if so, update corresponding entries in struct zlist */ + if (pZipListEntry->len == ZIP_UWORD32_MAX) + { + pZipListEntry->len = LLG(pTemp); + pTemp += 8; + } + + if (pZipListEntry->siz == ZIP_UWORD32_MAX) + { + pZipListEntry->siz = LLG(pTemp); + pTemp += 8; + } + return zip64_entry; +} + +/* adds a zip64 extra field to the data the cextra member of zlist points to. If + * there is already a zip64 extra field present delete it first. + */ +local int add_central_zip64_extra_field(pZipListEntry) + struct zlist far *pZipListEntry; +{ + char *pExtraFieldPtr; + char *pTemp; + ush usTemp; + ush efsize = 0; + ush esize; + ush oldefsize; + extent len; + int used_zip64 = 0; + + /* get length of ef based on which fields exceed limits */ + /* AppNote says: + * The order of the fields in the ZIP64 extended + * information record is fixed, but the fields will + * only appear if the corresponding Local or Central + * directory record field is set to 0xFFFF or 0xFFFFFFFF. + */ + efsize = ZIP_EF_HEADER_SIZE; /* type + size */ + if (pZipListEntry->len > ZIP_UWORD32_MAX || force_zip64 == 1) { + /* compressed size */ + efsize += 8; + used_zip64 = 1; + } + if (pZipListEntry->siz > ZIP_UWORD32_MAX) { + /* uncompressed size */ + efsize += 8; + used_zip64 = 1; + } + if (pZipListEntry->off > ZIP_UWORD32_MAX) { + /* offset */ + efsize += 8; + used_zip64 = 1; + } + if (pZipListEntry->dsk > ZIP_UWORD16_MAX) { + /* disk number */ + efsize += 4; + used_zip64 = 1; + } + + if (used_zip64 && force_zip64 == 0) { + zipwarn("Large entry support disabled using -fz- but needed", ""); + return ZE_BIG; + } + + /* malloc zip64 extra field? */ + if( pZipListEntry->cextra == NULL ) + { + if (efsize == ZIP_EF_HEADER_SIZE) { + return ZE_OK; + } + if ((pExtraFieldPtr = pZipListEntry->cextra = (char *) malloc(efsize)) == NULL) { + return ZE_MEM; + } + pZipListEntry->cext = efsize; + } + else + { + /* check if we have a "large file" extra field ... */ + pExtraFieldPtr = get_extra_field(ZIP64_EF_TAG, pZipListEntry->cextra, pZipListEntry->cext); + if( pExtraFieldPtr == NULL ) + { + /* ... we don't, so re-malloc enough memory for the old extra data plus + * the size of the zip64 extra field + */ + if ((pExtraFieldPtr = (char *) malloc(efsize + pZipListEntry->cext)) == NULL) { + return ZE_MEM; + } + /* move the old extra field */ + memmove(pExtraFieldPtr, pZipListEntry->cextra, pZipListEntry->cext); + free(pZipListEntry->cextra); + pZipListEntry->cextra = pExtraFieldPtr; + pExtraFieldPtr += pZipListEntry->cext; + pZipListEntry->cext += efsize; + } + else + { + /* ... we have. sort out the existing zip64 extra field and remove it from + * pZipListEntry->cextra, re-malloc enough memory for the old extra data + * left plus the size of the zip64 extra field + */ + usTemp = SH(pExtraFieldPtr + 2); + /* if pZipListEntry->cextra == pExtraFieldPtr and pZipListEntry->cext == usTemp + efsize + * we should have only one extra field, and this is a zip64 extra field. as some + * zip tools seem to require fixed zip64 extra fields we have to check if + * usTemp + ZIP_EF_HEADER_SIZE is equal to ZIP64_LARGE_FILE_HEAD_SIZE. if it + * isn't, we free the old extra field and allocate memory for a new one + */ + if( pZipListEntry->cext == (extent)(usTemp + ZIP_EF_HEADER_SIZE) ) + { + /* just Zip64 extra field in extra field */ + if( pZipListEntry->cext != efsize ) + { + /* wrong size */ + if ((pExtraFieldPtr = (char *) malloc(efsize)) == NULL) { + return ZE_MEM; + } + free(pZipListEntry->cextra); + pZipListEntry->cextra = pExtraFieldPtr; + pZipListEntry->cext = efsize; + } + } + else + { + /* get the old Zip64 extra field out and add new */ + oldefsize = usTemp + ZIP_EF_HEADER_SIZE; + if ((pTemp = (char *) malloc(pZipListEntry->cext - oldefsize + efsize)) == NULL) { + return ZE_MEM; + } + len = (extent)(pExtraFieldPtr - pZipListEntry->cextra); + memcpy(pTemp, pZipListEntry->cextra, len); + memcpy(pTemp + len, pExtraFieldPtr + oldefsize, + pZipListEntry->cext - oldefsize - len); + pZipListEntry->cext -= oldefsize; + pExtraFieldPtr = pTemp + pZipListEntry->cext; + pZipListEntry->cext += efsize; + free(pZipListEntry->cextra); + pZipListEntry->cextra = pTemp; + } + } + } + + /* set zip64 extra field members */ + write_ushort_to_mem(ZIP64_EF_TAG, pExtraFieldPtr); + write_ushort_to_mem((ush) (efsize - ZIP_EF_HEADER_SIZE), pExtraFieldPtr + 2); + esize = ZIP_EF_HEADER_SIZE; + if (pZipListEntry->len > ZIP_UWORD32_MAX || force_zip64 == 1) { + write_int64_to_mem(pZipListEntry->len, pExtraFieldPtr + esize); + esize += 8; + } + if (pZipListEntry->siz > ZIP_UWORD32_MAX) { + write_int64_to_mem(pZipListEntry->siz, pExtraFieldPtr + esize); + esize += 8; + } + if (pZipListEntry->off > ZIP_UWORD32_MAX) { + write_int64_to_mem(pZipListEntry->off, pExtraFieldPtr + esize); + esize += 8; + } + if (pZipListEntry->dsk > ZIP_UWORD16_MAX) { + write_ulong_to_mem(pZipListEntry->dsk, pExtraFieldPtr + esize); + } + + /* un' wech */ + return ZE_OK; +} + +#if 0 +/* Remove extra field in local extra field + * Return 1 if found, else 0 + * 12/28/05 + */ +local int remove_local_extra_field(pZEntry, tag) + struct zlist far *pZEntry; + ulg tag; +{ + char *pExtra; + char *pOldExtra; + char *pOldTemp; + char *pTemp; + ush newEFSize; + ush usTemp; + ush blocksize; + + /* check if we have the extra field ... */ + pOldExtra = get_extra_field( (ush)tag, pZEntry->extra, pZEntry->ext ); + if (pOldExtra) + { + /* We have. Get rid of it. */ + blocksize = SH( pOldExtra + 2 ); + newEFSize = pZEntry->ext - blocksize; + pExtra = (char *) malloc( newEFSize ); + if( pExtra == NULL ) + ziperr(ZE_MEM, "Remove Local Extra Field"); + /* move all before EF */ + usTemp = (extent) (pOldExtra - pZEntry->extra); + pTemp = pExtra; + memcpy( pTemp, pZEntry->extra, usTemp ); + /* move all after old Zip64 EF */ + pTemp = pExtra + usTemp; + pOldTemp = pOldExtra + blocksize; + usTemp = pZEntry->ext - usTemp - blocksize; + memcpy( pTemp, pOldTemp, usTemp); + /* replace extra fields */ + pZEntry->ext = newEFSize; + free(pZEntry->extra); + pZEntry->extra = pExtra; + return 1; + } else { + return 0; + } +} + +/* Remove extra field in central extra field + * Return 1 if found, else 0 + * 12/28/05 + */ +local int remove_central_extra_field(pZEntry, tag) + struct zlist far *pZEntry; + ulg tag; +{ + char *pExtra; + char *pOldExtra; + char *pOldTemp; + char *pTemp; + ush newEFSize; + ush usTemp; + ush blocksize; + + /* check if we have the extra field ... */ + pOldExtra = get_extra_field( (ush)tag, pZEntry->cextra, pZEntry->cext ); + if (pOldExtra) + { + /* We have. Get rid of it. */ + blocksize = SH( pOldExtra + 2 ); + newEFSize = pZEntry->cext - blocksize; + pExtra = (char *) malloc( newEFSize ); + if( pExtra == NULL ) + ziperr(ZE_MEM, "Remove Local Extra Field"); + /* move all before EF */ + usTemp = (extent) (pOldExtra - pZEntry->cextra); + pTemp = pExtra; + memcpy( pTemp, pZEntry->cextra, usTemp ); + /* move all after old Zip64 EF */ + pTemp = pExtra + usTemp; + pOldTemp = pOldExtra + blocksize; + usTemp = pZEntry->cext - usTemp - blocksize; + memcpy( pTemp, pOldTemp, usTemp); + /* replace extra fields */ + pZEntry->cext = newEFSize; + free(pZEntry->cextra); + pZEntry->cextra = pExtra; + return 1; + } else { + return 0; + } +} +#endif + +/* Add Zip64 extra field to local header + * 10/5/03 EG + */ +local int add_local_zip64_extra_field(pZEntry) + struct zlist far *pZEntry; +{ + char *pZ64Extra; + char *pOldZ64Extra; + char *pOldTemp; + char *pTemp; + ush newEFSize; + ush usTemp; + ush blocksize; + ush Z64LocalLen = ZIP_EF_HEADER_SIZE + /* tag + EF Data Len */ + 8 + /* original uncompressed length of file */ + 8; /* compressed size of file */ + + /* malloc zip64 extra field? */ + /* after the below pZ64Extra should point to start of Zip64 extra field */ + if (pZEntry->ext == 0 || pZEntry->extra == NULL) + { + /* get new extra field */ + pZ64Extra = pZEntry->extra = (char *) malloc(Z64LocalLen); + if (pZEntry->extra == NULL) { + ziperr( ZE_MEM, "Zip64 local extra field" ); + } + pZEntry->ext = Z64LocalLen; + } + else + { + /* check if we have a Zip64 extra field ... */ + pOldZ64Extra = get_extra_field( ZIP64_EF_TAG, pZEntry->extra, pZEntry->ext ); + if (pOldZ64Extra == NULL) + { + /* ... we don't, so re-malloc enough memory for the old extra data plus */ + /* the size of the zip64 extra field */ + pZ64Extra = (char *) malloc( Z64LocalLen + pZEntry->ext ); + if (pZ64Extra == NULL) + ziperr( ZE_MEM, "Zip64 Extra Field" ); + /* move old extra field and update pointer and length */ + memmove( pZ64Extra, pZEntry->extra, pZEntry->ext); + free( pZEntry->extra ); + pZEntry->extra = pZ64Extra; + pZ64Extra += pZEntry->ext; + pZEntry->ext += Z64LocalLen; + } + else + { + /* ... we have. Sort out the existing zip64 extra field and remove it + * from pZEntry->extra, re-malloc enough memory for the old extra data + * left plus the size of the zip64 extra field */ + blocksize = SH( pOldZ64Extra + 2 ); + /* If the right length then go with it, else get rid of it and add a new extra field + * to existing block. */ + if (blocksize == Z64LocalLen - ZIP_EF_HEADER_SIZE) + { + /* looks good */ + pZ64Extra = pOldZ64Extra; + } + else + { + newEFSize = pZEntry->ext - (blocksize + ZIP_EF_HEADER_SIZE) + Z64LocalLen; + pZ64Extra = (char *) malloc( newEFSize ); + if( pZ64Extra == NULL ) + ziperr(ZE_MEM, "Zip64 Extra Field"); + /* move all before Zip64 EF */ + usTemp = (extent) (pOldZ64Extra - pZEntry->extra); + pTemp = pZ64Extra; + memcpy( pTemp, pZEntry->extra, usTemp ); + /* move all after old Zip64 EF */ + pTemp = pZ64Extra + usTemp; + pOldTemp = pOldZ64Extra + ZIP_EF_HEADER_SIZE + blocksize; + usTemp = pZEntry->ext - usTemp - blocksize; + memcpy( pTemp, pOldTemp, usTemp); + /* replace extra fields */ + pZEntry->ext = newEFSize; + free(pZEntry->extra); + pZEntry->extra = pZ64Extra; + pZ64Extra = pTemp + usTemp; + } + } + } + /* set/update zip64 extra field members */ + write_ushort_to_mem(ZIP64_EF_TAG, pZ64Extra); + write_ushort_to_mem((ush) (Z64LocalLen - ZIP_EF_HEADER_SIZE), pZ64Extra + 2); + write_int64_to_mem(pZEntry->len, pZ64Extra + 2 + 2); + write_int64_to_mem(pZEntry->siz, pZ64Extra + 2 + 2 + 8); + + return ZE_OK; +} + +# endif /* ZIP64_SUPPORT */ + +#ifdef UNICODE_SUPPORT +/* Add UTF-8 path extra field + * 10/11/05 + */ +local int add_Unicode_Path_local_extra_field(pZEntry) + struct zlist far *pZEntry; +{ + char *pUExtra; + char *pOldUExtra; + char *pOldTemp; + char *pTemp; +#ifdef WIN32_OEM + char *inameLocal; +#endif + ush newEFSize; + ush usTemp; + ush ULen = strlen(pZEntry->uname); + ush blocksize; + ulg chksum = CRCVAL_INITIAL; + ush ULocalLen = ZIP_EF_HEADER_SIZE + /* tag + EF Data Len */ + 1 + /* version */ + 4 + /* iname chksum */ + ULen; /* UTF-8 path */ + + /* malloc Unicode Path extra field? */ + /* after the below pUExtra should point to start of Unicode Path extra field */ + if (pZEntry->ext == 0 || pZEntry->extra == NULL) + { + /* get new extra field */ + pUExtra = pZEntry->extra = (char *) malloc(ULocalLen); + if (pZEntry->extra == NULL) { + ziperr( ZE_MEM, "UTF-8 Path local extra field" ); + } + pZEntry->ext = ULocalLen; + } + else + { + /* check if we have a Unicode Path extra field ... */ + pOldUExtra = get_extra_field( UTF8_PATH_EF_TAG, pZEntry->extra, pZEntry->ext ); + if (pOldUExtra == NULL) + { + /* ... we don't, so re-malloc enough memory for the old extra data plus */ + /* the size of the UTF-8 Path extra field */ + pUExtra = (char *) malloc( ULocalLen + pZEntry->ext ); + if (pUExtra == NULL) + ziperr( ZE_MEM, "UTF-8 Path Extra Field" ); + /* move old extra field and update pointer and length */ + memmove( pUExtra, pZEntry->extra, pZEntry->ext); + free( pZEntry->extra ); + pZEntry->extra = pUExtra; + pUExtra += pZEntry->ext; + pZEntry->ext += ULocalLen; + } + else + { + /* ... we have. Sort out the existing UTF-8 Path extra field and remove it + * from pZEntry->extra, re-malloc enough memory for the old extra data + * left plus the size of the UTF-8 Path extra field */ + blocksize = SH( pOldUExtra + 2 ); + /* If the right length then go with it, else get rid of it and add a new extra field + * to existing block. */ + if (blocksize == ULocalLen - ZIP_EF_HEADER_SIZE) + { + /* looks good */ + pUExtra = pOldUExtra; + } + else + { + newEFSize = pZEntry->ext - (blocksize + ZIP_EF_HEADER_SIZE) + ULocalLen; + pUExtra = (char *) malloc( newEFSize ); + if( pUExtra == NULL ) + ziperr(ZE_MEM, "UTF-8 Path Extra Field"); + /* move all before UTF-8 Path EF */ + usTemp = (extent) (pOldUExtra - pZEntry->extra); + pTemp = pUExtra; + memcpy( pTemp, pZEntry->extra, usTemp ); + /* move all after old UTF-8 Path EF */ + pTemp = pUExtra + usTemp; + pOldTemp = pOldUExtra + ZIP_EF_HEADER_SIZE + blocksize; + usTemp = pZEntry->ext - usTemp - blocksize; + memcpy( pTemp, pOldTemp, usTemp); + /* replace extra fields */ + pZEntry->ext = newEFSize; + free(pZEntry->extra); + pZEntry->extra = pUExtra; + pUExtra = pTemp + usTemp; + } + } + } + + /* + * Compute the Adler-16 checksum of iname + */ +/* + chksum = adler16(ADLERVAL_INITIAL, + (uch *)(pZEntry->iname), strlen(pZEntry->iname)); +*/ + +#ifdef WIN32_OEM + if ((inameLocal = malloc(strlen(pZEntry->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "write Unicode"); + } + /* if oem translation done convert back for checksum */ + if ((pZEntry->vem & 0xff00) == 0) { + /* get original */ + INTERN_TO_OEM(pZEntry->iname, inameLocal); + } else { + strcpy(inameLocal, pZEntry->iname); + } +#else +# define inameLocal (pZEntry->iname) +#endif + + chksum = crc32(chksum, (uch *)(inameLocal), strlen(inameLocal)); + +#ifdef WIN32_OEM + free(inameLocal); +#else +# undef inameLocal +#endif + + /* set/update UTF-8 Path extra field members */ + /* tag header */ + write_ushort_to_mem(UTF8_PATH_EF_TAG, pUExtra); + /* data size */ + write_ushort_to_mem((ush) (ULocalLen - ZIP_EF_HEADER_SIZE), pUExtra + 2); + /* version */ + *(pUExtra + 2 + 2) = 1; + /* iname chksum */ + write_ulong_to_mem(chksum, pUExtra + 2 + 2 + 1); + /* UTF-8 path */ + write_string_to_mem(pZEntry->uname, pUExtra + 2 + 2 + 1 + 4); + + return ZE_OK; +} + +local int add_Unicode_Path_cen_extra_field(pZEntry) + struct zlist far *pZEntry; +{ + char *pUExtra; + char *pOldUExtra; + char *pOldTemp; + char *pTemp; +#ifdef WIN32_OEM + char *inameLocal; +#endif + ush newEFSize; + ush usTemp; + ush ULen = strlen(pZEntry->uname); + ush blocksize; + ulg chksum = CRCVAL_INITIAL; + ush UCenLen = ZIP_EF_HEADER_SIZE + /* tag + EF Data Len */ + 1 + /* version */ + 4 + /* checksum */ + ULen; /* UTF-8 path */ + + /* malloc Unicode Path extra field? */ + /* after the below pUExtra should point to start of Unicode Path extra field */ + if (pZEntry->cext == 0 || pZEntry->cextra == NULL) + { + /* get new extra field */ + pUExtra = pZEntry->cextra = (char *) malloc(UCenLen); + if (pZEntry->cextra == NULL) { + ziperr( ZE_MEM, "UTF-8 Path cen extra field" ); + } + pZEntry->cext = UCenLen; + } + else + { + /* check if we have a Unicode Path extra field ... */ + pOldUExtra = get_extra_field( UTF8_PATH_EF_TAG, pZEntry->cextra, pZEntry->cext ); + if (pOldUExtra == NULL) + { + /* ... we don't, so re-malloc enough memory for the old extra data plus */ + /* the size of the UTF-8 Path extra field */ + pUExtra = (char *) malloc( UCenLen + pZEntry->cext ); + if (pUExtra == NULL) + ziperr( ZE_MEM, "UTF-8 Path Extra Field" ); + /* move old extra field and update pointer and length */ + memmove( pUExtra, pZEntry->cextra, pZEntry->cext); + free( pZEntry->cextra ); + pZEntry->cextra = pUExtra; + pUExtra += pZEntry->cext; + pZEntry->cext += UCenLen; + } + else + { + /* ... we have. Sort out the existing UTF-8 Path extra field and remove it + * from pZEntry->extra, re-malloc enough memory for the old extra data + * left plus the size of the UTF-8 Path extra field */ + blocksize = SH( pOldUExtra + 2 ); + /* If the right length then go with it, else get rid of it and add a new extra field + * to existing block. */ + if (blocksize == UCenLen - ZIP_EF_HEADER_SIZE) + { + /* looks good */ + pUExtra = pOldUExtra; + } + else + { + newEFSize = pZEntry->cext - (blocksize + ZIP_EF_HEADER_SIZE) + UCenLen; + pUExtra = (char *) malloc( newEFSize ); + if( pUExtra == NULL ) + ziperr(ZE_MEM, "UTF-8 Path Extra Field"); + /* move all before UTF-8 Path EF */ + usTemp = (extent) (pOldUExtra - pZEntry->cextra); + pTemp = pUExtra; + memcpy( pTemp, pZEntry->cextra, usTemp ); + /* move all after old UTF-8 Path EF */ + pTemp = pUExtra + usTemp; + pOldTemp = pOldUExtra + ZIP_EF_HEADER_SIZE + blocksize; + usTemp = pZEntry->cext - usTemp - blocksize; + memcpy( pTemp, pOldTemp, usTemp); + /* replace extra fields */ + pZEntry->cext = newEFSize; + free(pZEntry->cextra); + pZEntry->cextra = pUExtra; + pUExtra = pTemp + usTemp; + } + } + } + + /* + * Compute the CRC-32 checksum of iname + */ +#ifdef WIN32_OEM + if ((inameLocal = malloc(strlen(pZEntry->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "write Unicode"); + } + /* if oem translation done convert back for checksum */ + if ((pZEntry->vem & 0xff00) == 0) { + /* get original */ + INTERN_TO_OEM(pZEntry->iname, inameLocal); + } else { + strcpy(inameLocal, pZEntry->iname); + } +#else +# define inameLocal (pZEntry->iname) +#endif + + chksum = crc32(chksum, (uch *)(inameLocal), strlen(inameLocal)); + +#ifdef WIN32_OEM + free(inameLocal); +#else +# undef inameLocal +#endif + + /* + * Compute the Adler-16 checksum of iname + */ +/* + chksum = adler16(ADLERVAL_INITIAL, + (uch *)(pZEntry->iname), strlen(pZEntry->iname)); +*/ + + /* set/update UTF-8 Path extra field members */ + /* tag header */ + write_ushort_to_mem(UTF8_PATH_EF_TAG, pUExtra); + /* data size */ + write_ushort_to_mem((ush) (UCenLen - ZIP_EF_HEADER_SIZE), pUExtra + 2); + /* version */ + *(pUExtra + 2 + 2) = 1; + /* iname checksum */ + write_ulong_to_mem(chksum, pUExtra + 2 + 2 + 1); + /* UTF-8 path */ + write_string_to_mem(pZEntry->uname, pUExtra + 2 + 2 + 1 + 4); + + return ZE_OK; +} +#endif /* def UNICODE_SUPPORT */ + + +zoff_t ffile_size OF((FILE *)); + + +/* 2004-12-06 SMS. + * ffile_size() returns reliable file size or EOF. + * May be used to detect large files in a small-file program. + */ +zoff_t ffile_size( file) +FILE *file; +{ + int sts; + size_t siz; + zoff_t ofs; + char waste[ 4]; + + /* Seek to actual EOF. */ + sts = zfseeko( file, 0, SEEK_END); + if (sts != 0) + { + /* fseeko() failed. (Unlikely.) */ + ofs = EOF; + } + else + { + /* Get apparent offset at EOF. */ + ofs = zftello( file); + if (ofs < 0) + { + /* Offset negative (overflow). File too big. */ + ofs = EOF; + } + else + { + /* Seek to apparent EOF offset. + Won't be at actual EOF if offset was truncated. + */ + sts = zfseeko( file, ofs, SEEK_SET); + if (sts != 0) + { + /* fseeko() failed. (Unlikely.) */ + ofs = EOF; + } + else + { + /* Read a byte at apparent EOF. Should set EOF flag. */ + siz = fread( waste, 1, 1, file); + if (feof( file) == 0) + { + /* Not at EOF, but should be. File too big. */ + ofs = EOF; + } + } + } + } + /* Seek to BOF. + * + * 2007-05-23 SMS. + * Note that a problem in a prehistoric VAX C run-time library + * requires that rewind() be used instead of fseek(), or else + * the EOF flag is not cleared properly. + */ + /* As WIN32 has this same problem (EOF not being cleared) when + * NO_ZIP64_SUPPORT is set but LARGE_FILE_SUPPORT is set on a + * small file, seems no reason not to always use rewind(). + * 8/5/07 EG + */ +#if 0 +#ifdef VAXC + sts = rewind( file); +#else /* def VAXC */ + sts = zfseeko( file, 0, SEEK_SET); +#endif /* def VAXC [else] */ +#endif + rewind(file); + + return ofs; +} + + +#ifndef UTIL + +local void zipoddities(z) +struct zlist far *z; +{ + if ((z->vem >> 8) >= NUM_HOSTS) + { + sprintf(errbuf, "made by version %d.%d on system type %d: ", + (ush)(z->vem & 0xff) / (ush)10, (ush)(z->vem & 0xff) % (ush)10, + z->vem >> 8); + zipwarn(errbuf, z->oname); + } + if (z->ver != 10 && z->ver != 11 && z->ver != 20) + { + sprintf(errbuf, "needs unzip %d.%d on system type %d: ", + (ush)(z->ver & 0xff) / (ush)10, + (ush)(z->ver & 0xff) % (ush)10, z->ver >> 8); + zipwarn(errbuf, z->oname); + } + + if ((fix == 2) && (z->flg != z->lflg)) + /* The comparision between central and local version of the + "general purpose bit flag" cannot be used from scanzipf_regnew(), + because in the "regular" zipfile processing, the local header reads + have been postponed until the actual entry processing takes place. + They have not yet been read when "zipoddities()" is called. + This change was neccessary to support multivolume archives. + */ + { + sprintf(errbuf, "local flags = 0x%04x, central = 0x%04x: ", + z->lflg, z->flg); + zipwarn(errbuf, z->oname); + } + else if (z->flg & ~0xf && (z->flg & ~0xf0) != UTF8_BIT) + /* Only bit in high byte we support is the new UTF-8 bit */ + { + sprintf(errbuf, "undefined bits used in flags = 0x%04x: ", z->flg); + zipwarn(errbuf, z->oname); + } + if (z->how > LAST_KNOWN_COMPMETHOD) { + sprintf(errbuf, "unknown compression method %u: ", z->how); + zipwarn(errbuf, z->oname); + } + if (z->dsk) + { + sprintf(errbuf, "starts on disk %lu: ", z->dsk); + zipwarn(errbuf, z->oname); + } + if (z->att!=ASCII && z->att!=BINARY && z->att!=__EBCDIC) + { + sprintf(errbuf, "unknown internal attributes = 0x%04x: ", z->att); + zipwarn(errbuf, z->oname); + } +# if 0 +/* This test is ridiculous, it produces an error message for almost every */ +/* platform of origin other than MS-DOS, Unix, VMS, and Acorn! Perhaps */ +/* we could test "if (z->dosflag && z->atx & ~0xffL)", but what for? */ + if (((n = z->vem >> 8) != 3) && n != 2 && n != 13 && z->atx & ~0xffL) + { + sprintf(errbuf, "unknown external attributes = 0x%08lx: ", z->atx); + zipwarn(errbuf, z->oname); + } +# endif + + /* This test is just annoying, as Zip itself does not write the same + extra fields to both the local and central headers. It's much more + complicated than this test implies. 3/17/05 */ +#if 0 + if (z->ext || z->cext) + { +# if 0 + if (z->ext && z->cext && z->extra != z->cextra) + { + sprintf(errbuf, + "local extra (%ld bytes) != central extra (%ld bytes): ", + (ulg)z->ext, (ulg)z->cext); + if (noisy) fprintf(mesg, "\tzip info: %s%s\n", errbuf, z->oname); + } +# if (!defined(RISCOS) && !defined(CMS_MVS)) + /* in noisy mode, extra field sizes are always reported */ + else if (noisy) +# else /* RISCOS || CMS_MVS */ +/* avoid warnings for zipfiles created on the same type of OS system! */ +/* or, was this warning really intended (eg. OS/2)? */ + /* Only give info if extra bytes were added by another system */ + else if (noisy && ((z->vem >> 8) != (OS_CODE >> 8))) +# endif /* ?(RISCOS || CMS_MVS) */ +# endif /* 0 */ + { + fprintf(mesg, "zip info: %s has %ld bytes of %sextra data\n", + z->oname, z->ext ? (ulg)z->ext : (ulg)z->cext, + z->ext ? (z->cext ? "" : "local ") : "central "); + } + } +#endif +} + + +#if 0 /* scanzipf_fix() no longer used */ +/* + * scanzipf_fix is called with zip -F or zip -FF + * read the file from front to back and pick up the pieces + * NOTE: there are still checks missing to see if the header + * that was found is *VALID* + * + * Still much work to do so can handle more cases. 1/18/04 EG + */ +local int scanzipf_fix(f) + FILE *f; /* zip file */ +/* + The name of the zip file is pointed to by the global "zipfile". The globals + zipbeg, cenbeg, zfiles, zcount, zcomlen, zcomment, and zsort are filled in. + Return an error code in the ZE_ class. +*/ +{ + ulg a = 0L; /* attributes returned by filetime() */ + char b[CENHEAD]; /* buffer for central headers */ + ush flg; /* general purpose bit flag */ + int m; /* mismatch flag */ + extent n; /* length of name */ + uzoff_t p; /* current file offset */ + uzoff_t s; /* size of data, start of central */ + struct zlist far * far *x; /* pointer last entry's link */ + struct zlist far *z; /* current zip entry structure */ + +#ifndef ZIP64_SUPPORT + +/* 2004-12-06 SMS. + * Check for too-big file before doing any serious work. + */ + if (ffile_size( f) == EOF) + return ZE_ZIP64; + +#endif /* ndef ZIP64_SUPPORT */ + + + /* Get any file attribute valid for this OS, to set in the central + * directory when fixing the archive: + */ +# ifndef UTIL + filetime(zipfile, &a, (zoff_t*)&s, NULL); +# endif + x = &zfiles; /* first link */ + p = 0; /* starting file offset */ +# ifdef HANDLE_AMIGA_SFX + amiga_sfx_offset = 0L; +# endif + + /* Find start of zip structures */ + for (;;) { + /* look for signature */ + while ((m = getc(f)) != EOF && m != 0x50) /* 0x50 == 'P' */ + { +# ifdef HANDLE_AMIGA_SFX + if (p == 0 && m == 0) + amiga_sfx_offset = 1L; + else if (amiga_sfx_offset) { + if ((p == 1 && m != 0) || (p == 2 && m != 3) + || (p == 3 && (uch) m != 0xF3)) + amiga_sfx_offset = 0L; + } +# endif /* HANDLE_AMIGA_SFX */ + p++; + } + /* found a P */ + b[0] = (char) m; + /* local - 11/2/03 EG */ + if (fread(b+1, 3, 1, f) != 1 || (s = LG(b)) == LOCSIG) + break; + /* why search for ENDSIG if doing only local - 11/2/03 EG + if (fread(b+1, 3, 1, f) != 1 || (s = LG(b)) == LOCSIG || s == ENDSIG) + break; + */ + /* back up */ + if (zfseeko(f, -3L, SEEK_CUR)) + return ferror(f) ? ZE_READ : ZE_EOF; + /* move 1 byte forward */ + p++; + } + zipbeg = p; +# ifdef HANDLE_AMIGA_SFX + if (amiga_sfx_offset && zipbeg >= 12 && (zipbeg & 3) == 0 + && fseek(f, -12L, SEEK_CUR) == 0 && fread(b, 12, 1, f) == 1 + && LG(b + 4) == 0xF1030000 /* 1009 in Motorola byte order */) + amiga_sfx_offset = zipbeg - 4; + else + amiga_sfx_offset = 0L; +# endif /* HANDLE_AMIGA_SFX */ + + /* Read local headers */ + while (LG(b) == LOCSIG) + { + if ((z = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL || + zcount + 1 < zcount) + return ZE_MEM; + if (fread(b, LOCHEAD, 1, f) != 1) { + farfree((zvoid far *)z); + break; + } + + z->ver = SH(LOCVER + b); + z->vem = (ush)(dosify ? 20 : OS_CODE + Z_MAJORVER * 10 + Z_MINORVER); + z->dosflag = dosify; + flg = z->flg = z->lflg = SH(LOCFLG + b); + z->how = SH(LOCHOW + b); + z->tim = LG(LOCTIM + b); /* time and date into one long */ + z->crc = LG(LOCCRC + b); + z->siz = LG(LOCSIZ + b); + z->len = LG(LOCLEN + b); + n = z->nam = SH(LOCNAM + b); + z->cext = z->ext = SH(LOCEXT + b); + + z->com = 0; + z->dsk = 0; + z->att = 0; + z->atx = dosify ? a & 0xff : a; /* Attributes from filetime() */ + z->mark = 0; + z->trash = 0; + + /* attention: this one breaks the VC optimizer (Release Build) */ + /* may be fixed - 11/1/03 EG */ + s = fix > 1 ? 0L : z->siz; /* discard compressed size with -FF */ + + /* Initialize all fields pointing to malloced data to NULL */ + z->zname = z->name = z->iname = z->extra = z->cextra = z->comment = NULL; + z->oname = NULL; +#ifdef UNICODE_SUPPORT + z->uname = z->zuname = z->ouname = NULL; +#endif + + /* Link into list */ + *x = z; + z->nxt = NULL; + x = &z->nxt; + + /* Read file name and extra field and skip data */ + if (n == 0) + { + sprintf(errbuf, "%lu", (ulg)zcount + 1); + zipwarn("zero-length name for entry #", errbuf); +# ifndef DEBUG + return ZE_FORM; +# endif + } + if ((z->iname = malloc(n+1)) == NULL || + (z->ext && (z->extra = malloc(z->ext)) == NULL)) + return ZE_MEM; + if (fread(z->iname, n, 1, f) != 1 || + (z->ext && fread(z->extra, z->ext, 1, f) != 1)) + return ferror(f) ? ZE_READ : ZE_EOF; + +# ifdef ZIP64_SUPPORT + /* adjust/update siz,len and off (to come: dsk) entries */ + /* PKZIP does not care of the version set in a CDH: if */ + /* there is a zip64 extra field assigned to a CDH PKZIP */ + /* uses it, we should do so, too. */ + zip64_entry = adjust_zip_local_entry(z); + /* z->siz may be updated */ + s = fix > 1 ? 0L : z->siz; /* discard compressed size with -FF */ +# endif + + if (s && zfseeko(f, (zoff_t)s, SEEK_CUR)) + return ferror(f) ? ZE_READ : ZE_EOF; + /* If there is an extended local header, s is either 0 or + * the correct compressed size. + */ + z->iname[n] = '\0'; /* terminate name */ + z->zname = in2ex(z->iname); /* convert to external name */ + if (z->zname == NULL) + return ZE_MEM; + z->name = z->zname; + z->cextra = z->extra; + if (noisy) fprintf(mesg, "zip: reading %s\n", z->zname); + + /* Save offset, update for next header */ + z->off = p; + p += 4 + LOCHEAD + n + z->ext + s; + zcount++; + + /* Skip extended local header if there is one */ + if ((flg & 8) != 0) { + /* Skip the compressed data if compressed size is unknown. + * For safety, we should use the central directory. + */ + if (s == 0) { + for (;;) { + while ((m = getc(f)) != EOF && m != 0x50) ; /* 0x50 == 'P' */ + b[0] = (char) m; + if (fread(b+1, 15, 1, f) != 1 || LG(b) == EXTLOCSIG) + break; + if (zfseeko(f, -15L, SEEK_CUR)) + return ferror(f) ? ZE_READ : ZE_EOF; + } +# ifdef ZIP64_SUPPORT + if (zip64_entry) { /* from extra field */ + /* all are 8 bytes */ + s = LG(4 + ZIP64_EXTSIZ + b); + } else { + s = LG(4 + EXTSIZ + b); + } +# else + s = LG(4 + EXTSIZ + b); +# endif + p += s; + if ((uzoff_t) zftello(f) != p+16L) { + zipwarn("bad extended local header for ", z->zname); + return ZE_FORM; + } + } else { + /* compressed size non-zero, assume that it is valid: */ + Assert(p == zftello(f), "bad compressed size with extended header"); + + if (zfseeko(f, p, SEEK_SET) || fread(b, 16, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + if (LG(b) != EXTLOCSIG) { + zipwarn("extended local header not found for ", z->zname); + return ZE_FORM; + } + } + /* overwrite the unknown values of the local header: */ + + /* already in host format */ +# ifdef ZIP64_SUPPORT + z->crc = LG(4 + ZIP64_EXTCRC + b); + z->siz = s; + z->len = LG(4 + ZIP64_EXTLEN + b); +# else + z->crc = LG(4 + EXTCRC + b); + z->siz = s; + z->len = LG(4 + EXTLEN + b); +# endif + + p += 16L; + } + else if (fix > 1) { + /* Don't trust the compressed size */ + for (;;) { + while ((m = getc(f)) != EOF && m != 0x50) p++; /* 0x50 == 'P' */ + b[0] = (char) m; + if (fread(b+1, 3, 1, f) != 1 || (s = LG(b)) == LOCSIG || s == CENSIG) + break; + if (zfseeko(f, -3L, SEEK_CUR)) + return ferror(f) ? ZE_READ : ZE_EOF; + p++; + } + s = p - (z->off + 4 + LOCHEAD + n + z->ext); + if (s != z->siz) { + fprintf(mesg, " compressed size %s, actual size %s for %s\n", + zip_fzofft(z->siz, NULL, "u"), zip_fzofft(s, NULL, "u"), + z->zname); + z->siz = s; + } + /* next LOCSIG already read at this point, don't read it again: */ + continue; + } + + /* Read next signature */ + if (fread(b, 4, 1, f) != 1) + break; + } + + s = p; /* save start of central */ + + if (LG(b) != CENSIG && noisy) { + fprintf(mesg, "zip warning: %s %s truncated.\n", zipfile, + fix > 1 ? "has been" : "would be"); + + if (fix == 1) { + fprintf(mesg, + "Retry with option -qF to truncate, with -FF to attempt full recovery\n"); + ZIPERR(ZE_FORM, NULL); + } + } + + cenbeg = s; + + if (zipbeg && noisy) + fprintf(mesg, "%s: adjusting offsets for a preamble of %s bytes\n", + zipfile, zip_fzofft(zipbeg, NULL, "u")); + return ZE_OK; +} /* end of function scanzipf_fix() */ +#endif /* never, scanzipf_fix() no longer used */ + +#endif /* !UTIL */ + +/* + * read_local + * + * Read the local header assumed at in_file file pointer. + * localz is the returned local header, z is the central directory entry. + * + * This is used by crypt.c. + * + * Return ZE code + */ +int readlocal(localz, z) + struct zlist far **localz; + struct zlist far *z; +{ + char buf[LOCHEAD + 1]; + struct zlist far *locz; + +#ifndef UTIL + ulg start_disk = 0; + uzoff_t start_offset = 0; + char *split_path; + + start_disk = z->dsk; + start_offset = z->off; + + /* don't assume reading the right disk */ + + if (start_disk != current_in_disk) { + if (in_file) { + fclose(in_file); + in_file = NULL; + } + } + + current_in_disk = start_disk; + + /* disks are archive.z01, archive.z02, ..., archive.zip */ + split_path = get_in_split_path(in_path, current_in_disk); + + if (in_file == NULL) { + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + /* could not open split */ + + /* Ask for directory with split. Updates in_path */ + if (ask_for_split_read_path(start_disk) != ZE_OK) { + return ZE_ABORT; + } + free(split_path); + split_path = get_in_split_path(in_path, start_disk); + } + } +#endif + + /* For utilities assume archive is on one disk for now */ + + if (zfseeko(in_file, z->off, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive fseek: ", strerror(errno)); + return ZE_READ; + } + if (!at_signature(in_file, "PK\03\04")) { + fclose(in_file); + in_file = NULL; + zipwarn("Did not find entry for ", z->iname); + return ZE_FORM; + } + + /* read local header */ + if (fread(buf, LOCHEAD, 1, in_file) != 1) { + int f = ferror(in_file); + zipwarn("reading local entry: ", strerror(errno)); + fclose(in_file); + return f ? ZE_READ : ZE_EOF; + } + + /* Local Header + local file header signature 4 bytes (0x04034b50) + version needed to extract 2 bytes + general purpose bit flag 2 bytes + compression method 2 bytes + last mod file time 2 bytes + last mod file date 2 bytes + crc-32 4 bytes + compressed size 4 bytes + uncompressed size 4 bytes + file name length 2 bytes + extra field length 2 bytes + + file name (variable size) + extra field (variable size) + */ + + if ((locz = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + zipwarn("reading entry", ""); + fclose(in_file); + return ZE_MEM; + } + + locz->ver = SH(LOCVER + buf); + locz->lflg = SH(LOCFLG + buf); + locz->how = SH(LOCHOW + buf); + locz->tim = LG(LOCTIM + buf); /* time and date into one long */ + locz->crc = LG(LOCCRC + buf); + locz->nam = SH(LOCNAM + buf); + locz->ext = SH(LOCEXT + buf); + + /* Initialize all fields pointing to malloced data to NULL */ + locz->zname = locz->name = locz->iname = locz->extra = NULL; + locz->oname = NULL; +#ifdef UNICODE_SUPPORT + locz->uname = NULL; + locz->zuname = NULL; + locz->ouname = NULL; +#endif + + /* Read file name, extra field and comment field */ + if ((locz->iname = malloc(locz->nam+1)) == NULL || + (locz->ext && (locz->extra = malloc(locz->ext)) == NULL)) + return ZE_MEM; + if (fread(locz->iname, locz->nam, 1, in_file) != 1 || + (locz->ext && fread(locz->extra, locz->ext, 1, in_file) != 1)) + return ferror(in_file) ? ZE_READ : ZE_EOF; + locz->iname[z->nam] = '\0'; /* terminate name */ +#ifdef UNICODE_SUPPORT + if (unicode_mismatch != 3) + read_Unicode_Path_local_entry(locz); +#endif +#ifdef WIN32 + { + /* translate archive name from OEM if came from OEM-charset environment */ + unsigned hostver = (z->vem & 0xff); + Ext_ASCII_TO_Native(locz->iname, (z->vem >> 8), hostver, + ((z->atx & 0xffff0000L) != 0), TRUE); + } +#endif + if ((locz->name = malloc(locz->nam+1)) == NULL) + return ZE_MEM; + strcpy(locz->name, locz->iname); + +#ifdef ZIP64_SUPPORT + zip64_entry = adjust_zip_local_entry(locz); +#endif + + /* Compare localz to z */ + if (locz->ver != z->ver) { + sprintf(errbuf, "Local Version Needed (%d) does not match CD (%d): ", locz->ver, z->ver); + zipwarn(errbuf, z->iname); + } + if (locz->lflg != z->flg) { + zipwarn("Local Entry Flag does not match CD: ", z->iname); + } + if (locz->crc != z->crc) { + zipwarn("Local Entry CRC does not match CD: ", z->iname); + } + + /* as copying get uncompressed and compressed sizes from central directory */ + locz->len = z->len; + locz->siz = z->siz; + + *localz = locz; + + return ZE_OK; +} /* end function readlocal() */ + +#if 0 /* following functions are not (no longer) used. */ +/* + * scanzipf_reg starts searching for the End Signature at the end of the file + * The End Signature points to the Central Directory Signature which points + * to the Local Directory Signature + * XXX probably some more consistency checks are needed + */ +local int scanzipf_reg(f) + FILE *f; /* zip file */ +/* + The name of the zip file is pointed to by the global "zipfile". The globals + zipbeg, cenbeg, zfiles, zcount, zcomlen, zcomment, and zsort are filled in. + Return an error code in the ZE_ class. +*/ +{ + char b[CENHEAD]; /* buffer for central headers */ + extent n; /* length of name */ + struct zlist far * far *x; /* pointer last entry's link */ + struct zlist far *z; /* current zip entry structure */ + char *t; /* temporary pointer */ + char far *u; /* temporary variable */ + int found; + char *buf; /* temp buffer for reading zipfile */ +# ifdef ZIP64_SUPPORT + ulg u4; /* unsigned 4 byte variable */ + char bf[8]; + uzoff_t u8; /* unsigned 8 byte variable */ + uzoff_t censiz; /* size of central directory */ + uzoff_t z64eocd; /* Zip64 End Of Central Directory record byte offset */ +# else + ush flg; /* general purpose bit flag */ + int m; /* mismatch flag */ +# endif + zoff_t deltaoff = 0; + + +#ifndef ZIP64_SUPPORT + + /* 2004-12-06 SMS. + * Check for too-big file before doing any serious work. + */ + if (ffile_size( f) == EOF) + return ZE_ZIP64; + +#endif /* ndef ZIP64_SUPPORT */ + + + buf = malloc(4096 + 4); + if (buf == NULL) + return ZE_MEM; + +#ifdef HANDLE_AMIGA_SFX + amiga_sfx_offset = (fread(buf, 1, 4, f) == 4 && LG(buf) == 0xF3030000); + /* == 1 if this file is an Amiga executable (presumably UnZipSFX) */ +#endif + /* detect spanning signature */ + zfseeko(f, 0, SEEK_SET); + read_split_archive = (fread(buf, 1, 4, f) == 4 && LG(buf) == 0x08074b50L); + found = 0; + t = &buf[4096]; + t[1] = '\0'; + t[2] = '\0'; + t[3] = '\0'; + /* back up as much as 4k from end */ + /* zip64 support 08/31/2003 R.Nausedat */ + if (zfseeko(f, -4096L, SEEK_END) == 0) { + zipbeg = (uzoff_t) (zftello(f) + 4096L); + /* back up 4k blocks and look for End Of CD signature */ + while (!found && zipbeg >= 4096) { + zipbeg -= 4096L; + buf[4096] = t[1]; + buf[4097] = t[2]; + buf[4098] = t[3]; +/* + * XXX error check ?? + */ + fread(buf, 1, 4096, f); + zfseeko(f, -8192L, SEEK_CUR); + t = &buf[4095]; +/* + * XXX far pointer arithmetic in DOS + */ + while (t >= buf) { + /* Check for ENDSIG ("PK\5\6" in ASCII) */ + if (LG(t) == ENDSIG) { + found = 1; +/* + * XXX error check ?? + * XXX far pointer arithmetic in DOS + */ + zipbeg += (uzoff_t) (t - buf); + zfseeko(f, (zoff_t) zipbeg + 4L, SEEK_SET); + break; + } + --t; + } + } + } + else + /* file less than 4k bytes */ + zipbeg = 4096L; +/* + * XXX warn: garbage at the end of the file ignored + */ + if (!found && zipbeg > 0) { + size_t s; + + zfseeko(f, 0L, SEEK_SET); + clearerr(f); + s = fread(buf, 1, (size_t) zipbeg, f); + /* add 0 bytes at end */ + buf[s] = t[1]; + buf[s + 1] = t[2]; + buf[s + 2] = t[3]; + t = &buf[s - 1]; +/* + * XXX far pointer comparison in DOS + */ + while (t >= buf) { + /* Check for ENDSIG ("PK\5\6" in ASCII) */ + if (LG(t) == ENDSIG) { + found = 1; +/* + * XXX far pointer arithmetic in DOS + */ + zipbeg = (ulg) (t - buf); + zfseeko(f, (zoff_t) zipbeg + 4L, SEEK_SET); + break; + } + --t; + } + } + free(buf); + if (!found) { + zipwarn("missing end signature--probably not a zip file (did you", ""); + zipwarn("remember to use binary mode when you transferred it?)", ""); + return ZE_FORM; + } + +/* + * Check for a Zip64 EOCD Locator signature - 12/10/04 EG + */ +#ifndef ZIP64_SUPPORT + /* If Zip64 not enabled check if archive being read is Zip64 */ + /* back up 24 bytes (size of Z64 EOCDL and ENDSIG) */ + if (zfseeko(f, -24, SEEK_CUR) != 0) { + perror("fseek"); + return ZE_FORM; /* XXX */ + } + /* read Z64 EOCDL if there */ + if (fread(b, 20, 1, f) != 1) { + return ZE_READ; + } + /* first 4 bytes are the signature if there */ + if (LG(b) == ZIP64_EOCDL_SIG) { + zipwarn("found Zip64 signature - this may be a Zip64 archive", ""); + zipwarn("PKZIP 4.5 or later needed - set ZIP64_SUPPORT in Zip 3", ""); + return ZE_ZIP64; + } + + /* now should be back at the EOCD signature */ + if (fread(b, 4, 1, f) != 1) { + zipwarn("unable to read after relative seek", ""); + return ZE_READ; + } + if (LG(b) != ENDSIG) { + zipwarn("unable to relative seek in archive", ""); + return ZE_FORM; + } +#if 0 + if (fseek(f, -4, SEEK_CUR) != 0) { + perror("fseek"); + return ZE_FORM; /* XXX */ + } +#endif +#endif + + /* Read end header */ + if (fread(b, ENDHEAD, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + if (SH(ENDDSK + b) || SH(ENDBEG + b) || + SH(ENDSUB + b) != SH(ENDTOT + b)) + zipwarn("multiple disk information ignored", ""); + zcomlen = SH(ENDCOM + b); + if (zcomlen) + { + if ((zcomment = malloc(zcomlen)) == NULL) + return ZE_MEM; + if (fread(zcomment, zcomlen, 1, f) != 1) + { + free((zvoid *)zcomment); + zcomment = NULL; + return ferror(f) ? ZE_READ : ZE_EOF; + } +#ifdef EBCDIC + if (zcomment) + memtoebc(zcomment, zcomment, zcomlen); +#endif /* EBCDIC */ + } +#ifdef ZIP64_SUPPORT + /* account for Zip64 EOCD Record and Zip64 EOCD Locator */ + + /* Z64 EOCDL should be just before EOCD (unless this is an empty archive) */ + cenbeg = zipbeg - ZIP64_EOCDL_OFS_SIZE; + /* check for empty archive */ + /* changed cenbeg to uzoff_t so instead of cenbeg >= 0 use new check - 5/23/05 EG */ + if (zipbeg >= ZIP64_EOCDL_OFS_SIZE) { + /* look for signature */ + if (zfseeko(f, cenbeg, SEEK_SET)) { + zipwarn("end of file seeking Z64EOCDL", ""); + return ZE_FORM; + } + if (fread(bf, 4, 1, f) != 1) { + ziperr(ZE_FORM, "read error"); + } + u4 = LG(bf); + if (u4 == ZIP64_EOCDL_SIG) { + /* found Zip64 EOCD Locator */ + /* check for disk information */ + zfseeko(f, cenbeg + ZIP64_EOCDL_OFS_TOTALDISKS, SEEK_SET); + if (fread(bf, 4, 1, f) != 1) { + ziperr(ZE_FORM, "read error"); + } + u4 = LG(bf); + if (u4 != 1) { + ziperr(ZE_FORM, "multiple disk archives not yet supported"); + } + + /* look for Zip64 EOCD Record */ + zfseeko(f, cenbeg + ZIP64_EOCDL_OFS_EOCD_START, SEEK_SET); + if (fread(bf, 8, 1, f) != 1) { + ziperr(ZE_FORM, "read error"); + } + z64eocd = LLG(bf); + if (zfseeko(f, z64eocd, SEEK_SET)) { + ziperr(ZE_FORM, "error searching for Z64 EOCD Record"); + } + if (fread(bf, 4, 1, f) != 1) { + ziperr(ZE_FORM, "read error"); + } + u4 = LG(bf); + if (u4 != ZIP64_EOCD_SIG) { + ziperr(ZE_FORM, "Z64 EOCD not found but Z64 EOCD Locator exists"); + } + /* get size of CD */ + zfseeko(f, z64eocd + ZIP64_EOCD_OFS_SIZE, SEEK_SET); + if (fread(bf, 8, 1, f) != 1) { + ziperr(ZE_FORM, "read error"); + } + censiz = LLG(bf); + /* get start of CD */ + zfseeko(f, z64eocd + ZIP64_EOCD_OFS_CD_START, SEEK_SET); + if (fread(bf, 8, 1, f) == (size_t) -1) { + ziperr(ZE_FORM, "read error"); + } + cenbeg = LLG(bf); + u8 = z64eocd - cenbeg; + deltaoff = adjust ? u8 - censiz : 0L; + } else { + /* assume no Locator and no Zip64 EOCD Record */ + censiz = LG(ENDSIZ + b); + cenbeg = LG(b + ENDOFF); + u8 = zipbeg - censiz; + deltaoff = adjust ? u8 - censiz : 0L; + } + } +#else /* !ZIP64_SUPPORT */ +/* + * XXX assumes central header immediately precedes end header + */ + /* start of central directory */ + cenbeg = zipbeg - LG(ENDSIZ + b); +/* +printf("start of central directory cenbeg %ld\n", cenbeg); +*/ + + /* offset to first entry of archive */ + deltaoff = adjust ? cenbeg - LG(b + ENDOFF) : 0L; +#endif /* ?ZIP64_SUPPORT */ + + if (zipbeg < ZIP64_EOCDL_OFS_SIZE) { + /* zip file seems empty */ + return ZE_OK; + } + + if (zfseeko(f, cenbeg, SEEK_SET) != 0) { + perror("fseek"); + return ZE_FORM; /* XXX */ + } + + x = &zfiles; /* first link */ + + if (fread(b, 4, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + + while (LG(b) == CENSIG) { + /* Read central header. The portion of the central header that should + be in common with local header is read raw, for later comparison. + (this requires that the offset of ext in the zlist structure + be greater than or equal to LOCHEAD) */ + if (fread(b, CENHEAD, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + if ((z = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) + return ZE_MEM; + z->vem = SH(CENVEM + b); + for (u = (char far *)(&(z->ver)), n = 0; n < (CENNAM-CENVER); n++) + u[n] = b[CENVER + n]; + z->nam = SH(CENNAM + b); /* used before comparing cen vs. loc */ + z->cext = SH(CENEXT + b); /* may be different from z->ext */ + z->com = SH(CENCOM + b); + z->dsk = SH(CENDSK + b); + z->att = SH(CENATT + b); + z->atx = LG(CENATX + b); + z->off = LG(CENOFF + b) + deltaoff; + z->dosflag = (z->vem & 0xff00) == 0; + + /* Initialize all fields pointing to malloced data to NULL */ + z->zname = z->name = z->iname = z->extra = z->cextra = z->comment = NULL; + z->oname = NULL; +#ifdef UNICODE_SUPPORT + z->uname = NULL; /* UTF-8 path */ + z->zuname = NULL; /* Escaped local version of uname */ + z->ouname = NULL; /* Display version of zuname */ +#endif + + /* Link into list */ + *x = z; + z->nxt = NULL; + x = &z->nxt; + + /* Read file name, extra field and comment field */ + if (z->nam == 0) + { + sprintf(errbuf, "%lu", (ulg)zcount + 1); + zipwarn("zero-length name for entry #", errbuf); +#ifndef DEBUG + farfree((zvoid far *)z); + return ZE_FORM; +#endif + } + if ((z->iname = malloc(z->nam+1)) == NULL || + (z->cext && (z->cextra = malloc(z->cext)) == NULL) || + (z->com && (z->comment = malloc(z->com)) == NULL)) + return ZE_MEM; + if (fread(z->iname, z->nam, 1, f) != 1 || + (z->cext && fread(z->cextra, z->cext, 1, f) != 1) || + (z->com && fread(z->comment, z->com, 1, f) != 1)) + return ferror(f) ? ZE_READ : ZE_EOF; + z->iname[z->nam] = '\0'; /* terminate name */ + +#ifdef EBCDIC + if (z->com) + memtoebc(z->comment, z->comment, z->com); +#endif /* EBCDIC */ + +#ifdef ZIP64_SUPPORT + /* zip64 support 08/31/2003 R.Nausedat */ + /* here, we have to read the len, siz etc values from the CD */ + /* entry as we might have to adjust them regarding their */ + /* correspronding zip64 extra fields. */ + /* also, we cannot compare the values from the CD entries with */ + /* the values from the LH as they might be different. */ + z->len = LG(CENLEN + b); + z->siz = LG(CENSIZ + b); + z->crc = LG(CENCRC + b); + z->tim = LG(CENTIM + b); /* time and date into one long */ + z->how = SH(CENHOW + b); + z->flg = SH(CENFLG + b); + z->ver = SH(CENVER + b); + /* adjust/update siz,len and off (to come: dsk) entries */ + /* PKZIP does not care of the version set in a CDH: if */ + /* there is a zip64 extra field assigned to a CDH PKZIP */ + /* uses it, we should do so, too. */ + adjust_zip_central_entry(z); +#endif /* ZIP64_SUPPORT */ + + /* Update zipbeg offset, prepare for next header */ + if (z->off < zipbeg) + zipbeg = z->off; + zcount++; + /* Read next signature */ + if (fread(b, 4, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + } + + /* Point to start of header list and read local headers */ + z = zfiles; + while (z != NULL) { + /* Read next signature */ + if (zfseeko(f, z->off, SEEK_SET) != 0 || fread(b, 4, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + if (LG(b) == LOCSIG) { + if (fread(b, LOCHEAD, 1, f) != 1) + return ferror(f) ? ZE_READ : ZE_EOF; + + z->lflg = SH(LOCFLG + b); + n = SH(LOCNAM + b); + z->ext = SH(LOCEXT + b); + + /* Compare name and extra fields */ + if (n != z->nam) + { +#ifdef EBCDIC + strtoebc(z->iname, z->iname); +#endif + zipwarn("name lengths in local and central differ for ", z->iname); + return ZE_FORM; + } + if ((t = malloc(z->nam)) == NULL) + return ZE_MEM; + if (fread(t, z->nam, 1, f) != 1) + { + free((zvoid *)t); + return ferror(f) ? ZE_READ : ZE_EOF; + } + if (memcmp(t, z->iname, z->nam)) + { + free((zvoid *)t); +#ifdef EBCDIC + strtoebc(z->iname, z->iname); +#endif + zipwarn("names in local and central differ for ", z->iname); + return ZE_FORM; + } + free((zvoid *)t); + if (z->ext) + { + if ((z->extra = malloc(z->ext)) == NULL) + return ZE_MEM; + if (fread(z->extra, z->ext, 1, f) != 1) + { + free((zvoid *)(z->extra)); + return ferror(f) ? ZE_READ : ZE_EOF; + } + if (z->ext == z->cext && memcmp(z->extra, z->cextra, z->ext) == 0) + { + free((zvoid *)(z->extra)); + z->extra = z->cextra; + } + } + +#ifdef ZIP64_SUPPORT /* zip64 support 09/02/2003 R.Nausedat */ + /* + for now the below is left out if ZIP64_SUPPORT is defined as the fields + len, siz and off in struct zlist are type of int64 if ZIP64_SUPPORT + is defined. In either way, the values read from the central directory + should be valid. comments are welcome + */ +#else /* !ZIP64_SUPPORT */ + /* Check extended local header if there is one */ + /* bit 3 */ + if ((z->lflg & 8) != 0) + { + char buf2[16]; + ulg s; /* size of compressed data */ + + s = LG(LOCSIZ + b); + if (s == 0) + s = LG((CENSIZ-CENVER) + (char far *)(&(z->ver))); + if (zfseeko(f, (z->off + (4+LOCHEAD) + z->nam + z->ext + s), SEEK_SET) + || (fread(buf2, 16, 1, f) != 1)) + return ferror(f) ? ZE_READ : ZE_EOF; + if (LG(buf2) != EXTLOCSIG) + { +# ifdef EBCDIC + strtoebc(z->iname, z->iname); +# endif + zipwarn("extended local header not found for ", z->iname); + return ZE_FORM; + } + /* overwrite the unknown values of the local header: */ + for (n = 0; n < 12; n++) + b[LOCCRC+n] = buf2[4+n]; + } + + /* Compare local header with that part of central header (except + for the reserved bits in the general purpose flags and except + for the already checked entry name length */ + /* If I have read this right we are stepping through the z struct + here as a byte array. Need to fix this. 5/25/2005 EG */ + u = (char far *)(&(z->ver)); + flg = SH((CENFLG-CENVER) + u); /* Save central flags word */ + u[CENFLG-CENVER+1] &= 0x1f; /* Mask reserved flag bits */ + b[LOCFLG+1] &= 0x1f; + for (m = 0, n = 0; n < LOCNAM; n++) { + if (b[n] != u[n]) + { + if (!m) + { + zipwarn("local and central headers differ for ", z->iname); + m = 1; + } + if (noisy) + { + sprintf(errbuf, " offset %u--local = %02x, central = %02x", + (unsigned)n, (uch)b[n], (uch)u[n]); + zipwarn(errbuf, ""); + } + } + } + if (m && !adjust) + return ZE_FORM; + + /* Complete the setup of the zlist entry by translating the remaining + * central header fields in memory, starting with the fields with + * highest offset. This order of the conversion commands takes into + * account potential buffer overlaps caused by structure padding. + */ + z->len = LG((CENLEN-CENVER) + u); + z->siz = LG((CENSIZ-CENVER) + u); + z->crc = LG((CENCRC-CENVER) + u); + z->tim = LG((CENTIM-CENVER) + u); /* time and date into one long */ + z->how = SH((CENHOW-CENVER) + u); + z->flg = flg; /* may be different from z->lflg */ + z->ver = SH((CENVER-CENVER) + u); +#endif /* ?ZIP64_SUPPORT */ + + /* Clear actions */ + z->mark = 0; + z->trash = 0; +#ifdef UNICODE_SUPPORT + if (unicode_mismatch != 3) { + read_Unicode_Path_entry(z); + if (z->uname) { + /* match based on converted Unicode name */ + z->name = utf8_to_local_string(z->uname); +# ifdef EBCDIC + /* z->zname is used for printing and must be coded in native charset */ + strtoebc(z->zname, z->name); +# else + if ((z->zname = malloc(strlen(z->name) + 1)) == NULL) { + ZIPERR(ZE_MEM, "scanzipf_reg"); + } + strcpy(z->zname, z->name); +# endif + z->oname = local_to_display_string(z->zname); + } else { + /* no UTF-8 path */ + if ((z->name = malloc(strlen(z->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "scanzipf_reg"); + } + strcpy(z->name, z->iname); + if ((z->zname = malloc(strlen(z->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "scanzipf_reg"); + } + strcpy(z->zname, z->iname); + z->oname = local_to_display_string(z->iname); + } + } +#else /* !UNICODE_SUPPORT */ +# ifdef UTIL +/* We only need z->iname in the utils */ + z->name = z->iname; +# ifdef EBCDIC +/* z->zname is used for printing and must be coded in native charset */ + if ((z->zname = malloc(z->nam+1)) == NULL) + return ZE_MEM; + strtoebc(z->zname, z->iname); +# else + z->zname = z->iname; +# endif +# else /* !UTIL */ + z->zname = in2ex(z->iname); /* convert to external name */ + if (z->zname == NULL) + return ZE_MEM; + z->name = z->zname; +# endif /* ?UTIL */ + if ((z->oname = malloc(strlen(z->zname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "scanzipf_reg"); + } + strcpy(z->oname, z->zname); +#endif /* ?UNICODE_SUPPORT */ + } + else { +#ifdef EBCDIC + strtoebc(z->iname, z->iname); +#endif + zipwarn("local header not found for ", z->iname); + return ZE_FORM; + } +#ifndef UTIL + if (verbose && fix == 0) + zipoddities(z); +#endif + z = z->nxt; + } + + if (zipbeg && noisy) + fprintf(mesg, "%s: %s a preamble of %s bytes\n", + zipfile, adjust ? "adjusting offsets for" : "found", + zip_fzofft(zipbeg, NULL, "u")); +#ifdef HANDLE_AMIGA_SFX + if (zipbeg < 12 || (zipbeg & 3) != 0 /* must be longword aligned */) + amiga_sfx_offset = 0; + else if (amiga_sfx_offset) { + char buf2[16]; + if (!fseek(f, zipbeg - 12, SEEK_SET) && fread(buf2, 12, 1, f) == 1) { + if (LG(buf2 + 4) == 0xF1030000 /* 1009 in Motorola byte order */) + /* could also check if LG(buf2) == 0xF2030000... no for now */ + amiga_sfx_offset = zipbeg - 4; + else + amiga_sfx_offset = 0L; + } + } +#endif /* HANDLE_AMIGA_SFX */ + return ZE_OK; +} /* end of function scanzipf_reg() */ +#endif /* never */ + + + + +/* find_next_signature + * + * Scan the file forward and look for the next PK signature. + * + * Return 1 if find one and leave file pointer pointing to next char + * after signature and set sigbuf to signature. + * + * Return 0 if not. Will be at EOF on return unless error. + * + */ + +local char sigbuf[4]; /* signature found */ + +#if 0 /* currently unused */ +/* copy signature */ +char *copy_sig(copyto, copyfrom) + char *copyto; + char *copyfrom; +{ + int i; + + for (i = 0; i < 4; i++) { + copyto[i] = copyfrom[i]; + } + return copyto; +} +#endif /* currently unused */ + + +local int find_next_signature(f) + FILE *f; +{ + int m; + /* + zoff_t here; + */ + + /* look for P K ? ? signature */ + + m = getc(f); + + /* + here = zftello(f); + */ + + while (m != EOF) + { + if (m == 0x50 /*'P' except EBCDIC*/) { + /* found a P */ + sigbuf[0] = (char) m; + + if ((m = getc(f)) == EOF) + break; + if (m != 0x4b /*'K' except EBCDIC*/) { + /* not a signature */ + ungetc(m, f); + } else { + /* found P K */ + sigbuf[1] = (char) m; + + if ((m = getc(f)) == EOF) + break; + if (m == 0x50 /*'P' except EBCDIC*/) { + /* not a signature but maybe start of new one */ + ungetc(m, f); + continue; + } else if (m >= 16) { + /* last 2 chars expect < 16 for signature */ + continue; + } + sigbuf[2] = (char) m; + + if ((m = getc(f)) == EOF) + break; + if (m == 0x50 /*'P' except EBCDIC*/) { + /* not a signature but maybe start of new one */ + ungetc(m, f); + continue; + } else if (m >= 16) { + /* last 2 chars expect < 16 */ + continue; + } + sigbuf[3] = (char) m; + + /* found possible signature */ + return 1; + } + } + m = getc(f); + } + if (ferror(f)) { + return 0; + } + + /* found nothing */ + return 0; +} + +/* find_signature + * + * Find signature. + * + * Return 1 if found and leave file pointing to next character + * after signature. Set sigbuf with signature. + * + * Return 0 if not found. + */ + +local int find_signature(f, signature) + FILE *f; + ZCONST char *signature; +{ + int i; + char sig[4]; + /* + zoff_t here = zftello(f); + */ + + for (i = 0; i < 4; i++) + sig[i] = signature[i]; + + /* for EBCDIC */ + if (sig[0] == 'P') + sig[0] = 0x50; + if (sig[1] == 'K') + sig[1] = 0x4b; + + while (!feof(f)) { + if (!find_next_signature(f)) { + return 0; + } else { + for (i = 0; i < 4; i++) { + if (sig[i] != sigbuf[i]) { + /* not a match */ + break; + } + } + if (i == 4) { + /* found it */ + return 1; + } + } + } + return 0; +} + + +/* is_signature + * + * Compare signatures + * + * Return 1 if the signatures match. + */ + +local int is_signature(sig1, sig2) + ZCONST char *sig1; + ZCONST char *sig2; +{ + int i; + char tsig1[4]; + char tsig2[4]; + + for (i = 0; i < 4; i++) { + tsig1[i] = sig1[i]; + tsig2[i] = sig2[i]; + } + + /* for EBCDIC */ + if (tsig1[0] == 'P') + tsig1[0] = 0x50; + if (tsig1[1] == 'K') + tsig1[1] = 0x4b; + + if (tsig2[0] == 'P') + tsig2[0] = 0x50; + if (tsig2[1] == 'K') + tsig2[1] = 0x4b; + + for (i = 0; i < 4; i++) { + if (tsig1[i] != tsig2[i]) { + /* not a match */ + break; + } + } + if (i == 4) { + /* found it */ + return 1; + } + return 0; +} + + +/* at_signature + * + * Is at signature in file + * + * Return 1 if at the signature and leave file pointing to next character + * after signature. + * + * Return 0 if not. + */ + +local int at_signature(f, signature) + FILE *f; + ZCONST char *signature; +{ + int i; + extent m; + char sig[4]; + char b[4]; + + for (i = 0; i < 4; i++) + sig[i] = signature[i]; + + /* for EBCDIC */ + if (sig[0] == 'P') + sig[0] = 0x50; + if (sig[1] == 'K') + sig[1] = 0x4b; + + m = fread(b, 1, 4, f); + if (m != 4) { + return 0; + } else { + for (i = 0; i < 4; i++) { + if (sig[i] != b[i]) { + /* not a match */ + break; + } + } + if (i == 4) { + /* found it */ + return 1; + } + } + return 0; +} + + +#ifndef UTIL + +local int scanzipf_fixnew() +/* + Scan an assumed broke archive from the beginning, salvaging what can. + + Generally scanzipf_regnew() is used for reading archives normally and + for fixing archives with a readable central directory using -F. This + scan is used by -FF and is for an archive that is unreadable by + scanzipf_regnew(). + + Start with the first file of the archive, either .z01 or .zip, and + look for local entries. Read local entries found and create zlist + entries for them. If we find central directory entries, read them + and update the zlist created while reading local entries. + + The input path for the .zip file is in in_path. If this is a multiple disk + archive get the paths for splits from in_path as we go. If a split is not in + the same directory as the last split we ask the user where it is and update + in_path. + */ +/* + This is old: + + The name of the zip file is pointed to by the global "zipfile". The globals + zipbeg, cenbeg, zfiles, zcount, zcomlen, zcomment, and zsort are filled in. + Return an error code in the ZE_ class. +*/ +{ + /* This function only reads the standard End-of-CentralDir record and the + standard CentralDir-Entry records directly. To conserve stack space, + only a buffer of minimal size is declared. + */ +# if CENHEAD > ENDHEAD +# define FIXSCAN_BUFSIZE CENHEAD +# else +# define FIXSCAN_BUFSIZE ENDHEAD +# endif + + char scbuf[FIXSCAN_BUFSIZE]; /* buffer big enough for headers */ + char *split_path; + ulg eocdr_disk; + uzoff_t eocdr_offset; + + uzoff_t current_offset = 0; /* offset before */ + uzoff_t offset = 0; /* location after return from seek */ + + int skip_disk = 0; /* 1 if user asks to skip current disk */ + int skipped_disk = 0; /* 1 if skipped start disk and start offset is useless */ + + int r = 0; /* zipcopy return */ + uzoff_t s; /* size of data, start of central */ + struct zlist far * far *x; /* pointer last entry's link */ + struct zlist far *z; /* current zip entry structure */ + int plen; + char *in_path_ext; + int in_central_directory = 0; /* found a central directory record */ + struct zlist far *cz; + uzoff_t cd_total_entries = 0; /* number of entries according to EOCDR */ + ulg in_cd_start_disk; /* central directory start disk */ + uzoff_t in_cd_start_offset; /* offset of start of cd on cd start disk */ + + + total_disks = 1000000; + + /* open the zipfile */ + /* This must be .zip file, even if it doesn't exist */ + + /* see if zipfile name ends in .zip */ + plen = strlen(in_path); + +#ifdef VMS + /* On VMS, adjust plen (and in_path_ext) to avoid the file version. */ + plen -= strlen(vms_file_version(in_path)); +#endif /* def VMS */ + in_path_ext = zipfile + plen - 4; + + if (plen >= 4 && + in_path_ext[0] == '.' && + toupper(in_path_ext[1]) == 'Z' && + in_path_ext[2] >= '0' && in_path_ext[2] <= '9' && + in_path_ext[3] >= '0' && in_path_ext[3] <= '9' && + (plen == 4 || (in_path_ext[4] >= '0' && in_path_ext[4] <= '9'))) { + /* This may be a split but not the end split */ + strcpy(errbuf, "if archive to fix is split archive, need to provide\n"); + strcat(errbuf, " path of the last split with .zip extension,\n"); + strcat(errbuf, " even if it doesn't exist (zip will ask for splits)"); + zipwarn(errbuf, ""); + return ZE_FORM; + } + + if ((in_file = zfopen(in_path, FOPR)) == NULL) { + zipwarn("could not open input archive: ", in_path); + } + else + { + +#ifndef ZIP64_SUPPORT + /* 2004-12-06 SMS. + * Check for too-big file before doing any serious work. + */ + if (ffile_size( in_file) == EOF) { + fclose(in_file); + in_file = NULL; + zipwarn("input file requires Zip64 support: ", in_path); + return ZE_ZIP64; + } +#endif /* ndef ZIP64_SUPPORT */ + + /* look for End Of Central Directory Record */ + + /* back up 64k (the max size of the EOCDR) from end */ + if (zfseeko(in_file, -0x40000L, SEEK_END) != 0) { + /* assume file is less than 64 KB so backup to beginning */ + if (zfseeko(in_file, 0L, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", in_path); + return ZE_READ; + } + } + + + /* find EOCD Record signature */ + if (!find_signature(in_file, "PK\05\06")) { + /* No End Of Central Directory Record */ + strcpy(errbuf, "Missing end (EOCDR) signature - either this archive\n"); + strcat(errbuf, " is not readable or the end is damaged"); + zipwarn(errbuf, ""); + } + else + { + /* at start of data after EOCDR signature */ + eocdr_offset = (uzoff_t) zftello(in_file); + + /* OK, it is possible this is not the last EOCDR signature (might be + EOCDR signature from a stored archive in the last 64 KB) and so not + the one we want. + + The below assumes the signature does not appear in the assumed + ASCII text .ZIP file comment. Even if something like UTF-8 + is stored in the comment, it's unlikely the binary \05 and \06 + will be in the comment text. + */ + while (find_signature(in_file, "PK\05\06")) { + eocdr_offset = (uzoff_t) zftello(in_file); + } + + /* found EOCDR */ + /* format is + end of central dir signature 4 bytes (0x06054b50) + number of this disk 2 bytes + number of the disk with the + start of the central directory 2 bytes + total number of entries in the + central directory on this disk 2 bytes + total number of entries in + the central directory 2 bytes + size of the central directory 4 bytes + offset of start of central + directory with respect to + the starting disk number 4 bytes + .ZIP file comment length 2 bytes + .ZIP file comment (variable size) + */ + + if (zfseeko(in_file, eocdr_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", in_path); + return ZE_READ; + } + + /* read the EOCDR */ + s = fread(scbuf, 1, ENDHEAD, in_file); + + /* make sure we read enough bytes */ + if (s < ENDHEAD) { + sprintf(errbuf, "End record (EOCDR) only %s bytes - assume truncated", + zip_fzofft(s, NULL, "u")); + zipwarn(errbuf, ""); + } + else + { + /* the first field should be number of this (the last) disk */ + eocdr_disk = (ulg)SH(scbuf); + total_disks = eocdr_disk + 1; + + /* assume this is this disk - if Zip64 it may not be as the + disk number may be bigger than this field can hold + */ + current_in_disk = total_disks - 1; + + /* Central Directory disk, offset, and total entries */ + in_cd_start_disk = (ulg)SH(scbuf + 2); + in_cd_start_offset = (uzoff_t)LG(scbuf + 12); + cd_total_entries = (uzoff_t)SH(scbuf + 6); + + /* the in_cd_start_disk should always be less than the total_disks, + unless the -1 flags are being used */ + if (total_disks < 0x10000 && in_cd_start_disk > total_disks) { + zipwarn("End record (EOCDR) has bad disk numbers - ignoring EOCDR", ""); + total_disks = 0; + } + else + { + /* length of zipfile comment */ + zcomlen = SH(scbuf + ENDCOM); + if (zcomlen) + { + if ((zcomment = malloc(zcomlen + 1)) == NULL) + return ZE_MEM; + if (fread(zcomment, zcomlen, 1, in_file) != 1) + { + free((zvoid *)zcomment); + zcomment = NULL; + zipwarn("zipfile comment truncated - ignoring", ""); + } else { + zcomment[zcomlen] = '\0'; + } +#ifdef EBCDIC + if (zcomment) + memtoebc(zcomment, zcomment, zcomlen); +#endif /* EBCDIC */ + } + } + if (total_disks != 1) + sprintf(errbuf, " Found end record (EOCDR) - says expect %lu splits", total_disks); + else + sprintf(errbuf, " Found end record (EOCDR) - says expect single disk archive"); + zipmessage(errbuf, ""); + if (zcomment) + zipmessage(" Found archive comment", ""); + } /* good EOCDR */ + + } /* found EOCDR */ + + /* if total disks is other than 1 then this is not start disk */ + /* if the EOCDR is bad, total_disks is 0 */ + + /* if total_disks = 0, then guess if this is a single-disk archive + by seeing if starts with local header */ + + if (total_disks == 0) { + int issig; + /* seek to top */ + if (zfseeko(in_file, 0, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", in_path); + return ZE_READ; + } + /* get next signature */ + issig = find_next_signature(in_file); + if (issig) { + current_in_offset = zftello(in_file); + if (current_in_offset == 4 && is_signature(sigbuf, "PK\03\03")) { + /* could be multi-disk aborted signature at top */ + /* skip */ + issig = find_next_signature(in_file); + } else if (current_in_offset <= 4 && is_signature(sigbuf, "PK\03\03")) { + /* multi-disk spanning signature */ + total_disks = 99999; + } + } + if (issig && total_disks == 0) { + current_in_offset = zftello(in_file); + + if (current_in_offset == 8 && is_signature(sigbuf, "PK\03\04")) { + + /* Local Header Record at top */ + + printf("Is this a single-disk archive? (y/n): "); + fflush(stdout); + + if (fgets(errbuf, 100, stdin) != NULL) { + if (errbuf[0] == 'y' || errbuf[0] == 'Y') { + total_disks = 1; + zipmessage(" Assuming single-disk archive", ""); + } + } + } + } + } + if (!noisy) + /* if quiet assume single-disk archive */ + total_disks = 1; + + if (total_disks == 1000000) { + /* still don't know, so ask */ + printf("Is this a single-disk archive? (y/n): "); + fflush(stdout); + + if (fgets(errbuf, 100, stdin) != NULL) { + if (errbuf[0] == 'y' || errbuf[0] == 'Y') { + total_disks = 1; + zipmessage(" Assuming single-disk archive", ""); + } + } + } + if (total_disks == 1000000) { + /* assume max */ + total_disks = 100000; + } + + } /* .zip file exists */ + + /* Skip reading the Zip64 EOCDL, Zip64 EOCDR, or central directory */ + + /* Now read the archive starting with first disk. Find local headers, + create entry in zlist, then copy entry to new archive */ + + /* Multi-volume file names end in .z01, .z02, ..., .z10, .zip for 11 disk archive */ + + /* Unless quiet, always close the in_path disk and ask user for first disk, + unless there is an End Of Central Directory record and that says there is + only one disk. + If quiet, assume the file pointed to is a single file archive to fix. */ + if (noisy && in_file) { + fclose(in_file); + in_file = NULL; + } + + /* Read the archive disks - no idea how many disks there are + since we can't trust the EOCDR and other end records + */ + zipmessage("Scanning for entries...", ""); + + for (current_in_disk = 0; current_in_disk < total_disks; current_in_disk++) { + /* get the path for this disk */ + split_path = get_in_split_path(in_path, current_in_disk); + + /* if in_file is not NULL then in_file is already open */ + if (in_file == NULL) { + /* open the split */ + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + int result; + /* could not open split */ + + /* Ask for directory with split. Updates global variable in_path */ + result = ask_for_split_read_path(current_in_disk); + if (result == ZE_ABORT) { + zipwarn("could not find split: ", split_path); + return ZE_ABORT; + } else if (result == ZE_EOF) { + zipmessage_nl("", 1); + zipwarn("user ended reading - closing archive", ""); + return ZE_EOF; + } else if (result == ZE_FORM) { + /* user asked to skip this disk */ + zipmessage_nl("", 1); + sprintf(errbuf, "skipping disk %lu ...\n", current_in_disk); + zipwarn(errbuf, ""); + skip_disk = 1; + break; + } + + split_path = get_in_split_path(in_path, current_in_disk); + } + if (skip_disk) { + /* skip this current disk - this works because central directory entries + can't be split across splits */ + skip_disk = 0; + skipped_disk = 1; + continue; + } + } + + if (skipped_disk) { + /* Not much to do here as between entries. Entries are copied + in zipcopy() and that has to handle missing disks while + reading data for an entry. + */ + } + + /* Main loop */ + /* Look for next signature and process it */ + while (find_next_signature(in_file)) { + current_in_offset = zftello(in_file); + + if (is_signature(sigbuf, "PK\05\06")) { + + /* End Of Central Directory Record */ + + sprintf(errbuf, "EOCDR found (%2lu %6s)...", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 1); + + + } else if (is_signature(sigbuf, "PK\06\06")) { + + /* Zip64 End Of Central Directory Record */ + + sprintf(errbuf, "Zip64 EOCDR found (%2lu %6s)...", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 1); + + + } else if (is_signature(sigbuf, "PK\06\07")) { + + /* Zip64 End Of Central Directory Locator */ + + sprintf(errbuf, "Zip64 EOCDL found (%2lu %6s)...", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 1); + + + } else if (is_signature(sigbuf, "PK\03\04")) { + + /* Local Header Record */ + + + if (verbose) { + sprintf(errbuf, " Local (%2lu %6s):", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 0); + } + + /* Create zlist entry. Most will be filled in by zipcopy(). */ + + if ((z = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + zipwarn("reading central directory", ""); + return ZE_MEM; + } + + z->vem = 0; + z->ver = 0; + z->flg = 0; + z->how = 0; + z->tim = 0; /* time and date into one long */ + z->crc = 0; + z->siz = 0; + z->len = 0; + z->nam = 0; /* used before comparing cen vs. loc */ + z->cext = 0; /* may be different from z->ext */ + z->com = 0; + z->dsk = 0; + z->att = 0; + z->atx = 0; + z->off = 0; + z->dosflag = 0; + + /* Initialize all fields pointing to malloced data to NULL */ + z->zname = z->name = z->iname = z->extra = z->cextra = z->comment = NULL; + z->oname = NULL; +#ifdef UNICODE_SUPPORT + z->uname = z->zuname = z->ouname = NULL; +#endif + + /* Attempt to copy entry */ + + r = zipcopy(z); + + if (in_central_directory) { + sprintf(errbuf, "Entry after central directory found (%2lu %6s)...", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 1); + in_central_directory = 0; + } + + if (r == ZE_EOF) + /* user said no more splits */ + break; + else if (r == ZE_OK) { + zcount++; + files_total++; + bytes_total += z->siz; + + /* Link into list */ + if (zfiles == NULL) + /* first link */ + x = &zfiles; + /* Link into list */ + *x = z; + z->nxt = NULL; + x = &z->nxt; + } + + } else if (is_signature(sigbuf, "PK\01\02")) { + + /* Central directory header */ + + + /* sort the zlist */ + if (in_central_directory == 0) { + zipmessage("Central Directory found...", ""); + /* If one or more files, sort by name */ + if (zcount) + { + struct zlist far * far *x; /* pointer into zsort array */ + struct zlist far *z; /* pointer into zfiles linked list */ + int i = 0; + extent zl_size = zcount * sizeof(struct zlist far *); + + if (zl_size / sizeof(struct zlist far *) != zcount || + (x = zsort = (struct zlist far **)malloc(zl_size)) == NULL) + return ZE_MEM; + for (z = zfiles; z != NULL; z = z->nxt) + x[i++] = z; + qsort((char *)zsort, zcount, sizeof(struct zlist far *), zqcmp); + + /* Skip Unicode searching */ + } + } + + if (verbose) { + sprintf(errbuf, " Cen (%2lu %6s): ", + current_in_disk + 1, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipmessage_nl(errbuf, 0); + } + + in_central_directory = 1; + + /* Read central directory entry */ + + /* central directory signature */ + + /* The format of a central directory record + central file header signature 4 bytes (0x02014b50) + version made by 2 bytes + version needed to extract 2 bytes + general purpose bit flag 2 bytes + compression method 2 bytes + last mod file time 2 bytes + last mod file date 2 bytes + crc-32 4 bytes + compressed size 4 bytes + uncompressed size 4 bytes + file name length 2 bytes + extra field length 2 bytes + file comment length 2 bytes + disk number start 2 bytes + internal file attributes 2 bytes + external file attributes 4 bytes + relative offset of local header 4 bytes + + file name (variable size) + extra field (variable size) + file comment (variable size) + */ + + if (fread(scbuf, CENHEAD, 1, in_file) != 1) { + zipwarn("reading central directory: ", strerror(errno)); + zipwarn("bad archive - error reading central directory", ""); + zipwarn("skipping this entry...", ""); + continue; + } + + if ((cz = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + zipwarn("reading central directory", ""); + return ZE_MEM; + } + + cz->vem = SH(CENVEM + scbuf); + cz->ver = SH(CENVER + scbuf); + cz->flg = SH(CENFLG + scbuf); + cz->how = SH(CENHOW + scbuf); + cz->tim = LG(CENTIM + scbuf); /* time and date into one long */ + cz->crc = LG(CENCRC + scbuf); + cz->siz = LG(CENSIZ + scbuf); + cz->len = LG(CENLEN + scbuf); + cz->nam = SH(CENNAM + scbuf); /* used before comparing cen vs. loc */ + cz->cext = SH(CENEXT + scbuf); /* may be different from z->ext */ + cz->com = SH(CENCOM + scbuf); + cz->dsk = SH(CENDSK + scbuf); + cz->att = SH(CENATT + scbuf); + cz->atx = LG(CENATX + scbuf); + cz->off = LG(CENOFF + scbuf); + cz->dosflag = (cz->vem & 0xff00) == 0; + + /* Initialize all fields pointing to malloced data to NULL */ + cz->zname = cz->name = cz->iname = cz->extra = cz->cextra = NULL; + cz->comment = cz->oname = NULL; +#ifdef UNICODE_SUPPORT + cz->uname = cz->zuname = cz->ouname = NULL; +#endif + + /* Read file name, extra field and comment field */ + if (cz->nam == 0) + { + sprintf(errbuf, "%lu", (ulg)zcount + 1); + zipwarn("zero-length name for entry #", errbuf); + zipwarn("skipping this entry...", ""); + continue; + } + if ((cz->iname = malloc(cz->nam+1)) == NULL || + (cz->cext && (cz->cextra = malloc(cz->cext + 1)) == NULL) || + (cz->com && (cz->comment = malloc(cz->com + 1)) == NULL)) + return ZE_MEM; + if (fread(cz->iname, cz->nam, 1, in_file) != 1 || + (cz->cext && fread(cz->cextra, cz->cext, 1, in_file) != 1) || + (cz->com && fread(cz->comment, cz->com, 1, in_file) != 1)) { + zipwarn("error reading entry: ", strerror(errno)); + zipwarn("skipping this entry...", ""); + continue; + } + cz->iname[cz->nam] = '\0'; /* terminate name */ + + /* Look up this name in zlist from local entries */ + z = zsearch(cz->iname); + + + if (z && z->tim == cz->tim) { + + /* Apparently as iname and date and time match this central + directory entry goes with this zlist entry */ + + if (verbose) { + /* cen dir name matches a local name */ + sprintf(errbuf, "updating: %s", cz->iname); + zipmessage_nl(errbuf, 0); + } + + if (z->crc != cz->crc) { + sprintf(errbuf, "local (%lu) and cen (%lu) crc mismatch", z->crc, cz->crc); + zipwarn(errbuf, ""); + } + + z->vem = cz->vem; + /* z->ver = cz->ver; */ + /* z->flg = cz->flg; */ + /* z->how = cz->how; */ + /* z->tim = cz->tim; */ /* time and date into one long */ + /* z->crc = cz->crc; */ + /* z->siz = cz->siz; */ + /* z->len = cz->len; */ + /* z->nam = cz->nam; */ /* used before comparing cen vs. loc */ + z->cext = cz->cext; /* may be different from z->ext */ + z->com = cz->com; + z->cextra = cz->cextra; + z->comment = cz->comment; + /* z->dsk = cz->dsk; */ + z->att = cz->att; + z->atx = cz->atx; + /* z->off = cz->off; */ + z->dosflag = cz->dosflag; + +#ifdef UNICODE_SUPPORT + if (unicode_mismatch != 3 && z->uname == NULL) { + if (z->flg & UTF8_BIT) { + /* path is UTF-8 */ + if ((z->uname = malloc(strlen(z->iname) + 1)) == NULL) { + ZIPERR(ZE_MEM, "reading archive"); + } + strcpy(z->uname, z->iname); + } else { + /* check for UTF-8 path extra field */ + read_Unicode_Path_entry(z); + } + } +#endif + +#ifdef WIN32 + /* Input path may be OEM */ + { + unsigned hostver = (z->vem & 0xff); + Ext_ASCII_TO_Native(z->iname, (z->vem >> 8), hostver, + ((z->atx & 0xffff0000L) != 0), FALSE); + } +#endif + +#ifdef EBCDIC + if (z->com) + memtoebc(z->comment, z->comment, z->com); +#endif /* EBCDIC */ +#ifdef WIN32 + /* Comment may be OEM */ + { + unsigned hostver = (z->vem & 0xff); + Ext_ASCII_TO_Native(z->comment, (z->vem >> 8), hostver, + ((z->atx & 0xffff0000L) != 0), FALSE); + } +#endif + +#ifdef ZIP64_SUPPORT + /* zip64 support 08/31/2003 R.Nausedat */ + /* here, we have to read the len, siz etc values from the CD */ + /* entry as we might have to adjust them regarding their */ + /* correspronding zip64 extra fields. */ + /* also, we cannot compare the values from the CD entries with */ + /* the values from the LH as they might be different. */ + + /* adjust/update siz,len and off (to come: dsk) entries */ + /* PKZIP does not care of the version set in a CDH: if */ + /* there is a zip64 extra field assigned to a CDH PKZIP */ + /* uses it, we should do so, too. */ + /* + adjust_zip_central_entry(z); + */ +#endif + + /* Update zipbeg beginning of archive offset, prepare for next header */ +/* + if (z->dsk == 0 && (!zipbegset || z->off < zipbeg)) { + zipbeg = z->off; + zipbegset = 1; + } + zcount++; + */ + +#ifndef UTIL + if (verbose) + zipoddities(z); +#endif + + current_offset = zftello(y); + + if (zfseeko(y, z->off, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("writing archive seek: ", strerror(errno)); + return ZE_WRITE; + } + + if (putlocal(z, PUTLOCAL_REWRITE) != ZE_OK) + zipwarn("Error rewriting local header", ""); + + if (zfseeko(y, current_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("write archive seek: ", strerror(errno)); + return ZE_WRITE; + } + offset = zftello(y); + if (current_offset != offset) { + fclose(in_file); + in_file = NULL; + zipwarn("seek after local: ", strerror(errno)); + return ZE_WRITE; + } + + if (verbose) + zipmessage_nl("", 1); + + } else { + /* cen dir name does not match local name */ + sprintf(errbuf, "no local entry: %s", cz->iname); + zipmessage_nl(errbuf, 1); + } + + } else if (zfiles == NULL && is_signature(sigbuf, "PK\07\010")) { + + /* assume spanning signature at top of archive */ + if (total_disks == 1) { + zipmessage(" Found spanning marker, but did not expect split (multi-disk) archive...", ""); + + } else if (total_disks > 1) { + zipmessage(" Found spanning marker - expected as this is split (multi-disk) archive...", ""); + + } else { + zipmessage(" Found spanning marker - could be split archive...", ""); + + } + + } else { + + /* this signature shouldn't be here */ + int c; + char errbuftemp[40]; + + strcpy(errbuf, "unexpected signature "); + for (c = 0; c < 4; c++) { + sprintf(errbuftemp, "%02x ", sigbuf[c]); + strcat(errbuf, errbuftemp); + } + sprintf(errbuftemp, "on disk %lu at %s\n", current_in_disk, + zip_fzofft(current_in_offset - 4, NULL, "u")); + strcat(errbuf, errbuftemp); + zipwarn(errbuf, ""); + zipwarn("skipping this signature...", ""); + } + + + } /* while reading file */ + + /* close disk and do next disk */ + if (in_file) + fclose(in_file); + in_file = NULL; + free(split_path); + + if (r == ZE_EOF) + /* user says no more splits */ + break; + + } /* for each disk */ + + return ZE_OK; + +} /* end of function scanzipf_fixnew() */ + +#endif /* !UTIL */ + + + + + + +/* ---------------------- */ +/* New regular scan */ + +/* + * scanzipf_regnew is similar to the orignal scanzipf_reg in that it + * reads the end of the archive and goes from there. Unlike that + * scan this one stops after reading the central directory and does + * not read the local headers. After the directory scan for new + * files is done in zip.c the zlist created here is used to read + * the old archive entries there. The local headers are read using + * readlocal() in zipcopy(). + * + * This scan assumes the zip file is well structured. If not it may + * fail and the new scanzipf_fixnew should be used. + * + * 2006-2-4, 2007-12-10 EG + */ + +local int scanzipf_regnew() +/* + The input path for the .zip file is in in_path. If a split archive, + the path for each split is created from the current disk number + and in_path. If a split is not in the same directory as the last + split we ask the user where it is and update in_path. + */ +/* + This is old but more or less still applies: + + The name of the zip file is pointed to by the global "zipfile". The globals + zipbeg, cenbeg, zfiles, zcount, zcomlen, zcomment, and zsort are filled in. + Return an error code in the ZE_ class. +*/ +{ + /* In this function, a local buffer is used to read in the following Zip + structures: + End-of-CentralDir record (EOCDR) (ENDHEAD) + Zip64-End-of-CentralDir-Record locator (Zip64 EOCDL) (EC64LOC) + Zip64-End-of-CentralDir record (Zip64 EOCDR) (EC64REC) + CentralDir-Entry record (CENHEAD) + To conserve valuable stack space, this buffer is sized to the largest + of these structures. + */ +# if CENHEAD > ENDHEAD +# define SCAN_BUFSIZE CENHEAD /* CENHEAD should be the larger struct */ +# else +# define SCAN_BUFSIZE ENDHEAD +# endif + +#ifdef ZIP64_SUPPORT +# if EC64REC > SCAN_BUFSIZE +# undef SCAN_BUFSIZE +# define SCAN_BUFSIZE EC64REC /* EC64 record should be largest struct */ +# endif +# if EC64LOC > SCAN_BUFSIZE +# undef SCAN_BUFSIZE +# define SCAN_BUFSIZE EC64LOC +# endif +#endif + + char scbuf[SCAN_BUFSIZE]; /* buffer just enough for all header types */ + char *split_path; + ulg eocdr_disk; + uzoff_t eocdr_offset; +# ifdef ZIP64_SUPPORT + ulg z64eocdr_disk; + uzoff_t z64eocdr_offset; + uzoff_t z64eocdr_size; + ush version_made; + ush version_needed = 0; + zoff_t zip64_eocdr_start; + zoff_t z64eocdl_offset; +# endif /* def ZIP64_SUPPORT */ + uzoff_t cd_total_entries; /* num of entries as read from (Zip64) EOCDR */ + ulg in_cd_start_disk; /* central directory start disk */ + uzoff_t in_cd_start_offset; /* offset of start of cd on cd start disk */ + uzoff_t adjust_offset = 0; /* bytes before first entry (size of sfx prefix) */ + uzoff_t cd_total_size = 0; /* total size of cd */ + + + int first_CD = 1; /* looking for first CD entry */ + int zipbegset = 0; + + int skip_disk = 0; /* 1 if user asks to skip current disk */ + int skipped_disk = 0; /* 1 if skipped start disk and start offset is useless */ + + uzoff_t s; /* size of data, start of central */ + struct zlist far * far *x; /* pointer last entry's link */ + struct zlist far *z; /* current zip entry structure */ + + + /* open the zipfile */ + if ((in_file = zfopen(in_path, FOPR)) == NULL) { + zipwarn("could not open input archive", in_path); + return ZE_OPEN; + } + +#ifndef ZIP64_SUPPORT + /* 2004-12-06 SMS. + * Check for too-big file before doing any serious work. + */ + if (ffile_size( in_file) == EOF) { + fclose(in_file); + in_file = NULL; + zipwarn("input file requires Zip64 support: ", in_path); + return ZE_ZIP64; + } +#endif /* ndef ZIP64_SUPPORT */ + + /* look for End Of Central Directory Record */ + + /* In a valid Zip archive, the EOCDR can be at most (64k-1 + ENDHEAD + 4) + bytes (=65557 bytes) from the end of the file. + We back up 128k, to allow some junk being appended to a Zip file. + */ + if ((zfseeko(in_file, -0x20000L, SEEK_END) != 0) || + /* Some fseek() implementations (e.g. MSC 8.0 16-bit) fail to signal + an error when seeking before the beginning of the file. + As work-around, we check the position returned by zftello() + for the error value -1. + */ + (zftello(in_file) == (zoff_t)-1L)) { + /* file is less than 128 KB so back up to beginning */ + if (zfseeko(in_file, 0L, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", in_path); + return ZE_READ; + } + } + + /* find EOCD Record signature */ + if (!find_signature(in_file, "PK\05\06")) { + /* No End Of Central Directory Record */ + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("bad archive - missing end signature", ""); + zipwarn("(If downloaded, was binary mode used? If not, the", ""); + zipwarn(" archive may be scrambled and not recoverable)", ""); + zipwarn("Can't use -F to fix (try -FF)", ""); + } else{ + zipwarn("missing end signature--probably not a zip file (did you", ""); + zipwarn("remember to use binary mode when you transferred it?)", ""); + zipwarn("(if you are trying to read a damaged archive try -F)", ""); + } + return ZE_FORM; + } + + /* at start of data after EOCDR signature */ + eocdr_offset = (uzoff_t) zftello(in_file); + + /* OK, it is possible this is not the last EOCDR signature (might be + EOCDR signature from a stored archive in the last 128 KB) and so not + the one we want. + + The below assumes the signature does not appear in the assumed ASCII text + .ZIP file comment. + */ + while (find_signature(in_file, "PK\05\06")) { + /* previous one was not the one */ + eocdr_offset = (uzoff_t) zftello(in_file); + } + + /* found EOCDR */ + /* format is + end of central dir signature 4 bytes (0x06054b50) + number of this disk 2 bytes + number of the disk with the + start of the central directory 2 bytes + total number of entries in the + central directory on this disk 2 bytes + total number of entries in + the central directory 2 bytes + size of the central directory 4 bytes + offset of start of central + directory with respect to + the starting disk number 4 bytes + .ZIP file comment length 2 bytes + .ZIP file comment (variable size) + */ + + if (zfseeko(in_file, eocdr_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", in_path); + return ZE_READ; + } + + /* read the EOCDR */ + s = fread(scbuf, 1, ENDHEAD, in_file); + + /* the first field should be number of this (the last) disk */ + eocdr_disk = (ulg)SH(scbuf); + total_disks = eocdr_disk + 1; + + /* Assume EOCDR disk is this disk. If a lot of disks, the Zip64 field + may be needed and this EOCDR field could be set to the Zip64 flag + value as the disk number may be bigger than this field can hold. + */ + current_in_disk = total_disks - 1; + + /* Central Directory disk, offset, and total entries */ + in_cd_start_disk = (ulg)SH(scbuf + ENDBEG); + in_cd_start_offset = (uzoff_t)LG(scbuf + ENDOFF); + cd_total_entries = (uzoff_t)SH(scbuf + ENDTOT); + cd_total_size = (uzoff_t)LG(scbuf + ENDSIZ); + + /* length of zipfile comment */ + zcomlen = SH(scbuf + ENDCOM); + if (zcomlen) + { + if ((zcomment = malloc(zcomlen + 1)) == NULL) + return ZE_MEM; + if (fread(zcomment, zcomlen, 1, in_file) != 1) + { + free((zvoid *)zcomment); + zcomment = NULL; + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + zcomment[zcomlen] = '\0'; +#ifdef EBCDIC + if (zcomment) + memtoebc(zcomment, zcomment, zcomlen); +#endif /* EBCDIC */ + } + + if (cd_total_entries == 0) { + /* empty archive */ + + fclose(in_file); + in_file = NULL; + return ZE_OK; + } + + /* if total disks is other than 1 then multi-disk archive */ + if (total_disks != 1) { + /* zipfile name must end in .zip for split archives */ + int plen = strlen(in_path); + char *in_path_ext; + + if (adjust) { + zipwarn("Adjusting split archives not yet supported", ""); + return ZE_FORM; + } + +#ifdef VMS + /* On VMS, adjust plen (and in_path_ext) to avoid the file version. */ + plen -= strlen(vms_file_version(in_path)); +#endif /* def VMS */ + in_path_ext = zipfile + plen - 4; + + if (plen < 4 || + in_path_ext[0] != '.' || + toupper(in_path_ext[1]) != 'Z' || + toupper(in_path_ext[2]) != 'I' || + toupper(in_path_ext[3]) != 'P') { + zipwarn("archive name must end in .zip for splits", ""); + fclose(in_file); + in_file = NULL; + return ZE_PARMS; + } + } + + /* if input or output are split archives, must be different archives */ + if ((total_disks != 1 || split_method) && !show_files && + strcmp(in_path, out_path) == 0) { + fclose(in_file); + in_file = NULL; + zipwarn("cannot update a split archive (use --out option)", ""); + return ZE_PARMS; + } + + /* if fixing archive, input and output must be different archives */ + if (fix == 1 && strcmp(in_path, out_path) == 0) { + fclose(in_file); + in_file = NULL; + zipwarn("must use --out when fixing an archive", ""); + return ZE_PARMS; + } + + + /* Get sfx offset if adjusting. Above we made sure not split archive. */ + /* Also check for an offset if fix and single disk archive. */ + if ((fix == 1 && total_disks == 1) || adjust) { + zoff_t cd_start; +# ifdef ZIP64_SUPPORT + zoff_t zip64_eocdr_start; +# endif + + /* First attempt. If the CD start offset and size are valid in the EOCDR + (meaning they are not the Zip64 flag values that say the actual values + are in the Zip64 EOCDR), we can use them to get the offset */ + if (in_cd_start_offset != 0xFFFFFFFF && cd_total_size != 0xFFFFFFFF) { + /* Search for start of central directory */ + /* There still might be a Zip64 EOCDR. This assumes if there is + a Zip64 EOCDR, it's version 1 and 52 bytes */ + cd_start = eocdr_offset - cd_total_size - 24 - 56; + if (zfseeko(in_file, cd_start, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("could not seek back to start of central directory: ", strerror(errno)); + zipwarn("(try -FF)", ""); + } else { + zipwarn("reading archive fseek: ", strerror(errno)); + } + return ZE_FORM; + } + if (find_signature(in_file, "PK\01\02")) { + /* Should now be after first central directory header signature in archive */ + adjust_offset = zftello(in_file) - 4 - in_cd_start_offset; + } else { + zipwarn("central dir not where expected - could not adjust offsets", ""); + zipwarn("(try -FF)", ""); + return ZE_FORM; + } + } else { + + /* Second attempt. We need the Zip64 EOCDL to get the offset */ + + /* + * Check for a Zip64 EOCD Locator signature + */ + + /* Format of Z64EOCD Locator is + zip64 end of central dir locator + signature 4 bytes (0x07064b50) + number of the disk with the + start of the zip64 end of + central directory 4 bytes + relative offset of the zip64 + end of central directory record 8 bytes + total number of disks 4 bytes + */ + + /* back up 20 bytes from EOCDR to Z64 EOCDL */ + if (zfseeko(in_file, eocdr_offset - 24, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("could not seek back to Zip64 EOCDL: ", strerror(errno)); + zipwarn("(try -FF)", ""); + } else { + zipwarn("reading archive fseek: ", strerror(errno)); + } + return ZE_FORM; + } + if (at_signature(in_file, "PK\06\07")) +#ifndef ZIP64_SUPPORT + { + fclose(in_file); + in_file = NULL; + zipwarn("found Zip64 signature - this may be a Zip64 archive", ""); + zipwarn("Need PKZIP 4.5 or later compatible zip", ""); + zipwarn("Set ZIP64_SUPPORT in Zip 3", ""); + return ZE_ZIP64; + } +#else /* ZIP64_SUPPORT */ + { + z64eocdl_offset = zftello(in_file) - 4; + + /* read Z64 EOCDL */ + if (fread(scbuf, EC64LOC, 1, in_file) != 1) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive: ", strerror(errno)); + return ZE_READ; + } + /* now should be back at the EOCD signature */ + if (!at_signature(in_file, "PK\05\06")) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to read EOCD after seek: ", in_path); + return ZE_READ; + } + + /* read disk and offset to Zip64 EOCDR and total disks */ + z64eocdr_disk = LG(scbuf); + z64eocdr_offset = LLG(scbuf + 4); + total_disks = LG(scbuf + 12); + + /* For now no split archives */ + if (total_disks != 1) { + zipwarn("Adjusting split archives not supported: ", in_path); + zipwarn("(try -FF)", ""); + return ZE_FORM; + } + + /* go to the Zip64 EOCDR */ + if (zfseeko(in_file, z64eocdr_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive fseek: ", strerror(errno)); + return ZE_FORM; + } + /* Should be at Zip64 EOCDR signature */ + if (at_signature(in_file, "PK\06\06")) { + /* apparently no offset */ + + } else { + /* Wasn't there, so calculate based on Zip64 EOCDL offset */ + + zip64_eocdr_start = z64eocdl_offset - 24 - 56; + if (zfseeko(in_file, zip64_eocdr_start, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("could not seek back to Zip64 EOCDR: ", strerror(errno)); + zipwarn("(try -FF)", ""); + } else { + zipwarn("reading archive fseek: ", strerror(errno)); + } + return ZE_FORM; + } + if (find_next_signature(in_file) && is_signature(sigbuf, "PK\06\06")) { + /* Should now be after Zip64 EOCDR signature in archive */ + adjust_offset = zftello(in_file) - 4 - z64eocdr_offset; + } else { + zipwarn("Could not determine offset of entries", ""); + zipwarn("(try -FF)", ""); + return ZE_FORM; + } + } + } +#endif + } + if (noisy) { + if (adjust_offset) { + sprintf(errbuf, "Zip entry offsets appear off by %s bytes - correcting...", + zip_fzofft(adjust_offset, NULL, NULL)); + } else { + sprintf(errbuf, "Zip entry offsets do not need adjusting"); + } + zipmessage(errbuf, ""); + } + } + + + /* + * Check for a Zip64 EOCD Locator signature + */ + + /* Format of Z64EOCD Locator is + zip64 end of central dir locator + signature 4 bytes (0x07064b50) + number of the disk with the + start of the zip64 end of + central directory 4 bytes + relative offset of the zip64 + end of central directory record 8 bytes + total number of disks 4 bytes + */ + + /* back up 20 bytes from EOCDR to Z64 EOCDL */ + if (zfseeko(in_file, eocdr_offset - 24, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("bad archive - could not seek back to Zip64 EOCDL: ", strerror(errno)); + zipwarn("(try -FF)", ""); + } else { + zipwarn("reading archive fseek: ", strerror(errno)); + } + return ZE_FORM; + } + if (at_signature(in_file, "PK\06\07")) +#ifndef ZIP64_SUPPORT + { + fclose(in_file); + in_file = NULL; + zipwarn("found Zip64 signature - this may be a Zip64 archive", ""); + zipwarn("Need PKZIP 4.5 or later compatible zip", ""); + zipwarn("Set ZIP64_SUPPORT in Zip 3", ""); + return ZE_ZIP64; + } +#else /* ZIP64_SUPPORT */ + { + z64eocdl_offset = zftello(in_file) - 4; + /* read Z64 EOCDL */ + if (fread(scbuf, EC64LOC, 1, in_file) != 1) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive: ", strerror(errno)); + return ZE_READ; + } + /* now should be back at the EOCD signature */ + if (!at_signature(in_file, "PK\05\06")) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to read EOCD after seek: ", in_path); + return ZE_READ; + } + + /* read disk and offset to Zip64 EOCDR and total disks */ + z64eocdr_disk = LG(scbuf); + z64eocdr_offset = LLG(scbuf + 4) + adjust_offset; + total_disks = LG(scbuf + 12); + + /* set the current disk */ + current_in_disk = total_disks - 1; + + /* Now need to read the Zip64 EOCD Record to get version needed + to extract */ + + if (z64eocdr_disk != total_disks - 1) { + /* Zip64 EOCDR not on this disk */ + + /* done with this disk (since apparently there are no CD entries + on it) */ + fclose(in_file); + in_file = NULL; + + /* get the path for the disk with the Zip64 EOCDR */ + split_path = get_in_split_path(in_path, z64eocdr_disk); + + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + /* could not open split */ + + /* Ask where this split is. This call also updates global in_path. */ + if (ask_for_split_read_path(z64eocdr_disk) != ZE_OK) { + return ZE_ABORT; + } + free(split_path); + split_path = get_in_split_path(in_path, z64eocdr_disk); + } + free(split_path); + } + + current_in_disk = z64eocdr_disk; + + /* go to the Zip64 EOCDR */ + if (zfseeko(in_file, z64eocdr_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive fseek: ", strerror(errno)); + return ZE_FORM; + } + /* Should be at Zip64 EOCDR signature */ + if (!at_signature(in_file, "PK\06\06")) { + /* Wasn't there, so calculate based on Zip64 EOCDL offset */ + zip64_eocdr_start = z64eocdl_offset - 24 - 56; + if (zfseeko(in_file, zip64_eocdr_start, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("bad archive - could not seek back to Zip64 EOCDR: ", strerror(errno)); + zipwarn("(try -FF)", ""); + } else { + zipwarn("reading archive fseek: ", strerror(errno)); + } + return ZE_FORM; + } + if (find_next_signature(in_file) && is_signature(sigbuf, "PK\06\06")) { + /* Should now be after Zip64 EOCDR signature in archive */ + adjust_offset = zftello(in_file) - 4 - z64eocdr_offset; + zipwarn("Zip64 EOCDR not found where expected - compensating", ""); + zipwarn("(try -A to adjust offsets)", ""); + } else { + fclose(in_file); + in_file = NULL; + if (fix == 1) { + zipwarn("bad archive - Zip64 EOCDR not found in split: ", in_path); + zipwarn("(try -FF)", ""); + } else { + zipwarn("Zip64 End Of Central Directory Record not found: ", in_path); + } + return ZE_FORM; + } + } + + /* + * Read the Z64 End Of Central Directory Record + */ + + /* The format of the Z64 EOCDR is + zip64 end of central dir + signature 4 bytes (0x06064b50) + size of zip64 end of central + directory record 8 bytes + version made by 2 bytes + version needed to extract 2 bytes + number of this disk 4 bytes + number of the disk with the + start of the central directory 4 bytes + total number of entries in the + central directory on this disk 8 bytes + total number of entries in the + central directory 8 bytes + size of the central directory 8 bytes + offset of start of central + directory with respect to + the starting disk number 8 bytes + (version 2 of the Zip64 EOCDR has more after this) + zip64 extensible data sector (variable size) + */ + + /* read the first 52 bytes of the Zip64 EOCDR (we don't support + version 2, which supports PKZip licensed features) + */ + s = fread(scbuf, 1, EC64REC, in_file); + if (s < EC64REC) { + if (fix == 1) { + zipwarn("bad archive - Zip64 EOCDR bad or truncated", ""); + zipwarn("(try -FF)", ""); + } else { + zipwarn("Zip64 EOCD Record bad or truncated", ""); + } + fclose(in_file); + in_file = NULL; + return ZE_FORM; + } + z64eocdr_size = LLG(scbuf); + version_made = SH(scbuf + 8); + version_needed = SH(scbuf + 10); + in_cd_start_disk = LG(scbuf + 16); + cd_total_entries = LLG(scbuf + 28); + in_cd_start_offset = LLG(scbuf + 44) + adjust_offset; + + if (version_needed > 46) { + int major = version_needed / 10; + int minor = version_needed - (major * 10); + sprintf(errbuf, "This archive requires version %d.%d", major, minor); + zipwarn(errbuf, ""); + zipwarn("Zip currently only supports up to version 4.6 archives", ""); + zipwarn("(up to 4.5 if bzip2 is not compiled in)", ""); + if (fix == 1) + zipwarn("If -F fails try -FF to try to salvage something", ""); + else if (fix == 2) + zipwarn("Attempting to salvage what can", ""); + else { + zipwarn("Try -F to attempt to read anyway", ""); + fclose(in_file); + in_file = NULL; + return ZE_FORM; + } + } + } +#endif /* ?ZIP64_SUPPORT */ + + /* Now read the central directory and create the zlist */ + + /* Multi-volume file names end in .z01, .z02, ..., .z10, .zip for 11 disk archive */ + + in_cd_start_offset += adjust_offset; + cenbeg = in_cd_start_offset; + zipbegset = 0; + zipbeg = 0; + first_CD = 1; + + /* if the central directory starts on other than this disk, close this disk */ + if (current_in_disk != in_cd_start_disk) { + /* close current disk */ + fclose(in_file); + in_file = NULL; + } + + /* Read the disks with the central directory in order - usually the + central directory fits on the last disk, but it doesn't have to. + */ + for (current_in_disk = in_cd_start_disk; + current_in_disk < total_disks; + current_in_disk++) { + /* get the path for this disk */ + if (current_in_disk == total_disks - 1) { + /* last disk is archive.zip */ + if ((split_path = malloc(strlen(in_path) + 1)) == NULL) { + zipwarn("reading archive: ", in_path); + return ZE_MEM; + } + strcpy(split_path, in_path); + } else { + /* other disks are archive.z01, archive.z02, ... */ + split_path = get_in_split_path(in_path, current_in_disk); + } + + /* if in_file is not NULL then in_file is already open */ + if (in_file == NULL) { + /* open the split */ + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + int result; + /* could not open split */ + + /* Ask for directory with split. Updates global variable in_path */ + result = ask_for_split_read_path(current_in_disk); + if (result == ZE_ABORT) { + zipwarn("could not find split: ", split_path); + return ZE_ABORT; + } else if (result == ZE_FORM) { + /* user asked to skip this disk */ + sprintf(errbuf, "skipping disk %lu ...\n", current_in_disk); + zipwarn(errbuf, ""); + skip_disk = 1; + break; + } + + if (current_in_disk == total_disks - 1) { + /* last disk is archive.zip */ + if ((split_path = malloc(strlen(in_path) + 1)) == NULL) { + zipwarn("reading archive: ", in_path); + return ZE_MEM; + } + strcpy(split_path, in_path); + } else { + /* other disks are archive.z01, archive.z02, ... */ + split_path = get_in_split_path(zipfile, current_in_disk); + } + } + if (skip_disk) { + /* skip this current disk - this works because central directory entries + can't be split across splits */ + skip_disk = 0; + skipped_disk = 1; + continue; + } + } + + if (skipped_disk) { + /* skipped start CD disk so start searching for CD signature at start of disk */ + first_CD = 0; + } else { + /* seek to the first CD entry */ + if (first_CD) { + if (zfseeko(in_file, in_cd_start_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("unable to seek in input file ", split_path); + return ZE_READ; + } + first_CD = 0; + x = &zfiles; /* first link */ + } + } + + /* Main loop */ + /* Look for next signature and process it */ + while (find_next_signature(in_file)) { + current_in_offset = zftello(in_file); + + if (is_signature(sigbuf, "PK\05\06")) { + /* End Of Central Directory Record */ + /* + fprintf(mesg, "EOCDR signature at %d / %I64d\n", + current_in_disk, current_in_offset - 4); + */ + break; + + } else if (is_signature(sigbuf, "PK\06\06")) { + /* Zip64 End Of Central Directory Record */ + /* + fprintf(mesg, "Zip64 EOCDR signature at %d / %I64d\n", + current_in_disk, current_in_offset - 4); + */ + break; + + } else if (!is_signature(sigbuf, "PK\01\02")) { + /* Not Central Directory Record */ + + /* this signature shouldn't be here */ + if (fix == 1) { + int c; + char errbuftemp[40]; + + strcpy(errbuf, "bad archive - unexpected signature "); + for (c = 0; c < 4; c++) { + sprintf(errbuftemp, "%02x ", sigbuf[c]); + strcat(errbuf, errbuftemp); + } + sprintf(errbuftemp, "on disk %lu at %s\n", current_in_disk, + zip_fzofft(current_in_offset - 4, NULL, "u")); + strcat(errbuf, errbuftemp); + zipwarn(errbuf, ""); + zipwarn("skipping this signature...", ""); + continue; + } else { + sprintf(errbuf, "unexpected signature on disk %lu at %s\n", + current_in_disk, zip_fzofft(current_in_offset - 4, NULL, "u")); + zipwarn(errbuf, ""); + zipwarn("archive not in correct format: ", split_path); + zipwarn("(try -F to attempt recovery)", ""); + fclose(in_file); + in_file = NULL; + return ZE_FORM; + } + } + + /* central directory signature */ + if (verbose && fix == 1) { + fprintf(mesg, "central directory header signature on disk %lu at %s\n", + current_in_disk, zip_fzofft(current_in_offset - 4, NULL, "u")); + } + + /* The format of a central directory record + central file header signature 4 bytes (0x02014b50) + version made by 2 bytes + version needed to extract 2 bytes + general purpose bit flag 2 bytes + compression method 2 bytes + last mod file time 2 bytes + last mod file date 2 bytes + crc-32 4 bytes + compressed size 4 bytes + uncompressed size 4 bytes + file name length 2 bytes + extra field length 2 bytes + file comment length 2 bytes + disk number start 2 bytes + internal file attributes 2 bytes + external file attributes 4 bytes + relative offset of local header 4 bytes + + file name (variable size) + extra field (variable size) + file comment (variable size) + */ + + if (fread(scbuf, CENHEAD, 1, in_file) != 1) { + zipwarn("reading central directory: ", strerror(errno)); + if (fix == 1) { + zipwarn("bad archive - error reading central directory", ""); + zipwarn("skipping this entry...", ""); + continue; + } else { + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + } + + if ((z = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + zipwarn("reading central directory", ""); + return ZE_MEM; + } + + z->vem = SH(CENVEM + scbuf); + z->ver = SH(CENVER + scbuf); + z->flg = SH(CENFLG + scbuf); + z->how = SH(CENHOW + scbuf); + z->tim = LG(CENTIM + scbuf); /* time and date into one long */ + z->crc = LG(CENCRC + scbuf); + z->siz = LG(CENSIZ + scbuf); + z->len = LG(CENLEN + scbuf); + z->nam = SH(CENNAM + scbuf); /* used before comparing cen vs. loc */ + z->cext = SH(CENEXT + scbuf); /* may be different from z->ext */ + z->com = SH(CENCOM + scbuf); + z->dsk = SH(CENDSK + scbuf); + z->att = SH(CENATT + scbuf); + z->atx = LG(CENATX + scbuf); + z->off = LG(CENOFF + scbuf); /* adjust_offset is added below */ + z->dosflag = (z->vem & 0xff00) == 0; + + /* Initialize all fields pointing to malloced data to NULL */ + z->zname = z->name = z->iname = z->extra = z->cextra = z->comment = NULL; + z->oname = NULL; +#ifdef UNICODE_SUPPORT + z->uname = z->zuname = z->ouname = NULL; +#endif + + /* Read file name, extra field and comment field */ + if (z->nam == 0) + { + sprintf(errbuf, "%lu", (ulg)zcount + 1); + zipwarn("zero-length name for entry #", errbuf); + if (fix == 1) { + zipwarn("skipping this entry...", ""); + continue; + } +#ifndef DEBUG + return ZE_FORM; +#endif + } + if ((z->iname = malloc(z->nam+1)) == NULL || + (z->cext && (z->cextra = malloc(z->cext)) == NULL) || + (z->com && (z->comment = malloc(z->com)) == NULL)) + return ZE_MEM; + if (fread(z->iname, z->nam, 1, in_file) != 1 || + (z->cext && fread(z->cextra, z->cext, 1, in_file) != 1) || + (z->com && fread(z->comment, z->com, 1, in_file) != 1)) { + if (fix == 1) { + zipwarn("error reading entry: ", strerror(errno)); + zipwarn("skipping this entry...", ""); + continue; + } + return ferror(in_file) ? ZE_READ : ZE_EOF; + } + z->iname[z->nam] = '\0'; /* terminate name */ +#ifdef UNICODE_SUPPORT + if (unicode_mismatch != 3) { + if (z->flg & UTF8_BIT) { + char *iname; + /* path is UTF-8 */ + if ((z->uname = malloc(strlen(z->iname) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->uname, z->iname); + /* Create a local name. If UTF-8 system this should also be UTF-8 */ + iname = utf8_to_local_string(z->uname); + if (iname) { + free(z->iname); + z->iname = iname; + } + else + zipwarn("illegal UTF-8 name: ", z->uname); + } else { + /* check for UTF-8 path extra field */ + read_Unicode_Path_entry(z); + } + } +#endif + +#ifdef WIN32 + /* Input path may be OEM */ + { + unsigned hostver = (z->vem & 0xff); + Ext_ASCII_TO_Native(z->iname, (z->vem >> 8), hostver, + ((z->atx & 0xffff0000L) != 0), FALSE); + } +#endif + +#ifdef EBCDIC + if (z->com) + memtoebc(z->comment, z->comment, z->com); +#endif /* EBCDIC */ +#ifdef WIN32 + /* Comment may be OEM */ + { + unsigned hostver = (z->vem & 0xff); + Ext_ASCII_TO_Native(z->comment, (z->vem >> 8), hostver, + ((z->atx & 0xffff0000L) != 0), FALSE); + } +#endif + +#ifdef ZIP64_SUPPORT + /* zip64 support 08/31/2003 R.Nausedat */ + /* here, we have to read the len, siz etc values from the CD */ + /* entry as we might have to adjust them regarding their */ + /* correspronding zip64 extra fields. */ + /* also, we cannot compare the values from the CD entries with */ + /* the values from the LH as they might be different. */ + + /* adjust/update siz,len and off (to come: dsk) entries */ + /* PKZIP does not care of the version set in a CDH: if */ + /* there is a zip64 extra field assigned to a CDH PKZIP */ + /* uses it, we should do so, too. */ + adjust_zip_central_entry(z); +#endif + /* if adjusting for sfx prefix, add the offset */ + if ((fix ==1 && total_disks == 1) || adjust) z->off += adjust_offset; + + /* Update zipbeg beginning of archive offset, prepare for next header */ + if (z->dsk == 0 && (!zipbegset || z->off < zipbeg)) { + zipbeg = z->off; + zipbegset = 1; + } + zcount++; + + /* Clear actions */ + z->mark = 0; + z->trash = 0; +#if defined(UNICODE_SUPPORT) && !defined(UTIL) + z->zname = in2ex(z->iname); /* convert to external name */ + if (z->zname == NULL) + return ZE_MEM; + if ((z->name = malloc(strlen(z->zname) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->name, z->zname); + z->oname = local_to_display_string(z->iname); + +# ifdef WIN32 + z->namew = NULL; + z->inamew = NULL; + z->znamew = NULL; +# endif + + if (unicode_mismatch != 3) { + if (z->uname) { + /* create zuname which is alternate zname for matching based on + converted Unicode name */ + char *name; + + /* Convert UTF-8 to current local character set */ + name = utf8_to_local_string(z->uname); + + if (name == NULL) { + /* + zipwarn("illegal UTF-8 name: ", z->uname); + */ + /* not able to convert name, so use iname */ + if ((name = malloc(strlen(z->iname) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(name, z->iname); + } + +# ifdef EBCDIC + /* z->zname is used for printing and must be coded in native charset */ + strtoebc(z->zuname, name); +# else /* !EBCDIC */ + if ((z->zuname = malloc(strlen(name) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->zuname, name); + /* For output to terminal */ + if (unicode_escape_all) { + char *ouname; + /* Escape anything not 7-bit ASCII */ + ouname = utf8_to_escape_string(z->uname); + if (ouname) + z->ouname = ouname; + else { + if ((z->ouname = malloc(strlen(name) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->ouname, name); + } + } else { + if ((z->ouname = malloc(strlen(name) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->ouname, name); + } +# ifdef WIN32 + + if (!no_win32_wide) { + z->inamew = utf8_to_wchar_string(z->uname); + z->znamew = in2exw(z->inamew); /* convert to external name */ + if (z->znamew == NULL) + return ZE_MEM; + } + + local_to_oem_string(z->ouname, z->ouname); + /* For matching. There seems to be something lost + in the translation from displaying a name in a + console window using zip -su on Win32 and using + that name in a command line to match what's in + the archive. This is klugy though. + */ + if ((z->wuname = malloc(strlen(z->ouname) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->wuname, z->ouname); + oem_to_local_string(z->wuname, z->wuname); +# endif /* WIN32 */ +# endif /* ?EBCDIC */ + } else { + /* no uname */ +# ifdef WIN32 + if (!no_win32_wide) { + z->inamew = local_to_wchar_string(z->iname); + z->znamew = in2exw(z->inamew); /* convert to external name */ + if (z->znamew == NULL) + return ZE_MEM; + } +# endif + } + } +#else /* !(UNICODE_SUPPORT && !UTIL) */ +# ifdef UTIL +/* We only need z->iname in the utils */ + z->name = z->iname; +# ifdef EBCDIC +/* z->zname is used for printing and must be coded in native charset */ + if ((z->zname = malloc(z->nam+1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strtoebc(z->zname, z->iname); +# else + z->zname = z->iname; +# endif +# else /* !UTIL */ + z->zname = in2ex(z->iname); /* convert to external name */ + if (z->zname == NULL) + return ZE_MEM; + z->name = z->zname; +# endif /* ?UTIL */ + if ((z->oname = malloc(strlen(z->zname) + 1)) == NULL) { + zipwarn("could not allocate memory: scanzipf_reg", ""); + return ZE_MEM; + } + strcpy(z->oname, z->zname); +#endif /* ?(UNICODE_SUPPORT && !UTIL) */ + +#ifndef UTIL + if (verbose && fix == 0) + zipoddities(z); +#endif + + /* Link into list */ + *x = z; + z->nxt = NULL; + x = &z->nxt; + + } /* while reading file */ + + /* close disk and do next disk */ + fclose(in_file); + in_file = NULL; + free(split_path); + + if (!is_signature(sigbuf, "PK\01\02")) { + /* if the last signature is not a CD signature and we get here then + hit either the Zip64 EOCDR or the EOCDR and done */ + break; + } + + } /* for each disk */ + + if (zcount != cd_total_entries) { + sprintf(errbuf, "expected %s entries but found %s", + zip_fzofft(cd_total_entries, NULL, "u"), + zip_fzofft(zcount, NULL, "u")); + zipwarn(errbuf, ""); + return ZE_FORM; + } + + return ZE_OK; + +} /* end of function scanzipf_regnew() */ + + + + + + + + +/* ---------------------- */ + + + + +/* + * readzipfile initializes the global variables that hold the zipfile + * directory info and opens the zipfile. For the actual zipfile scan, + * the subroutine scanzipf_reg() or scanzipf_fix() is called, + * depending on the mode of operation (regular processing, or zipfix mode). + */ +int readzipfile() +/* + The name of the zip file is pointed to by the global "zipfile". + The globals zipbeg, zfiles, zcount, and zcomlen are initialized. + Return an error code in the ZE_ class. +*/ +{ + FILE *f; /* zip file */ + int retval; /* return code */ + int readable; /* 1 if zipfile exists and is readable */ + + /* Initialize zip file info */ + zipbeg = 0; + zfiles = NULL; /* Points to first header */ + zcount = 0; /* number of files */ + zcomlen = 0; /* zip file comment length */ + retval = ZE_OK; + f = NULL; /* shut up some compilers */ + zipfile_exists = 0; + + /* If zip file exists, read headers and check structure */ +#ifdef VMS + if (zipfile == NULL || !(*zipfile) || !strcmp(zipfile, "-")) + return ZE_OK; + { + int rtype; + + if ((VMSmunch(zipfile, GET_RTYPE, (char *)&rtype) == RMS$_NORMAL) && + (rtype == FAT$C_VARIABLE)) { + fprintf(mesg, + "\n Error: zipfile is in variable-length record format. Please\n\ + run \"bilf b %s\" to convert the zipfile to fixed-length\n\ + record format.\n\n", zipfile); + return ZE_FORM; + } + } + readable = ((f = zfopen(zipfile, FOPR)) != NULL); +#else /* !VMS */ + readable = (zipfile != NULL && *zipfile && strcmp(zipfile, "-")); + if (readable) { + readable = ((f = zfopen(zipfile, FOPR)) != NULL); + } +#endif /* ?VMS */ + + /* skip check if streaming */ + if (!readable) { + if (!zip_to_stdout && fix != 2 && strcmp(in_path, out_path)) { + /* If -O used then in_path must exist */ + if (fix == 1) + zipwarn("No .zip file found\n ", + "(If all you have are splits (.z01, .z02, ...) and no .zip, try -FF)"); + ZIPERR(ZE_OPEN, zipfile); + } + } else { + zipfile_exists = 1; + } + +#ifdef MVS + /* Very nasty special case for MVS. Just because the zipfile has been + * opened for reading does not mean that we can actually read the data. + * Typical JCL to create a zipfile is + * + * //ZIPFILE DD DISP=(NEW,CATLG),DSN=prefix.ZIP, + * // SPACE=(CYL,(10,10)) + * + * That creates a VTOC entry with an end of file marker (DS1LSTAR) of zero. + * Alas the VTOC end of file marker is only used when the file is opened in + * append mode. When a file is opened in read mode, the "other" end of file + * marker is used, a zero length data block signals end of file when reading. + * With a brand new file which has not been written to yet, it is undefined + * what you read off the disk. In fact you read whatever data was in the same + * disk tracks before the zipfile was allocated. You would be amazed at the + * number of application programmers who still do not understand this. Makes + * for interesting and semi-random errors, GIGO. + * + * Newer versions of SMS will automatically write a zero length block when a + * file is allocated. However not all sites run SMS or they run older levels + * so we cannot rely on that. The only safe thing to do is close the file, + * open in append mode (we already know that the file exists), close it again, + * reopen in read mode and try to read a data block. Opening and closing in + * append mode will write a zero length block where DS1LSTAR points, making + * sure that the VTOC and internal end of file markers are in sync. Then it + * is safe to read data. If we cannot read one byte of data after all that, + * it is a brand new zipfile and must not be read. + */ + if (readable) + { + char c; + fclose(f); + /* append mode */ + if ((f = zfopen(zipfile, "ab")) == NULL) { + ZIPERR(ZE_OPEN, zipfile); + } + fclose(f); + /* read mode again */ + if ((f = zfopen(zipfile, FOPR)) == NULL) { + ZIPERR(ZE_OPEN, zipfile); + } + if (fread(&c, 1, 1, f) != 1) { + /* no actual data */ + readable = 0; + fclose(f); + } + else{ + fseek(f, 0, SEEK_SET); /* at least one byte in zipfile, back to the start */ + } + } +#endif /* MVS */ + + /* ------------------------ */ + /* new file read */ + + + +#ifndef UTIL + if (fix == 2) { + scanzipf_fixnew(); + } + else +#endif + if (readable) + { + /* close file as the new scan opens the splits as needed */ + fclose(f); +# ifndef UTIL + retval = (fix == 2 && !adjust) ? scanzipf_fixnew() : scanzipf_regnew(); +# else + retval = scanzipf_regnew(); +# endif + } + + if (fix != 2 && readable) + { + /* If one or more files, sort by name */ + if (zcount) + { + struct zlist far * far *x; /* pointer into zsort array */ + struct zlist far *z; /* pointer into zfiles linked list */ + extent zl_size = zcount * sizeof(struct zlist far *); + + if (zl_size / sizeof(struct zlist far *) != zcount || + (x = zsort = (struct zlist far **)malloc(zl_size)) == NULL) + return ZE_MEM; + for (z = zfiles; z != NULL; z = z->nxt) + *x++ = z; + qsort((char *)zsort, zcount, sizeof(struct zlist far *), zqcmp); + +#ifdef UNICODE_SUPPORT + /* sort by zuname (local conversion of UTF-8 name) */ + if (zl_size / sizeof(struct zlist far *) != zcount || + (x = zusort = (struct zlist far **)malloc(zl_size)) == NULL) + return ZE_MEM; + for (z = zfiles; z != NULL; z = z->nxt) + *x++ = z; + qsort((char *)zusort, zcount, sizeof(struct zlist far *), zuqcmp); +#endif + } + } + + /* ------------------------ */ + + return retval; +} /* end of function readzipfile() */ + + +int putlocal(z, rewrite) + struct zlist far *z; /* zip entry to write local header for */ + int rewrite; /* did seek to rewrite */ +/* Write a local header described by *z to file *f. Return an error code + in the ZE_ class. */ +{ + /* If any of compressed size (siz), uncompressed size (len), offset(off), or + disk number (dsk) is larger than can fit in the below standard fields then a + Zip64 flag value is stored and a Zip64 extra field is created. + Only siz and len are in the local header while all can be in the central + directory header. + + For the local header if the extra field is created must store both + uncompressed and compressed sizes. + + This assumes that for large entries the compressed size won't need a + Zip64 extra field if the uncompressed size did not. This assumption should + only fail for a large file of nearly totally uncompressable data. + + If streaming stdin in and use_descriptors is set then always create a Zip64 + extra field flagging the data descriptor as being in Zip64 format. This is + needed as don't know if need Zip64 or not when need to set Zip64 flag in + local header. + + If rewrite is set then don't count bytes written for splits + */ + char *block = NULL; /* mem block to write to */ + extent offset = 0; /* offset into block */ + extent blocksize = 0; /* size of block */ +#ifdef UNICODE_SUPPORT + ush nam = z->nam; /* size of name to write to header */ + int use_uname = 0; /* write uname to header */ +#endif +#ifdef ZIP64_SUPPORT + int streaming_in = 0; /* streaming stdin */ + int was_zip64 = 0; + + /* If input is stdin then streaming stdin. No problem with that. + + The problem is updating the local header data in the output once the sizes + and crc are known. If the output is not seekable, then need data descriptors + and also need to assume Zip64 will be needed as don't know yet. Even if the + output is seekable, if the input is streamed need to write the Zip64 extra field + before writing the data or there won't be room for it later if we need it. + */ + streaming_in = (strcmp(z->name, "-") == 0); + + if (!rewrite) { + zip64_entry = 0; + /* initial local header */ + if (z->siz > ZIP_UWORD32_MAX || z->len > ZIP_UWORD32_MAX || + force_zip64 == 1 || (force_zip64 != 0 && streaming_in)) + { + /* assume Zip64 */ + if (force_zip64 == 0) { + zipwarn("Entry too big:", z->oname); + ZIPERR(ZE_BIG, "Large entry support disabled with -fz- but needed"); + } + zip64_entry = 1; /* header of this entry has a field needing Zip64 */ + if (z->ver < ZIP64_MIN_VER) + z->ver = ZIP64_MIN_VER; + was_zip64 = 1; + } + } else { + /* rewrite */ + was_zip64 = zip64_entry; + zip64_entry = 0; + if (z->siz > ZIP_UWORD32_MAX || z->len > ZIP_UWORD32_MAX || + force_zip64 == 1 || (force_zip64 != 0 && streaming_in)) + { + /* Zip64 entry */ + zip64_entry = 1; + } + if (force_zip64 == 0 && zip64_entry) { + /* tried to force into standard entry but needed Zip64 entry */ + zipwarn("Entry too big:", z->oname); + ZIPERR(ZE_BIG, "Large entry support disabled with -fz- but entry needs"); + } + /* Normally for a large archive if the input file is less than 4 GB then + the compressed or stored version should be less than 4 GB. If this + assumption is wrong this catches it. This is a problem even if not + streaming as the Zip64 extra field was not written and now there's no + room for it. */ + if (was_zip64 == 0 && zip64_entry == 1) { + /* guessed wrong and need Zip64 */ + zipwarn("Entry too big:", z->oname); + if (force_zip64 == 0) { + ZIPERR(ZE_BIG, "Compressed/stored entry unexpectedly large - do not use -fz-"); + } else { + ZIPERR(ZE_BIG, "Poor compression resulted in unexpectedly large entry - try -fz"); + } + } + if (zip64_entry) { + /* Zip64 entry still */ + /* this archive needs Zip64 (version 4.5 unzipper) */ + zip64_archive = 1; + if (z->ver < ZIP64_MIN_VER) + z->ver = ZIP64_MIN_VER; + } else { + /* it turns out we do not need Zip64 */ + zip64_entry = 0; + } + if (was_zip64 && zip64_entry != 1) { + z->ver = 20; + } + } + + +#endif /* ZIP64_SUPPORT */ + + /* Instead of writing to the file as we go, to do splits we have to write it + to memory and see if it will fit before writing the entire local header. + If the local header doesn't fit we need to save it for the next disk. + */ + +#ifdef ZIP64_SUPPORT + if (zip64_entry || was_zip64) + /* update extra field */ + add_local_zip64_extra_field( z ); +#endif /* ZIP64_SUPPORT */ + +#ifdef UNICODE_SUPPORT +# if 0 + /* if UTF-8 bit is set on an existing entry, assume it should be */ + /* clear the UTF-8 flag */ + z->flg &= ~UTF8_BIT; + z->lflg &= ~UTF8_BIT; +# endif + + if (z->uname) { + /* need UTF-8 name */ + if (utf8_force || using_utf8) { + z->lflg |= UTF8_BIT; + z->flg |= UTF8_BIT; + } + if (z->flg & UTF8_BIT) { + /* If this flag is set, then restore UTF-8 as path name */ + use_uname = 1; + nam = strlen(z->uname); + } else { + /* use extra field */ + add_Unicode_Path_local_extra_field(z); + } + } else { + /* clear UTF-8 bit as not needed */ + z->flg &= ~UTF8_BIT; + z->lflg &= ~UTF8_BIT; + } +#endif + + append_ulong_to_mem(LOCSIG, &block, &offset, &blocksize); /* local file header signature */ + append_ushort_to_mem(z->ver, &block, &offset, &blocksize); /* version needed to extract */ + append_ushort_to_mem(z->lflg, &block, &offset, &blocksize); /* general purpose bit flag */ + append_ushort_to_mem(z->how, &block, &offset, &blocksize); /* compression method */ + append_ulong_to_mem(z->tim, &block, &offset, &blocksize); /* last mod file date time */ + append_ulong_to_mem(z->crc, &block, &offset, &blocksize); /* crc-32 */ +#ifdef ZIP64_SUPPORT /* zip64 support 09/02/2003 R.Nausedat */ + /* changes 10/5/03 EG */ + if (zip64_entry) { + append_ulong_to_mem(0xFFFFFFFF, &block, &offset, &blocksize); /* compressed size */ + append_ulong_to_mem(0xFFFFFFFF, &block, &offset, &blocksize); /* uncompressed size */ + } else { + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize);/* compressed size */ + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize);/* uncompressed size */ + } +#else + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize); /* compressed size */ + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize); /* uncompressed size */ +#endif +#ifdef UNICODE_SUPPORT + append_ushort_to_mem(nam, &block, &offset, &blocksize); /* file name length */ +#else + append_ushort_to_mem(z->nam, &block, &offset, &blocksize); /* file name length */ +#endif + + append_ushort_to_mem(z->ext, &block, &offset, &blocksize); /* extra field length */ + +#ifdef UNICODE_SUPPORT + if (use_uname) { + /* path is UTF-8 */ + append_string_to_mem(z->uname, nam, &block, &offset, &blocksize); + } else +#endif +#ifdef WIN32_OEM + /* store name in OEM character set in archive */ + if ((z->vem & 0xff00) == 0) + { + char *oem; + + if ((oem = malloc(strlen(z->iname) + 1)) == NULL) + ZIPERR(ZE_MEM, "putlocal oem"); + INTERN_TO_OEM(z->iname, oem); + append_string_to_mem(oem, z->nam, &block, &offset, &blocksize); /* file name */ + free(oem); + } else { + append_string_to_mem(z->iname, z->nam, &block, &offset, &blocksize); /* file name */ + } +#else + append_string_to_mem(z->iname, z->nam, &block, &offset, &blocksize); /* file name */ +#endif + if (z->ext) { + append_string_to_mem(z->extra, z->ext, &block, &offset, &blocksize); /* extra field */ + } + + /* write the header */ + if (rewrite == PUTLOCAL_REWRITE) { + /* use fwrite as seeked back and not extending the archive */ + /* also if split_method 1 write to file with local header */ + if (split_method == 1) { + if (fwrite(block, 1, offset, current_local_file) != offset) { + free(block); + return ZE_TEMP; + } + /* now can close the split if local header on previous split */ + if (current_local_disk != current_disk) { + close_split(current_local_disk, current_local_file, current_local_tempname); + current_local_file = NULL; + free(current_local_tempname); + } + } else { + /* not doing splits */ + if (fwrite(block, 1, offset, y) != offset) { + free(block); + return ZE_TEMP; + } + } + } else { + /* do same if archive not split or split_method 2 with descriptors */ + /* use bfwrite which counts bytes for splits */ + if (bfwrite(block, 1, offset, BFWRITE_LOCALHEADER) != offset) { + free(block); + return ZE_TEMP; + } + } + free(block); + return ZE_OK; +} + +int putextended(z) + struct zlist far *z; /* zip entry to write local header for */ + /* This is the data descriptor. + * Write an extended local header described by *z to file *f. + * Return an error code in the ZE_ class. */ +{ + /* write to mem block then write to file 3/10/2005 */ + char *block = NULL; /* mem block to write to */ + extent offset = 0; /* offset into block */ + extent blocksize = 0; /* size of block */ + + append_ulong_to_mem(EXTLOCSIG, &block, &offset, &blocksize); /* extended local signature */ + append_ulong_to_mem(z->crc, &block, &offset, &blocksize); /* crc-32 */ +#ifdef ZIP64_SUPPORT + if (zip64_entry) { + /* use Zip64 entries */ + append_int64_to_mem(z->siz, &block, &offset, &blocksize); /* compressed size */ + append_int64_to_mem(z->len, &block, &offset, &blocksize); /* uncompressed size */ + /* This is rather klugy as the AppNote handles this poorly. Typically + we don't know at this point if we are writing a Zip64 archive or not, + unless a file has needed Zip64. This is particularly annoying here + when deciding the size of the data descriptor (extended local header) + fields as the appnote says the uncompressed and compressed sizes + should be 8 bytes if the archive is Zip64 and 4 bytes if not. + + One interpretation is the version of the archive is determined from + the Version Needed To Extract field in the Zip64 End Of Central Directory + record and so either an archive should start as Zip64 and write all data + descriptors with 8-byte fields or store everything until all the files + are processed and then write everything to the archive as changing the + sizes of the data descriptors is messy and just not feasible when + streaming to standard output. This is not easily workable and others + use the different interpretation below. + + This was the old thought: + We always write a standard data descriptor. If the file has a large + uncompressed or compressed size we set the field to the max field + value, which we are defining as flagging the field as having a Zip64 + value that doesn't fit. As the CRC happens before the variable size + fields the CRC is still valid and can be used to check the file. We + always use deflate if streaming so signatures should not appear in + the data and all local header signatures should be valid, allowing a + streaming unzip to find entries by local header signatures, if max size + values in the data descriptor sizes ignore them, and extract the file and + check it using the CRC. If not streaming the central directory is available + so just use those values which are correct. + + After discussions with other groups this is the current thinking: + + Apparent industry interpretation for data descriptors: + Data descriptor size is determined for each entry. If the local header + version needed to extract is 45 or higher then the entry can use Zip64 + data descriptors but more checking is needed. If Zip64 extra field is + present then assume data descriptor is Zip64 and local version needed + to extract should be 45 or higher. If standard data descriptor then + local size fields are set to 0 and correct sizes are in standard data descriptor. + If Zip64 data descriptor then local sizes are set to -1, Zip64 extra field + sizes are set to 0, and the correct sizes are in the Zip64 data descriptor. + + So do this: + If an entry is standard and the archive is updatable then seek back and + update the local header. No change. + + If an entry is zip64 and the archive is updatable assume the Zip64 extra + field was created and update it. No change. + + If data descriptors are needed then assume the archive is Zip64. This is + a change and means if ZIP64_SUPPORT is enabled that any non-updatable archive + will be in Zip64 format and use Zip64 data descriptors. This should be + compatible with other zippers that depend on the current (though not perfect) + AppNote description. + + If anyone has some ideas on this I'd like to hear them. + + 3/20/05 EG + + Only assume need Zip64 if the input size is unknown. If the input size is + known we can assume Zip64 if the input is larger than 4 GB and assume not + otherwise. If the output is seekable we still need to create the Zip64 + extra field if the input size is unknown so we can seek back and update it. + 12/28/05 EG + Updated 5/21/06 EG + */ + } else { + /* for encryption */ + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize); /* compressed size */ + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize); /* uncompressed size */ + } +#else + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize); /* compressed size */ + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize); /* uncompressed size */ +#endif + /* write the header */ + if (bfwrite(block, 1, offset, BFWRITE_HEADER) != offset) { + free(block); + return ZE_TEMP; + } + free(block); + return ZE_OK; +} + +int putcentral(z) + struct zlist far *z; /* zip entry to write central header for */ +/* Write a central header described by *z to file *f. Return an error code + in the ZE_ class. */ +/* output now uses bfwrite which writes global y */ +{ + /* If any of compressed size (siz), uncompressed size (len), offset(off), or + disk number (dsk) is larger than can fit in the below standard fields then a + Zip64 flag value is stored and a Zip64 extra field is created. + Only siz and len are in the local header while all are in the central directory + header. + + For the central directory header just store the fields required. All previous fields + must be stored though. So can store none (no extra field), just uncompressed size + (len), len then siz, len then siz then off, or len then siz then off then dsk, in + those orders. 10/6/03 EG + */ + + /* write to mem block then write to file 3/10/2005 EG */ + char *block = NULL; /* mem block to write to */ + extent offset = 0; /* offset into block */ + extent blocksize = 0; /* size of block */ + uzoff_t off = 0; /* offset to start of local header */ + ush nam = z->nam; /* size of name to write to header */ +#ifdef UNICODE_SUPPORT + int use_uname = 0; /* write uname to header */ +#endif + +#ifdef ZIP64_SUPPORT /* zip64 support 09/02/2003 R.Nausedat */ + int iRes; +#endif + +#ifdef UNICODE_SUPPORT + if (z->uname) { + if (utf8_force) { + z->flg |= UTF8_BIT; + } + if (z->flg & UTF8_BIT) { + /* If this flag is set, then restore UTF-8 as path name */ + use_uname = 1; + nam = strlen(z->uname); + } else { + add_Unicode_Path_cen_extra_field(z); + } + } else { + /* clear UTF-8 bit as not needed */ + z->flg &= ~UTF8_BIT; + z->lflg &= ~UTF8_BIT; + } +#endif + + off = z->off; + +#ifdef ZIP64_SUPPORT /* zip64 support 09/02/2003 R.Nausedat */ + if (z->siz > ZIP_UWORD32_MAX || z->len > ZIP_UWORD32_MAX || + z->off > ZIP_UWORD32_MAX || z->dsk > ZIP_UWORD16_MAX || (force_zip64 == 1)) + { + iRes = add_central_zip64_extra_field(z); + if( iRes != ZE_OK ) + return iRes; + } + + append_ulong_to_mem(CENSIG, &block, &offset, &blocksize); /* central file header signature */ + append_ushort_to_mem(z->vem, &block, &offset, &blocksize); /* version made by */ + append_ushort_to_mem(z->ver, &block, &offset, &blocksize); /* version needed to extract */ + append_ushort_to_mem(z->flg, &block, &offset, &blocksize); /* general purpose bit flag */ + append_ushort_to_mem(z->how, &block, &offset, &blocksize); /* compression method */ + append_ulong_to_mem(z->tim, &block, &offset, &blocksize); /* last mod file date time */ + append_ulong_to_mem(z->crc, &block, &offset, &blocksize); /* crc-32 */ + if (z->siz > ZIP_UWORD32_MAX) + { + /* instead of z->siz */ + append_ulong_to_mem(ZIP_UWORD32_MAX, &block, &offset, &blocksize); /* compressed size */ + } + else + { + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize); /* compressed size */ + } + /* if forcing Zip64 just force first ef field */ + if (z->len > ZIP_UWORD32_MAX || (force_zip64 == 1)) + { + /* instead of z->len */ + append_ulong_to_mem(ZIP_UWORD32_MAX, &block, &offset, &blocksize); /* uncompressed size */ + } + else + { + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize); /* uncompressed size */ + } + append_ushort_to_mem(nam, &block, &offset, &blocksize); /* file name length */ + append_ushort_to_mem(z->cext, &block, &offset, &blocksize); /* extra field length */ + append_ushort_to_mem(z->com, &block, &offset, &blocksize); /* file comment length */ + + if (z->dsk > ZIP_UWORD16_MAX) + { + /* instead of z->dsk */ + append_ushort_to_mem((ush)ZIP_UWORD16_MAX, &block, &offset, &blocksize); /* Zip64 flag */ + } + else + { + append_ushort_to_mem((ush)z->dsk, &block, &offset, &blocksize); /* disk number start */ + } + append_ushort_to_mem(z->att, &block, &offset, &blocksize); /* internal file attributes */ + append_ulong_to_mem(z->atx, &block, &offset, &blocksize); /* external file attributes */ + if (off > ZIP_UWORD32_MAX) + { + /* instead of z->off */ + append_ulong_to_mem(ZIP_UWORD32_MAX, &block, &offset, &blocksize); /* Zip64 flag */ + } + else + { + append_ulong_to_mem((ulg)off, &block, &offset, &blocksize); /* offset of local header */ + } + +#else /* !ZIP64_SUPPORT */ + + append_ulong_to_mem(CENSIG, &block, &offset, &blocksize); /* central file header signature */ + append_ushort_to_mem(z->vem, &block, &offset, &blocksize); /* version made by */ + append_ushort_to_mem(z->ver, &block, &offset, &blocksize); /* version needed to extract */ + append_ushort_to_mem(z->flg, &block, &offset, &blocksize); /* general purpose bit flag */ + append_ushort_to_mem(z->how, &block, &offset, &blocksize); /* compression method */ + append_ulong_to_mem(z->tim, &block, &offset, &blocksize); /* last mod file date time */ + append_ulong_to_mem(z->crc, &block, &offset, &blocksize); /* crc-32 */ + append_ulong_to_mem((ulg)z->siz, &block, &offset, &blocksize); /* compressed size */ + append_ulong_to_mem((ulg)z->len, &block, &offset, &blocksize); /* uncompressed size */ + append_ushort_to_mem(nam, &block, &offset, &blocksize); /* file name length */ + append_ushort_to_mem(z->cext, &block, &offset, &blocksize); /* extra field length */ + append_ushort_to_mem(z->com, &block, &offset, &blocksize); /* file comment length */ + append_ushort_to_mem((ush)z->dsk, &block, &offset, &blocksize); /* disk number start */ + append_ushort_to_mem(z->att, &block, &offset, &blocksize); /* internal file attributes */ + append_ulong_to_mem(z->atx, &block, &offset, &blocksize); /* external file attributes */ + append_ulong_to_mem((ulg)off, &block, &offset, &blocksize); /* relative offset of local header */ + +#endif /* ZIP64_SUPPORT */ + +#ifdef EBCDIC + if (z->com) + memtoasc(z->comment, z->comment, z->com); +#endif /* EBCDIC */ + +#ifdef UNICODE_SUPPORT + if (use_uname) { + /* path is UTF-8 */ + append_string_to_mem(z->uname, nam, &block, &offset, &blocksize); + } else +#endif +#ifdef WIN32_OEM + /* store name in OEM character set in archive */ + if ((z->vem & 0xff00) == 0) + { + char *oem; + + if ((oem = malloc(strlen(z->iname) + 1)) == NULL) + ZIPERR(ZE_MEM, "putcentral oem"); + INTERN_TO_OEM(z->iname, oem); + append_string_to_mem(oem, z->nam, &block, &offset, &blocksize); + free(oem); + } else { + append_string_to_mem(z->iname, z->nam, &block, &offset, &blocksize); + } +#else + append_string_to_mem(z->iname, z->nam, &block, &offset, &blocksize); +#endif + + if (z->cext) { + append_string_to_mem(z->cextra, z->cext, &block, &offset, &blocksize); + } + if (z->com) { +#ifdef WIN32_OEM + /* store comment in OEM character set in archive */ + if ((z->vem & 0xff00) == 0) + { + char *oem; + + if ((oem = malloc(strlen(z->comment) + 1)) == NULL) + ZIPERR(ZE_MEM, "putcentral oem comment"); + INTERN_TO_OEM(z->comment, oem); + append_string_to_mem(oem, z->com, &block, &offset, &blocksize); + free(oem); + } else { + append_string_to_mem(z->comment, z->com, &block, &offset, &blocksize); + } +#else + append_string_to_mem(z->comment, z->com, &block, &offset, &blocksize); +#endif + } + + /* write the header */ + if (bfwrite(block, 1, offset, BFWRITE_CENTRALHEADER) != offset) { + free(block); + return ZE_TEMP; + } + free(block); + + return ZE_OK; +} + + +/* Write the end of central directory data to file y. Return an error code + in the ZE_ class. */ + +int putend( OFT( uzoff_t) n, + OFT( uzoff_t) s, + OFT( uzoff_t) c, + OFT( extent) m, + OFT( char *) z + ) +#ifdef NO_PROTO + uzoff_t n; /* number of entries in central directory */ + uzoff_t s; /* size of central directory */ + uzoff_t c; /* offset of central directory */ + extent m; /* length of zip file comment (0 if none) */ + char *z; /* zip file comment if m != 0 */ +#endif /* def NO_PROTO */ +{ +#ifdef ZIP64_SUPPORT /* zip64 support 09/05/2003 R.Nausedat */ + ush vem; /* version made by */ + int iNeedZip64 = 0; + + char *block = NULL; /* mem block to write to */ + extent offset = 0; /* offset into block */ + extent blocksize = 0; /* size of block */ + + /* we have to create a zip64 archive if we have more than 64k - 1 entries, */ + /* if the CD is > 4 GB or if the offset to the CD > 4 GB. even if the CD start */ + /* is < 4 GB and CD start + CD size > 4GB we do not need a zip64 archive since */ + /* the offset entry in the CD tail is still valid. [note that there are other */ + /* reasons for needing a Zip64 archive though, such as an uncompressed */ + /* size > 4 GB for an entry but the entry compresses below 4 GB, so the archive */ + /* is Zip64 but the CD does not need Zip64.] */ + /* order of the zip/zip64 records in a zip64 archive: */ + /* central directory */ + /* zip64 end of central directory record */ + /* zip64 end of central directory locator */ + /* end of central directory record */ + + /* check zip64_archive instead of force_zip64 3/19/05 */ + + zip64_eocd_disk = current_disk; + zip64_eocd_offset = bytes_this_split; + + if( n > ZIP_UWORD16_MAX || s > ZIP_UWORD32_MAX || c > ZIP_UWORD32_MAX || + zip64_archive ) + { + ++iNeedZip64; + /* write zip64 central dir tail: */ + /* */ + /* 4 bytes zip64 end of central dir signature (0x06064b50) */ + append_ulong_to_mem((ulg)ZIP64_CENTRAL_DIR_TAIL_SIG, &block, &offset, &blocksize); + /* 8 bytes size of zip64 end of central directory record */ + /* a fixed size unless the end zip64 extensible data sector is used. - 3/19/05 EG */ + /* also note that AppNote 6.2 creates version 2 of this record for + central directory encryption - 3/19/05 EG */ + append_int64_to_mem((zoff_t)ZIP64_CENTRAL_DIR_TAIL_SIZE, &block, &offset, &blocksize); + + /* 2 bytes version made by */ + vem = OS_CODE + Z_MAJORVER * 10 + Z_MINORVER; + append_ushort_to_mem(vem, &block, &offset, &blocksize); + + /* APPNOTE says that zip64 archives should have at least version 4.5 + in the "version needed to extract" field */ + /* 2 bytes version needed to extract */ + append_ushort_to_mem(ZIP64_MIN_VER, &block, &offset, &blocksize); + + /* 4 bytes number of this disk */ + append_ulong_to_mem(current_disk, &block, &offset, &blocksize); + /* 4 bytes number of the disk with the start of the central directory */ + append_ulong_to_mem(cd_start_disk, &block, &offset, &blocksize); + /* 8 bytes total number of entries in the central directory on this disk */ + append_int64_to_mem(cd_entries_this_disk, &block, &offset, &blocksize); + /* 8 bytes total number of entries in the central directory */ + append_int64_to_mem(n, &block, &offset, &blocksize); + /* 8 bytes size of the central directory */ + append_int64_to_mem(s, &block, &offset, &blocksize); + /* 8 bytes offset of start of central directory with respect to the starting disk number */ + append_int64_to_mem(cd_start_offset, &block, &offset, &blocksize); + /* zip64 extensible data sector (variable size), we don't use it... */ + + /* write zip64 end of central directory locator: */ + /* */ + /* 4 bytes zip64 end of central dir locator signature (0x07064b50) */ + append_ulong_to_mem(ZIP64_CENTRAL_DIR_TAIL_END_SIG, &block, &offset, &blocksize); + /* 4 bytes number of the disk with the start of the zip64 end of central directory */ + append_ulong_to_mem(zip64_eocd_disk, &block, &offset, &blocksize); + /* 8 bytes relative offset of the zip64 end of central directory record, that is */ + /* offset of CD + CD size */ + append_int64_to_mem(zip64_eocd_offset, &block, &offset, &blocksize); + /* PUTLLG(l64Temp, f); */ + /* 4 bytes total number of disks */ + append_ulong_to_mem(current_disk + 1, &block, &offset, &blocksize); + } + + /* end of central dir signature */ + append_ulong_to_mem(ENDSIG, &block, &offset, &blocksize); + /* mv archives to come :) */ + /* for now use n for all */ + /* 2 bytes number of this disk */ + if (current_disk < 0xFFFF) + append_ushort_to_mem((ush)current_disk, &block, &offset, &blocksize); + else + append_ushort_to_mem((ush)0xFFFF, &block, &offset, &blocksize); + /* 2 bytes number of the disk with the start of the central directory */ + if (cd_start_disk == (ulg)-1) + cd_start_disk = 0; + if (cd_start_disk < 0xFFFF) + append_ushort_to_mem((ush)cd_start_disk, &block, &offset, &blocksize); + else + append_ushort_to_mem((ush)0xFFFF, &block, &offset, &blocksize); + /* 2 bytes total number of entries in the central directory on this disk */ + if (cd_entries_this_disk < 0xFFFF) + append_ushort_to_mem((ush)cd_entries_this_disk, &block, &offset, &blocksize); + else + append_ushort_to_mem((ush)0xFFFF, &block, &offset, &blocksize); + /* 2 bytes total number of entries in the central directory */ + if (total_cd_entries < 0xFFFF) + append_ushort_to_mem((ush)total_cd_entries, &block, &offset, &blocksize); + else + append_ushort_to_mem((ush)0xFFFF, &block, &offset, &blocksize); + if( s > ZIP_UWORD32_MAX ) + /* instead of s */ + append_ulong_to_mem(ZIP_UWORD32_MAX, &block, &offset, &blocksize); + else + /* 4 bytes size of the central directory */ + append_ulong_to_mem((ulg)s, &block, &offset, &blocksize); + if(force_zip64 == 1 || cd_start_offset > ZIP_UWORD32_MAX) + /* instead of cd_start_offset */ + append_ulong_to_mem(ZIP_UWORD32_MAX, &block, &offset, &blocksize); + else + /* 4 bytes offset of start of central directory with respect to the starting disk number */ + append_ulong_to_mem((ulg)cd_start_offset, &block, &offset, &blocksize); + +#else /* !ZIP64_SUPPORT */ + char *block = NULL; /* mem block to write to */ + extent offset = 0; /* offset into block */ + extent blocksize = 0; /* size of block */ + + /* end of central dir signature */ + append_ulong_to_mem(ENDSIG, &block, &offset, &blocksize); + /* 2 bytes number of this disk */ + append_ushort_to_mem((ush)current_disk, &block, &offset, &blocksize); + /* 2 bytes number of the disk with the start of the central directory */ + append_ushort_to_mem((ush)cd_start_disk, &block, &offset, &blocksize); + /* 2 bytes total number of entries in the central directory on this disk */ + append_ushort_to_mem((ush)cd_entries_this_disk, &block, &offset, &blocksize); + /* 2 bytes total number of entries in the central directory */ + append_ushort_to_mem((ush)n, &block, &offset, &blocksize); + /* 4 bytes size of the central directory */ + append_ulong_to_mem((ulg)s, &block, &offset, &blocksize); + /* 4 bytes offset of start of central directory with respect to the starting disk number */ + append_ulong_to_mem((ulg)cd_start_offset, &block, &offset, &blocksize); +#endif /* ZIP64_SUPPORT */ + + /* size of comment */ + append_ushort_to_mem((ush)m, &block, &offset, &blocksize); + /* Write the comment, if any */ +#ifdef EBCDIC + memtoasc(z, z, m); +#endif + if (m) { + /* PKWare defines the archive comment to be ASCII only so no OEM conversion */ + append_string_to_mem(z, m, &block, &offset, &blocksize); + } + + /* write the block */ + if (bfwrite(block, 1, offset, BFWRITE_HEADER) != offset) { + free(block); + return ZE_TEMP; + } + free(block); + +#ifdef HANDLE_AMIGA_SFX + if (amiga_sfx_offset && zipbeg /* -J zeroes this */) { + s = zftello(y); + while (s & 3) s++, putc(0, f); /* final marker must be longword aligned */ + PUTLG(0xF2030000 /* 1010 in Motorola byte order */, f); + c = (s - amiga_sfx_offset - 4) / 4; /* size of archive part in longwords */ + if (zfseeko(y, amiga_sfx_offset, SEEK_SET) != 0) + return ZE_TEMP; + c = ((c >> 24) & 0xFF) | ((c >> 8) & 0xFF00) + | ((c & 0xFF00) << 8) | ((c & 0xFF) << 24); /* invert byte order */ + PUTLG(c, y); + zfseeko(y, 0, SEEK_END); /* just in case */ + } +#endif + + return ZE_OK; +} /* end function putend() */ + + + +/* Note: a zip "entry" includes a local header (which includes the file + name), an encryption header if encrypting, the compressed data + and possibly an extended local header. */ + +int zipcopy(z) + struct zlist far *z; /* zip entry to copy */ +/* Copy the zip entry described by *z from in_file to y. Return an + error code in the ZE_ class. Also update tempzn by the number of bytes + copied. */ +/* Now copies to global output file y */ +/* Handle entries that span disks */ +/* If fix == 2, assume in_file is pointing to a local header and fill + in z from local header */ +{ + uzoff_t n; /* holds local header offset */ + ulg e = 0; /* extended local header size */ + ulg start_disk = 0; + uzoff_t start_offset = 0; + char *split_path; + char buf[LOCHEAD + 1]; + struct zlist far *localz; + int r; + + + Trace((stderr, "zipcopy %s\n", z->zname)); + + /* if fix == 2 assume in_file open and pointing at local header */ + if (fix != 2) { + start_disk = z->dsk; + start_offset = z->off; + + /* don't assume reading the right disk */ + + /* if start not on current disk then close current disk */ + if (start_disk != current_in_disk) { + if (in_file) { + fclose(in_file); + in_file = NULL; + } + } + + current_in_disk = start_disk; + + /* disks are archive.z01, archive.z02, ..., archive.zip */ + split_path = get_in_split_path(in_path, current_in_disk); + + if (in_file == NULL) { + while ((in_file = zfopen(split_path, FOPR)) == NULL) { + /* could not open split */ + + if (!noisy) { + ZIPERR(ZE_OPEN, split_path); + } + + /* Ask for directory with split. Updates global in_path */ + r = ask_for_split_read_path(start_disk); + if (r == ZE_ABORT) { + /* user abort */ + return ZE_ABORT; + } else if ((fix == 1 || fix == 2) && r == ZE_FORM) { + /* user asks to skip this disk */ + return ZE_FORM; + } + free(split_path); + split_path = get_in_split_path(in_path, start_disk); + } + } + + if (zfseeko(in_file, start_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive fseek: ", strerror(errno)); + return ZE_READ; + } + } /* fix != 2 */ + + if (fix != 2 && !at_signature(in_file, "PK\03\04")) { + fclose(in_file); + in_file = NULL; + zipwarn("Did not find entry for ", z->iname); + return ZE_FORM; + } + + /* read local header */ + if (fread(buf, LOCHEAD, 1, in_file) != 1) { + int f = ferror(in_file); + zipwarn("reading local entry: ", strerror(errno)); + if (fix != 2) + fclose(in_file); + return f ? ZE_READ : ZE_EOF; + } + + /* Local Header + local file header signature 4 bytes (0x04034b50) + version needed to extract 2 bytes + general purpose bit flag 2 bytes + compression method 2 bytes + last mod file time 2 bytes + last mod file date 2 bytes + crc-32 4 bytes + compressed size 4 bytes + uncompressed size 4 bytes + file name length 2 bytes + extra field length 2 bytes + + file name (variable size) + extra field (variable size) + */ + + if ((localz = (struct zlist far *)farmalloc(sizeof(struct zlist))) == NULL) { + zipwarn("reading entry", ""); + if (fix != 2) + fclose(in_file); + return ZE_MEM; + } + + localz->ver = SH(LOCVER + buf); + localz->lflg = SH(LOCFLG + buf); + localz->how = SH(LOCHOW + buf); + localz->tim = LG(LOCTIM + buf); /* time and date into one long */ + localz->crc = LG(LOCCRC + buf); + localz->nam = SH(LOCNAM + buf); + localz->ext = SH(LOCEXT + buf); + if (fix == 2) { + localz->siz = LG(LOCSIZ + buf); + localz->len = LG(LOCLEN + buf); + } + + if (fix == 2) { + /* Do some sanity checks to make reasonably sure this is a local header */ + ush os = localz->ver >> 8; + ush pkver = localz->ver - os; + + /* OS - currently 0 - 18 (AppNote 6.3) and 30 (ATHEOS) */ + if (os > 40) { + sprintf(errbuf, "Illegal host system mapping in local header: %d", os); + zipwarn(errbuf, ""); + zipwarn("Skipping: ", z->iname); + return ZE_FORM; + } + /* PK Version - currently 10 - 62 (AppNote 6.2.2) */ + /* If PKZip central directory encryption is used (62), the local header + values could be masked values. Specifically, as of AppNote 6.2.2 + the time, crc-32, and uncompressed file size are masked and the + file name is also replaced with a hex entry count. Should + still be able to recover the entries, but they may be unreadable + without the 62 support fields. */ + if (pkver > 100) { + sprintf(errbuf, "Illegal PK version mapping in local header: %d", pkver); + zipwarn(errbuf, ""); + zipwarn("Skipping: ", z->iname); + return ZE_FORM; + } + /* Currently compression method is defined as 0 - 19 and 98 (AppNote 6.3) */ + /* We can still copy an entry we can't read, but something over 200 is + probably illegal */ + if (localz->how > 200) { + sprintf(errbuf, "Unrecognized compression method in local header: %d", localz->how); + zipwarn(errbuf, ""); + zipwarn("Skipping: ", z->iname); + return ZE_FORM; + } + + /* It's hard to make guesses on the other fields. Suggestions welcome. */ + } + + /* Initialize all fields pointing to malloced data to NULL */ + localz->zname = localz->name = localz->iname = localz->extra = NULL; + localz->oname = NULL; +#ifdef UNICODE_SUPPORT + localz->uname = NULL; +#endif + + /* Read file name, extra field and comment field */ + if ((localz->iname = malloc(localz->nam+1)) == NULL || + (localz->ext && (localz->extra = malloc(localz->ext)) == NULL)) + return ZE_MEM; + if (fread(localz->iname, localz->nam, 1, in_file) != 1 || + (localz->ext && fread(localz->extra, localz->ext, 1, in_file) != 1)) + return ferror(in_file) ? ZE_READ : ZE_EOF; + localz->iname[localz->nam] = '\0'; /* terminate name */ + if ((localz->name = malloc(localz->nam+1)) == NULL) + return ZE_MEM; + strcpy(localz->name, localz->iname); + +#ifdef ZIP64_SUPPORT + zip64_entry = adjust_zip_local_entry(localz); +#endif + + localz->vem = 0; + if (fix != 2) { + /* Need vem to determine if iname is Win32 OEM name */ + localz->vem = z->vem; + +#ifdef UNICODE_SUPPORT + if (unicode_mismatch != 3) { + if (z->flg & UTF8_BIT) { + char *iname; + /* path is UTF-8 */ + localz->uname = localz->iname; + iname = utf8_to_local_string(localz->uname); + if (iname == NULL) { + /* a bad UTF-8 character in name likely - go with (probably messed up) uname */ + if ((localz->iname = malloc(strlen(localz->uname) + 1)) == NULL) { + return ZE_MEM; + } + strcpy(localz->iname, localz->uname); + } else { + /* go with local character set iname */ + localz->iname = iname; + } + } else { + /* check for UTF-8 path extra field */ + read_Unicode_Path_local_entry(localz); + } + } +#endif + +#ifdef WIN32_OEM + /* If fix == 2 and reading local headers first, vem is not in the local + header so we don't know when to do OEM translation, as the ver field + is set to MSDOS (0) by all unless something specific is needed. + However, if local header has a Unicode path extra field, we can get + the real file name from there. */ + if ((z->vem & 0xff00) == 0) + /* assume archive name is OEM if from DOS */ + oem_to_local_string(localz->iname, localz->iname); +#endif + } + + if (fix == 2) { +# ifdef WIN32 +# ifdef UNICODE_SUPPORT + localz->namew = NULL; + localz->inamew = NULL; + localz->znamew = NULL; + z->namew = NULL; + z->inamew = NULL; + z->znamew = NULL; +# endif +# endif + /* set z from localz */ + z->flg = localz->lflg; + z->len = localz->len; + z->siz = localz->siz; + + } else { + /* Compare localz to z */ + if (localz->ver != z->ver) { + zipwarn("Local Version Needed To Extract does not match CD: ", z->iname); + } + if (localz->lflg != z->flg) { + zipwarn("Local Entry Flag does not match CD: ", z->iname); + } + if (!(z->flg & 8)) { + if (localz->crc != z->crc) { + zipwarn("Local Entry CRC does not match CD: ", z->iname); + } + } + if (fix != 3 && strcmp(localz->iname, z->iname) != 0) { + zipwarn("Local Entry name does not match CD: ", z->iname); + } + + /* as copying get uncompressed and compressed sizes from central directory */ + localz->len = z->len; + localz->siz = z->siz; + } + +#if 0 + if (fix > 1) { + if (zfseeko(in_file, z->off + n, SEEK_SET)) /* seek to compressed data */ + return ferror(in_file) ? ZE_READ : ZE_EOF; + + if (fix > 2) { + /* Update length of entry's name, it may have been changed. This is + needed to support the ZipNote ability to rename archive entries. */ + z->nam = strlen(z->iname); + n = (uzoff_t)((LOCHEAD) + (ulg)z->nam + (ulg)z->ext); + } + + /* do not trust the old compressed size */ + if (putlocal(z, PUTLOCAL_WRITE) != ZE_OK) + return ZE_TEMP; + + z->off = tempzn; + tempzn += n; + n = z->siz; + } else { + if (zfseeko(in_file, z->off, SEEK_SET)) /* seek to local header */ + return ferror(in_file) ? ZE_READ : ZE_EOF; + + z->off = tempzn; + n += z->siz; + } +#endif + + /* from zipnote */ + if (fix == 3) { + /* Update length of entry's name, as it may have been changed. This is + needed to support the ZipNote ability to rename archive entries. */ + localz->nam = z->nam = strlen(z->iname); + /* update local name */ + free(localz->iname); + if ((localz->iname = malloc(strlen(z->iname) + 1)) == NULL) { + zipwarn("out of memory in zipcopy", ""); + return ZE_MEM; + } + strcpy(localz->iname, z->iname); + } + + /* update disk and offset */ + z->dsk = current_disk; + z->off = bytes_this_split; + + /* copy the compressed data and the extended local header if there is one */ + + /* copy the compressed data. We recreate the local header as the local + header can't be split and putlocal ensures it won't. Also, since we + use siz and len from the central directory, we don't need the extended + local header if there is one, unless the file is encrypted as then the + extended header is used to indicate crypt head uses file time instead + of crc as the password check. + + If fix = 2 then we don't have the central directory yet so keep + any data descriptors. */ + + if (fix != 2 && !(z->flg & 1)) { + /* Not encrypted */ + localz->flg = z->flg &= ~8; + z->lflg = localz->lflg &= ~8; + } + + e = 0; + if (z->lflg & 8) { +#ifdef ZIP64_SUPPORT + if (zip64_entry) + e = 24; + else +#endif + e = 16; + } + /* 4 is signature */ + n = 4 + (uzoff_t)((LOCHEAD) + (ulg)(localz->nam) + (ulg)(localz->ext)); + + n += e + z->siz; + tempzn += n; + + /* Output name */ + if (fix == 2) { + if ((z->oname = malloc(strlen(localz->iname) + 1)) == NULL) { + return ZE_MEM; + } + strcpy(z->oname, localz->iname); +#ifndef UTIL +# ifdef WIN32 + /* Win9x console always uses OEM character coding, and + WinNT console is set to OEM charset by default, too */ + _INTERN_OEM(z->oname); +# endif +#endif + sprintf(errbuf, " copying: %s ", z->oname); + zipmessage_nl(errbuf, 0); + } + + if (fix == 2) + z->crc = localz->crc; + else + localz->crc = z->crc; + + if (putlocal(localz, PUTLOCAL_WRITE) != ZE_OK) + return ZE_TEMP; + + /* + if (zfseeko(in_file, start_offset, SEEK_SET) != 0) { + fclose(in_file); + in_file = NULL; + zipwarn("reading archive fseek: ", strerror(errno)); + return ZE_READ; + } + */ + + /* copy the data */ + if (fix == 2 && localz->lflg & 8) + /* read to data descriptor */ + r = bfcopy((uzoff_t) -2); + else + r = bfcopy(localz->siz); + + if (r == ZE_ABORT) { + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + ZIPERR(ZE_ABORT, "Could not find split"); + } + + if (r == ZE_EOF || skip_this_disk) { + /* missing disk */ + zipwarn("aborting: ", z->oname); + + if (r == ZE_OK) + r = ZE_FORM; + + if (fix == 2) { +#ifdef DEBUG + zoff_t here = zftello(y); +#endif + + /* fix == 2 skips right to next disk */ + skip_this_disk = 0; + + /* seek back in output to start of this entry so can overwrite */ + if (zfseeko(y, current_local_offset, SEEK_SET) != 0) { + ZIPERR(ZE_WRITE, "seek failed on output file"); + } + bytes_this_split = current_local_offset; + tempzn = current_local_offset; + } + + /* tell scan to skip this entry */ + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + return r; + } + + if (fix == 2 && z->flg & 8) { + /* this entry should have a data descriptor */ + /* only -FF needs to read the descriptor as other modes + rely on the central directory */ + if (des_good) { + /* found an apparently good data descriptor */ + localz->crc = des_crc; + localz->siz = des_csize; + localz->len = des_usize; + } else { + /* no end to this entry found */ + zipwarn("no end of stream entry found: ", z->oname); + zipwarn("rewinding and scanning for later entries", ""); + + /* seek back in output to start of this entry so can overwrite */ + if (zfseeko(y, current_local_offset, SEEK_SET) != 0){ + + } + + /* tell scan to skip this entry */ + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + return ZE_FORM; + } + } + + if (z->flg & 8) { + putextended(localz); + } + + /* now can close the split if local header on previous split */ + if (split_method == 1 && current_local_disk != current_disk) { + close_split(current_local_disk, current_local_file, current_local_tempname); + current_local_file = NULL; + free(current_local_tempname); + } + + /* update local header and close start split */ + /* to use this need to seek back, do this, then come back + if (putlocal(localz, PUTLOCAL_REWRITE) != ZE_OK) + r = ZE_TEMP; + */ + + if (fix == 2) { + z->ver = localz->ver; + z->how = localz->how; + z->tim = localz->tim; + z->crc = localz->crc; + z->lflg = localz->lflg; + z->flg = localz->lflg; + z->len = localz->len; + z->siz = localz->siz; + z->nam = localz->nam; + z->ext = localz->ext; + z->extra = localz->extra; + /* copy local extra fields to central directory for now */ + z->cext = localz->ext; + z->cextra = NULL; + if (localz->ext) { + if ((z->cextra = malloc(localz->ext + 1)) == NULL) { + return ZE_MEM; + } + strcpy(z->cextra, localz->extra); + } + z->com = 0; + z->att = 0; + z->atx = 0; + z->name = localz->name; + z->iname = localz->iname; +#ifdef UNICODE_SUPPORT + z->uname = localz->uname; +#endif + if ((z->zname = malloc(localz->nam + 1)) == NULL) { + return ZE_MEM; + } + strcpy(z->zname, z->iname); + } else { + if (localz->ext) free(localz->extra); + if (localz->nam) free(localz->iname); + if (localz->nam) free(localz->name); +#ifdef UNICODE_SUPPORT + if (localz->uname) free(localz->uname); +#endif + free(localz); + } + + if (fix == 2) { + sprintf(errbuf, " (%s bytes)", zip_fzofft(z->siz, NULL, "u")); + zipmessage_nl(errbuf, 1); + + if (r == ZE_READ) { + zipwarn("entry truncated: ", z->oname); + sprintf(errbuf, "expected compressed/stored size %s, actual %s", + zip_fzofft(localz->siz, NULL, "u"), zip_fzofft(bytes_this_entry, NULL, "u")); + zipwarn(errbuf, ""); + } + } + + return r; +} + + + +#ifndef UTIL + +#ifdef USE_EF_UT_TIME + +local int ef_scan_ut_time(ef_buf, ef_len, ef_is_cent, z_utim) +char *ef_buf; /* buffer containing extra field */ +extent ef_len; /* total length of extra field */ +int ef_is_cent; /* flag indicating "is central extra field" */ +iztimes *z_utim; /* return storage: atime, mtime, ctime */ +/* This function scans the extra field for EF_TIME or EF_IZUNIX blocks + * containing Unix style time_t (GMT) values for the entry's access, creation + * and modification time. + * If a valid block is found, all time stamps are copied to the iztimes + * structure. + * The presence of an EF_TIME or EF_IZUNIX2 block results in ignoring + * all data from probably present obsolete EF_IZUNIX blocks. + * If multiple blocks of the same type are found, only the information from + * the last block is used. + * The return value is the EF_TIME Flags field (simulated in case of an + * EF_IZUNIX block) or 0 in case of failure. + */ +{ + int flags = 0; + unsigned eb_id; + extent eb_len; + int have_new_type_eb = FALSE; + + if (ef_len == 0 || ef_buf == NULL) + return 0; + + Trace((stderr,"\nef_scan_ut_time: scanning extra field of length %u\n", + (unsigned)ef_len)); + while (ef_len >= EB_HEADSIZE) { + eb_id = SH(EB_ID + ef_buf); + eb_len = SH(EB_LEN + ef_buf); + + if (eb_len > (ef_len - EB_HEADSIZE)) { + /* Discovered some extra field inconsistency! */ + Trace((stderr,"ef_scan_ut_time: block length %u > rest ef_size %u\n", + (unsigned)eb_len, (unsigned)(ef_len - EB_HEADSIZE))); + break; + } + + switch (eb_id) { + case EF_TIME: + flags &= ~0x00ff; /* ignore previous IZUNIX or EF_TIME fields */ + have_new_type_eb = TRUE; + if ( eb_len >= EB_UT_MINLEN && z_utim != NULL) { + unsigned eb_idx = EB_UT_TIME1; + Trace((stderr,"ef_scan_ut_time: Found TIME extra field\n")); + flags |= (ef_buf[EB_HEADSIZE+EB_UT_FLAGS] & 0x00ff); + if ((flags & EB_UT_FL_MTIME)) { + if ((eb_idx+4) <= eb_len) { + z_utim->mtime = LG((EB_HEADSIZE+eb_idx) + ef_buf); + eb_idx += 4; + Trace((stderr," Unix EF modtime = %ld\n", z_utim->mtime)); + } else { + flags &= ~EB_UT_FL_MTIME; + Trace((stderr," Unix EF truncated, no modtime\n")); + } + } + if (ef_is_cent) { + break; /* central version of TIME field ends here */ + } + if (flags & EB_UT_FL_ATIME) { + if ((eb_idx+4) <= eb_len) { + z_utim->atime = LG((EB_HEADSIZE+eb_idx) + ef_buf); + eb_idx += 4; + Trace((stderr," Unix EF acctime = %ld\n", z_utim->atime)); + } else { + flags &= ~EB_UT_FL_ATIME; + } + } + if (flags & EB_UT_FL_CTIME) { + if ((eb_idx+4) <= eb_len) { + z_utim->ctime = LG((EB_HEADSIZE+eb_idx) + ef_buf); + /* eb_idx += 4; */ /* superfluous for now ... */ + Trace((stderr," Unix EF cretime = %ld\n", z_utim->ctime)); + } else { + flags &= ~EB_UT_FL_CTIME; + } + } + } + break; + + case EF_IZUNIX2: + if (!have_new_type_eb) { + flags &= ~0x00ff; /* ignore any previous IZUNIX field */ + have_new_type_eb = TRUE; + } + break; + + case EF_IZUNIX: + if (eb_len >= EB_UX_MINLEN) { + Trace((stderr,"ef_scan_ut_time: Found IZUNIX extra field\n")); + if (have_new_type_eb) { + break; /* Ignore IZUNIX extra field block ! */ + } + z_utim->atime = LG((EB_HEADSIZE+EB_UX_ATIME) + ef_buf); + z_utim->mtime = LG((EB_HEADSIZE+EB_UX_MTIME) + ef_buf); + Trace((stderr," Unix EF access time = %ld\n",z_utim->atime)); + Trace((stderr," Unix EF modif. time = %ld\n",z_utim->mtime)); + flags |= (EB_UT_FL_MTIME | EB_UT_FL_ATIME); /* signal success */ + } + break; + + case EF_THEOS: +/* printf("Not implemented yet\n"); */ + break; + + default: + break; + } + /* Skip this extra field block */ + ef_buf += (eb_len + EB_HEADSIZE); + ef_len -= (eb_len + EB_HEADSIZE); + } + + return flags; +} + +int get_ef_ut_ztime(z, z_utim) +struct zlist far *z; +iztimes *z_utim; +{ + int r; + +#ifdef IZ_CHECK_TZ + if (!zp_tz_is_valid) return 0; +#endif + + /* First, scan local extra field. */ + r = ef_scan_ut_time(z->extra, z->ext, FALSE, z_utim); + + /* If this was not successful, try central extra field, but only if + it is really different. */ + if (!r && z->cext > 0 && z->cextra != z->extra) + r = ef_scan_ut_time(z->cextra, z->cext, TRUE, z_utim); + + return r; +} + +#endif /* USE_EF_UT_TIME */ + + +local void cutpath(p, delim) +char *p; /* path string */ +int delim; /* path component separator char */ +/* Cut the last path component off the name *p in place. + * This should work on both internal and external names. + */ +{ + char *r; /* pointer to last path delimiter */ + +#ifdef VMS /* change [w.x.y]z to [w.x]y.DIR */ + if ((r = MBSRCHR(p, ']')) != NULL) + { + *r = 0; + if ((r = MBSRCHR(p, '.')) != NULL) + { + *r = ']'; + strcat(r, ".DIR;1"); /* this assumes a little padding--see PAD */ + } else { + *p = 0; + } + } else { + if ((r = MBSRCHR(p, delim)) != NULL) + *r = 0; + else + *p = 0; + } +#else /* !VMS */ + if ((r = MBSRCHR(p, delim)) != NULL) + *r = 0; + else + *p = 0; +#endif /* ?VMS */ +} + +int trash() +/* Delete the compressed files and the directories that contained the deleted + files, if empty. Return an error code in the ZE_ class. Failure of + destroy() or deletedir() is ignored. */ +{ + extent i; /* counter on deleted names */ + extent n; /* number of directories to delete */ + struct zlist far **s; /* table of zip entries to handle, sorted */ + struct zlist far *z; /* current zip entry */ + + /* Delete marked names and count directories */ + n = 0; + for (z = zfiles; z != NULL; z = z->nxt) + if (z->mark == 1 || z->trash) + { + z->mark = 1; + if (z->iname[z->nam - 1] != (char)0x2f) { /* don't unlink directory */ + if (verbose) + fprintf(mesg, "zip diagnostic: deleting file %s\n", z->name); + if (destroy(z->name)) { + zipwarn("error deleting ", z->name); + } + /* Try to delete all paths that lead up to marked names. This is + * necessary only with the -D option. + */ + if (!dirnames) { + cutpath(z->name, '/'); /* XXX wrong ??? */ + /* Below apparently does not work for Russian OEM but + '/' should be same as 0x2f for ascii and most ports so + changed it. Did not trace through the mappings but + maybe 0x2F is mapped differently on OEM_RUSS - EG 2/28/2003 */ + /* CS, 5/14/2005: iname is the byte array read from and written + to the zip archive; it MUST be ASCII (compatible)!!! + If something goes wrong with OEM_RUSS, there is a charcode + mapping error between external name (z->name) and iname somewhere + in the in2ex & ex2in code. The charcode translation should be + checked. + This code line is changed back to the original code. */ + /* CS, 6/12/2005: What is handled here is the difference between + ASCII charsets and non-ASCII charsets like the family of EBCDIC + charsets. On these systems, the slash character '/' is not coded + as 0x2f but as 0x61 (the ASCII 'a'). The iname struct member holds + the name as stored in the Zip file, which are ASCII or translated + into ASCII for new entries, whereas the "name" struct member hold + the external name, coded in the native charset of the system + (EBCDIC on EBCDIC systems) */ + /* cutpath(z->iname, '/'); */ /* QQQ ??? */ + cutpath(z->iname, 0x2f); /* 0x2f = ascii['/'] */ + z->nam = strlen(z->iname); + if (z->nam > 0) { + z->iname[z->nam - 1] = (char)0x2f; + z->iname[z->nam++] = '\0'; + } + if (z->nam > 0) n++; + } + } else { + n++; + } + } + + /* Construct the list of all marked directories. Some may be duplicated + * if -D was used. + */ + if (n) + { + if ((s = (struct zlist far **)malloc(n*sizeof(struct zlist far *))) == + NULL) + return ZE_MEM; + n = 0; + for (z = zfiles; z != NULL; z = z->nxt) { + if (z->mark && z->nam > 0 && z->iname[z->nam - 1] == (char)0x2f /* '/' */ + && (n == 0 || strcmp(z->name, s[n-1]->name) != 0)) { + s[n++] = z; + } + } + /* Sort the files in reverse order to get subdirectories first. + * To avoid problems with strange naming conventions as in VMS, + * we sort on the internal names, so x/y/z will always be removed + * before x/y. On VMS, x/y/z > x/y but [x.y.z] < [x.y] + */ + qsort((char *)s, n, sizeof(struct zlist far *), rqcmp); + + for (i = 0; i < n; i++) { + char *p = s[i]->name; + if (*p == '\0') continue; + if (p[strlen(p) - 1] == '/') { /* keep VMS [x.y]z.dir;1 intact */ + p[strlen(p) - 1] = '\0'; + } + if (i == 0 || strcmp(s[i]->name, s[i-1]->name) != 0) { + if (verbose) { + fprintf(mesg, "deleting directory %s (if empty) \n", + s[i]->name); + } + deletedir(s[i]->name); + } + } + free((zvoid *)s); + } + return ZE_OK; +} + +#endif /* !UTIL */ diff --git a/third_party/infozip/zip/zipnote.c b/third_party/infozip/zip/zipnote.c new file mode 100644 index 000000000..9835ccd1d --- /dev/null +++ b/third_party/infozip/zip/zipnote.c @@ -0,0 +1,705 @@ +/* clang-format off */ +/* + zipnote.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * zipnote.c by Mark Adler. + */ +#define __ZIPNOTE_C + +#ifndef UTIL +#define UTIL +#endif +#include "third_party/infozip/zip/zip.h" +#define DEFCPYRT /* main module: enable copyright string defines! */ +#include "third_party/infozip/zip/revision.h" +#include "libc/calls/calls.h" +#include "libc/fmt/fmt.h" +#include "libc/fmt/conv.h" +#include "libc/alg/alg.h" +#include "libc/log/log.h" +#include "libc/stdio/temp.h" + +/* Calculate size of static line buffer used in write (-w) mode. */ +#define WRBUFSIZ 2047 +/* The line buffer size should be at least as large as FNMAX. */ +#if FNMAX > WRBUFSIZ +# undef WRBUFSIZ +# define WRBUFSIZ FNMAX +#endif + +/* Character to mark zip entry names in the comment file */ +#define MARK '@' +#define MARKE " (comment above this line)" +#define MARKZ " (zip file comment below this line)" + +/* Temporary zip file pointer */ +local FILE *tempzf; + + +/* Local functions */ +local void handler OF((int)); +local void license OF((void)); +local void help OF((void)); +local void version_info OF((void)); +local void putclean OF((char *, extent)); +/* getline name conflicts with GNU getline() function */ +local char *zgetline OF((char *, extent)); +local int catalloc OF((char * far *, char *)); +int main OF((int, char **)); + +/* keep compiler happy until implement long options - 11/4/2003 EG */ +struct option_struct far options[] = { + /* short longopt value_type negatable ID name */ + {"h", "help", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + /* the end of the list */ + {NULL, NULL, o_NO_VALUE, o_NOT_NEGATABLE, 0, NULL} /* end has option_ID = 0 */ + }; + +#ifdef MACOS +#define ziperr(c, h) zipnoteerr(c, h) +#define zipwarn(a, b) zipnotewarn(a, b) + +void zipnoteerr(int c, ZCONST char *h); +void zipnotewarn(ZCONST char *a, ZCONST char *b); +#endif + +#ifdef QDOS +#define exit(p1) QDOSexit() +#endif + +int set_filetype(out_path) + char *out_path; +{ +#ifdef __BEOS__ + /* Set the filetype of the zipfile to "application/zip" */ + setfiletype( out_path, "application/zip" ); +#endif + +#ifdef __ATHEOS__ + /* Set the filetype of the zipfile to "application/x-zip" */ + setfiletype(out_path, "application/x-zip"); +#endif + +#ifdef MACOS + /* Set the Creator/Type of the zipfile to 'IZip' and 'ZIP ' */ + setfiletype(out_path, 'IZip', 'ZIP '); +#endif + +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(out_path, 0xDDC); +#endif + return ZE_OK; +} + +/* rename a split + * A split has a tempfile name until it is closed, then + * here rename it as out_path the final name for the split. + */ +int rename_split(temp_name, out_path) + char *temp_name; + char *out_path; +{ + int r; + /* Replace old zip file with new zip file, leaving only the new one */ + if ((r = replace(out_path, temp_name)) != ZE_OK) + { + zipwarn("new zip file left as: ", temp_name); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing split file"); + } + if (zip_attributes) { + setfileattr(out_path, zip_attributes); + } + return ZE_OK; +} + +void zipmessage_nl(a, nl) +ZCONST char *a; /* message string to output */ +int nl; /* 1 = add nl to end */ +/* If nl false, print a message to mesg without new line. + If nl true, print and add new line. If logfile is + open then also write message to log file. */ +{ + if (noisy) { + fprintf(mesg, "%s", a); + if (nl) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } else { + mesg_line_started = 1; + } + fflush(mesg); + } +} + +void zipmessage(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a message to mesg and flush. Also write to log file if + open. Write new line first if current line has output already. */ +{ + if (noisy) { + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "%s%s\n", a, b); + mesg_line_started = 0; + fflush(mesg); + } +} + +void ziperr(c, h) +int c; /* error code from the ZE_ class */ +ZCONST char *h; /* message about how it happened */ +/* Issue a message for the error, clean up files and memory, and exit. */ +{ + if (PERR(c)) + perror("zipnote error"); + fprintf(mesg, "zipnote error: %s (%s)\n", ZIPERRORS(c), h); + if (tempzf != NULL) + fclose(tempzf); + if (tempzip != NULL) + { + destroy(tempzip); + free((zvoid *)tempzip); + } + if (zipfile != NULL) + free((zvoid *)zipfile); + EXIT(c); +} + + +local void handler(s) +int s; /* signal number (ignored) */ +/* Upon getting a user interrupt, abort cleanly using ziperr(). */ +{ +#ifndef MSDOS + putc('\n', mesg); +#endif /* !MSDOS */ + ziperr(ZE_ABORT, "aborting"); + s++; /* keep some compilers happy */ +} + + +void zipwarn(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a warning message to mesg (usually stderr) and return. */ +{ + fprintf(mesg, "zipnote warning: %s%s\n", a, b); +} + + +local void license() +/* Print license information to stdout. */ +{ + extent i; /* counter for copyright array */ + + for (i = 0; i < sizeof(swlicense)/sizeof(char *); i++) + puts(swlicense[i]); +} + + +local void help() +/* Print help (along with license info) to stdout. */ +{ + extent i; /* counter for help array */ + + /* help array */ + static ZCONST char *text[] = { +"", +"ZipNote %s (%s)", +#ifdef VM_CMS +"Usage: zipnote [-w] [-q] [-b fm] zipfile", +#else +"Usage: zipnote [-w] [-q] [-b path] zipfile", +#endif +" the default action is to write the comments in zipfile to stdout", +" -w write the zipfile comments from stdin", +#ifdef VM_CMS +" -b use \"fm\" as the filemode for the temporary zip file", +#else +" -b use \"path\" for the temporary zip file", +#endif +" -q quieter operation, suppress some informational messages", +" -h show this help -v show version info -L show software license", +"", +"Example:", +#ifdef VMS +" define/user sys$output foo.tmp", +" zipnote foo.zip", +" edit foo.tmp", +" ... then you edit the comments, save, and exit ...", +" define/user sys$input foo.tmp", +" zipnote -w foo.zip", +#else +#ifdef RISCOS +" zipnote foo/zip > foo/tmp", +" foo/tmp", +" ... then you edit the comments, save, and exit ...", +" zipnote -w foo/zip < foo/tmp", +#else +#ifdef VM_CMS +" zipnote foo.zip > foo.tmp", +" xedit foo tmp", +" ... then you edit the comments, save, and exit ...", +" zipnote -w foo.zip < foo.tmp", +#else +" zipnote foo.zip > foo.tmp", +" ed foo.tmp", +" ... then you edit the comments, save, and exit ...", +" zipnote -w foo.zip < foo.tmp", +#endif /* VM_CMS */ +#endif /* RISCOS */ +#endif /* VMS */ +"", +" \"@ name\" can be followed by an \"@=newname\" line to change the name" + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) { + printf(copyright[i], "zipnote"); + putchar('\n'); + } + for (i = 0; i < sizeof(text)/sizeof(char *); i++) + { + printf(text[i], VERSION, REVDATE); + putchar('\n'); + } +} + +/* + * XXX put this in version.c + */ + +local void version_info() +/* Print verbose info about program version and compile time options + to stdout. */ +{ + extent i; /* counter in text arrays */ + + /* Options info array */ + static ZCONST char *comp_opts[] = { +#ifdef DEBUG + "DEBUG", +#endif + NULL + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) + { + printf(copyright[i], "zipnote"); + putchar('\n'); + } + + for (i = 0; i < sizeof(versinfolines)/sizeof(char *); i++) + { + printf(versinfolines[i], "ZipNote", VERSION, REVDATE); + putchar('\n'); + } + + version_local(); + + puts("ZipNote special compilation options:"); + for (i = 0; (int)i < (int)(sizeof(comp_opts)/sizeof(char *) - 1); i++) + { + printf("\t%s\n",comp_opts[i]); + } + if (i == 0) + puts("\t[none]"); +} + + +local void putclean(s, n) +char *s; /* string to write to stdout */ +extent n; /* length of string */ +/* Write the string s to stdout, filtering out control characters that are + not tab or newline (mainly to remove carriage returns), and prefix MARK's + and backslashes with a backslash. Also, terminate with a newline if + needed. */ +{ + int c; /* next character in string */ + int e; /* last character written */ + + e = '\n'; /* if empty, write nothing */ + while (n--) + { + c = *(uch *)s++; + if (c == MARK || c == '\\') + putchar('\\'); + if (c >= ' ' || c == '\t' || c == '\n') + { e=c; putchar(e); } + } + if (e != '\n') + putchar('\n'); +} + + +local char *zgetline(buf, size) +char *buf; +extent size; +/* Read a line of text from stdin into string buffer 'buf' of size 'size'. + In case of buffer overflow or EOF, a NULL pointer is returned. */ +{ + char *line; + unsigned len; + + line = fgets(buf, size, stdin); + if (line != NULL && (len = strlen(line)) > 0) { + if (len == size-1 && line[len-1] != '\n') { + /* buffer is full and record delimiter not seen -> overflow */ + line = NULL; + } else { + /* delete trailing record delimiter */ + if (line[len-1] == '\n') line[len-1] = '\0'; + } + } + return line; +} + + +local int catalloc(a, s) +char * far *a; /* pointer to a pointer to a malloc'ed string */ +char *s; /* string to concatenate on a */ +/* Concatentate the string s to the malloc'ed string pointed to by a. + Preprocess s by removing backslash escape characters. */ +{ + char *p; /* temporary pointer */ + char *q; /* temporary pointer */ + + for (p = q = s; *q; *p++ = *q++) + if (*q == '\\' && *(q+1)) + q++; + *p = 0; + if ((p = malloc(strlen(*a) + strlen(s) + 3)) == NULL) + return ZE_MEM; + strcat(strcat(strcpy(p, *a), **a ? "\r\n" : ""), s); + free((zvoid *)*a); + *a = p; + return ZE_OK; +} + + +#ifndef USE_ZIPNOTEMAIN +int main(argc, argv) +#else +int zipnotemain(argc, argv) +#endif +int argc; /* number of tokens in command line */ +char **argv; /* command line tokens */ +/* Write the comments in the zipfile to stdout, or read them from stdin. */ +{ + char abf[WRBUFSIZ+1]; /* input line buffer */ + char *a; /* pointer to line buffer or NULL */ + zoff_t c; /* start of central directory */ + int k; /* next argument type */ + char *q; /* steps through option arguments */ + int r; /* arg counter, temporary variable */ + zoff_t s; /* length of central directory */ + int t; /* attributes of zip file */ + int w; /* true if updating zip file from stdin */ + FILE *x; /* input file for testing if can write it */ + struct zlist far *z; /* steps through zfiles linked list */ + +#ifdef THEOS + setlocale(LC_CTYPE, "I"); +#endif + +#ifdef UNICODE_SUPPORT +# ifdef UNIX + /* For Unix, set the locale to UTF-8. Any UTF-8 locale is + OK and they should all be the same. This allows seeing, + writing, and displaying (if the fonts are loaded) all + characters in UTF-8. */ + { + char *loc; + + /* + loc = setlocale(LC_CTYPE, NULL); + printf(" Initial language locale = '%s'\n", loc); + */ + + loc = setlocale(LC_CTYPE, "en_US.UTF-8"); + + /* + printf("langinfo %s\n", nl_langinfo(CODESET)); + */ + + if (loc != NULL) { + /* using UTF-8 character set so can set UTF-8 GPBF bit 11 */ + using_utf8 = 1; + /* + printf(" Locale set to %s\n", loc); + */ + } else { + /* + printf(" Could not set Unicode UTF-8 locale\n"); + */ + } + } +# endif +#endif + + /* If no args, show help */ + if (argc == 1) + { + help(); + EXIT(ZE_OK); + } + + /* Direct info messages to stderr; stdout is used for data output. */ + mesg = stderr; + + init_upper(); /* build case map table */ + + /* Go through args */ + zipfile = tempzip = NULL; + tempzf = NULL; + signal(SIGINT, handler); +#ifdef SIGTERM /* AMIGA has no SIGTERM */ + signal(SIGTERM, handler); +#endif +#ifdef SIGABRT + signal(SIGABRT, handler); +#endif +#ifdef SIGBREAK + signal(SIGBREAK, handler); +#endif +#ifdef SIGBUS + signal(SIGBUS, handler); +#endif +#ifdef SIGILL + signal(SIGILL, handler); +#endif +#ifdef SIGSEGV + signal(SIGSEGV, handler); +#endif + k = w = 0; + for (r = 1; r < argc; r++) + if (*argv[r] == '-') { + if (argv[r][1]) + for (q = argv[r]+1; *q; q++) + switch (*q) + { + case 'b': /* Specify path for temporary file */ + if (k) + ziperr(ZE_PARMS, "use -b before zip file name"); + else + k = 1; /* Next non-option is path */ + break; + case 'h': /* Show help */ + help(); EXIT(ZE_OK); + case 'l': case 'L': /* Show copyright and disclaimer */ + license(); EXIT(ZE_OK); + case 'q': /* Quiet operation, suppress info messages */ + noisy = 0; break; + case 'v': /* Show version info */ + version_info(); EXIT(ZE_OK); + case 'w': + w = 1; break; + default: + ziperr(ZE_PARMS, "unknown option"); + } + else + ziperr(ZE_PARMS, "zip file cannot be stdin"); + } else + if (k == 0) + { + if (zipfile == NULL) + { + if ((zipfile = ziptyp(argv[r])) == NULL) + ziperr(ZE_MEM, "was processing arguments"); + } + else + ziperr(ZE_PARMS, "can only specify one zip file"); + } + else + { + tempath = argv[r]; + k = 0; + } + if (zipfile == NULL) + ziperr(ZE_PARMS, "need to specify zip file"); + + if ((in_path = malloc(strlen(zipfile) + 1)) == NULL) { + ziperr(ZE_MEM, "input"); + } + strcpy(in_path, zipfile); + + /* Read zip file */ + if ((r = readzipfile()) != ZE_OK) + ziperr(r, zipfile); + if (zfiles == NULL) + ziperr(ZE_NAME, zipfile); + + /* Put comments to stdout, if not -w */ + if (!w) + { + for (z = zfiles; z != NULL; z = z->nxt) + { + printf("%c %s\n", MARK, z->zname); + putclean(z->comment, z->com); + printf("%c%s\n", MARK, MARKE); + } + printf("%c%s\n", MARK, MARKZ); + putclean(zcomment, zcomlen); + EXIT(ZE_OK); + } + + /* If updating comments, make sure zip file is writeable */ + if ((x = fopen(zipfile, "a")) == NULL) + ziperr(ZE_CREAT, zipfile); + fclose(x); + t = getfileattr(zipfile); + + /* Process stdin, replacing comments */ + z = zfiles; + while ((a = zgetline(abf, WRBUFSIZ+1)) != NULL && + (a[0] != MARK || strcmp(a + 1, MARKZ))) + { /* while input and not file comment */ + if (a[0] != MARK || a[1] != ' ') /* better be "@ name" */ + ziperr(ZE_NOTE, "unexpected input"); + while (z != NULL && strcmp(a + 2, z->zname)) + z = z->nxt; /* allow missing entries in order */ + if (z == NULL) + ziperr(ZE_NOTE, "unknown entry name"); + if ((a = zgetline(abf, WRBUFSIZ+1)) != NULL && a[0] == MARK && a[1] == '=') + { + if (z->name != z->iname) + free((zvoid *)z->iname); + if ((z->iname = malloc(strlen(a+1))) == NULL) + ziperr(ZE_MEM, "was changing name"); +#ifdef EBCDIC + strtoasc(z->iname, a+2); +#else + strcpy(z->iname, a+2); +#endif + +/* + * Don't update z->nam here, we need the old value a little later..... + * The update is handled in zipcopy(). + */ + a = zgetline(abf, WRBUFSIZ+1); + } + if (z->com) /* change zip entry comment */ + free((zvoid *)z->comment); + z->comment = malloc(1); *(z->comment) = 0; + while (a != NULL && *a != MARK) + { + if ((r = catalloc(&(z->comment), a)) != ZE_OK) + ziperr(r, "was building new zipentry comments"); + a = zgetline(abf, WRBUFSIZ+1); + } + z->com = strlen(z->comment); + z = z->nxt; /* point to next entry */ + } + if (a != NULL) /* change zip file comment */ + { + zcomment = malloc(1); *zcomment = 0; + while ((a = zgetline(abf, WRBUFSIZ+1)) != NULL) + if ((r = catalloc(&zcomment, a)) != ZE_OK) + ziperr(r, "was building new zipfile comment"); + zcomlen = strlen(zcomment); + } + + /* Open output zip file for writing */ +#if defined(UNIX) && !defined(NO_MKSTEMP) + { + int yd; + int i; + + /* use mkstemp to avoid race condition and compiler warning */ + + if (tempath != NULL) + { + /* if -b used to set temp file dir use that for split temp */ + if ((tempzip = malloc(strlen(tempath) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, tempath); + if (lastchar(tempzip) != '/') + strcat(tempzip, "/"); + } + else + { + /* create path by stripping name and appending template */ + if ((tempzip = malloc(strlen(zipfile) + 12)) == NULL) { + ZIPERR(ZE_MEM, "allocating temp filename"); + } + strcpy(tempzip, zipfile); + for(i = strlen(tempzip); i > 0; i--) { + if (tempzip[i - 1] == '/') + break; + } + tempzip[i] = '\0'; + } + strcat(tempzip, "ziXXXXXX"); + + if ((yd = mkstemp(tempzip)) == EOF) { + ZIPERR(ZE_TEMP, tempzip); + } + if ((tempzf = y = fdopen(yd, FOPW)) == NULL) { + ZIPERR(ZE_TEMP, tempzip); + } + } +#else + if ((tempzf = y = fopen(tempzip = tempname(zipfile), FOPW)) == NULL) + ziperr(ZE_TEMP, tempzip); +#endif + + /* Open input zip file again, copy preamble if any */ + if ((in_file = fopen(zipfile, FOPR)) == NULL) + ziperr(ZE_NAME, zipfile); + + if (zipbeg && (r = bfcopy(zipbeg)) != ZE_OK) + ziperr(r, r == ZE_TEMP ? tempzip : zipfile); + tempzn = zipbeg; + + /* Go through local entries, copying them over as is */ + fix = 3; /* needed for zipcopy if name changed */ + for (z = zfiles; z != NULL; z = z->nxt) { + if ((r = zipcopy(z)) != ZE_OK) + ziperr(r, "was copying an entry"); + } + fclose(x); + + /* Write central directory and end of central directory with new comments */ + if ((c = zftello(y)) == (zoff_t)-1) /* get start of central */ + ziperr(ZE_TEMP, tempzip); + for (z = zfiles; z != NULL; z = z->nxt) + if ((r = putcentral(z)) != ZE_OK) + ziperr(r, tempzip); + if ((s = zftello(y)) == (zoff_t)-1) /* get end of central */ + ziperr(ZE_TEMP, tempzip); + s -= c; /* compute length of central */ + if ((r = putend((zoff_t)zcount, s, c, zcomlen, zcomment)) != ZE_OK) + ziperr(r, tempzip); + tempzf = NULL; + if (fclose(y)) + ziperr(ZE_TEMP, tempzip); + if ((r = replace(zipfile, tempzip)) != ZE_OK) + { + zipwarn("new zip file left as: ", tempzip); + free((zvoid *)tempzip); + tempzip = NULL; + ziperr(r, "was replacing the original zip file"); + } + free((zvoid *)tempzip); + tempzip = NULL; + setfileattr(zipfile, t); +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(zipfile,0xDDC); +#endif + free((zvoid *)zipfile); + zipfile = NULL; + + /* Done! */ + RETURN(0); +} diff --git a/third_party/infozip/zip/zipsplit.c b/third_party/infozip/zip/zipsplit.c new file mode 100644 index 000000000..858f06f6c --- /dev/null +++ b/third_party/infozip/zip/zipsplit.c @@ -0,0 +1,983 @@ +/* clang-format off */ +/* + zipsplit.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * zipsplit.c by Mark Adler. + */ +#define __ZIPSPLIT_C + +#ifndef UTIL +#define UTIL +#endif +#include "third_party/infozip/zip/zip.h" +#define DEFCPYRT /* main module: enable copyright string defines! */ +#include "third_party/infozip/zip/revision.h" +#include "libc/calls/calls.h" +#include "libc/fmt/fmt.h" +#include "libc/fmt/conv.h" +#include "libc/alg/alg.h" +#include "libc/log/log.h" + +#define DEFSIZ 36000L /* Default split size (change in help() too) */ +#ifdef MSDOS +# define NL 2 /* Number of bytes written for a \n */ +#else /* !MSDOS */ +# define NL 1 /* Number of bytes written for a \n */ +#endif /* ?MSDOS */ +#ifdef RISCOS +# define INDEX "zipspl/idx" /* Name of index file */ +# define TEMPL_FMT "%%0%dld" +# define TEMPL_SIZ 13 +# define ZPATH_SEP '.' +#else +#ifdef QDOS +# define ZPATH_SEP '_' +# define INDEX "zipsplit_idx" /* Name of index file */ +# define TEMPL_FMT "%%0%dld_zip" +# define TEMPL_SIZ 17 +# define exit(p1) QDOSexit() +#else +#ifdef VM_CMS +# define INDEX "zipsplit.idx" /* Name of index file */ +# define TEMPL_FMT "%%0%dld.zip" +# define TEMPL_SIZ 21 +# define ZPATH_SEP '.' +#else +# define INDEX "zipsplit.idx" /* Name of index file */ +# define TEMPL_FMT "%%0%dld.zip" +# define TEMPL_SIZ 17 +# define ZPATH_SEP '.' +#endif /* VM_CMS */ +#endif /* QDOS */ +#endif /* RISCOS */ + +#ifdef MACOS +#define ziperr(c, h) zipspliterr(c, h) +#define zipwarn(a, b) zipsplitwarn(a, b) +void zipsplitwarn(ZCONST char *a, ZCONST char *b); +void zipspliterr(int c, ZCONST char *h); +#endif /* MACOS */ + +/* Local functions */ +local zvoid *talloc OF((extent)); +local void tfree OF((zvoid *)); +local void tfreeall OF((void)); +local void handler OF((int)); +local void license OF((void)); +local void help OF((void)); +local void version_info OF((void)); +local extent simple OF((uzoff_t *, extent, uzoff_t, uzoff_t)); +local int descmp OF((ZCONST zvoid *, ZCONST zvoid *)); +local extent greedy OF((uzoff_t *, extent, uzoff_t, uzoff_t)); +local int retry OF((void)); +int main OF((int, char **)); + + +/* Output zip files */ +local char template[TEMPL_SIZ]; /* name template for output files */ +local int zipsmade = 0; /* number of zip files made */ +local int indexmade = 0; /* true if index file made */ +local char *path = NULL; /* space for full name */ +local char *name; /* where name goes in path[] */ + + +/* The talloc() and tree() routines extend malloc() and free() to keep + track of all allocated memory. Then the tfreeall() routine uses this + information to free all allocated memory before exiting. */ + +#define TMAX 6 /* set intelligently by examining the code */ +zvoid *talls[TMAX]; /* malloc'ed pointers to track */ +int talln = 0; /* number of entries in talls[] */ + + +int set_filetype(out_path) + char *out_path; +{ +#ifdef __BEOS__ + /* Set the filetype of the zipfile to "application/zip" */ + setfiletype( out_path, "application/zip" ); +#endif + +#ifdef __ATHEOS__ + /* Set the filetype of the zipfile to "application/x-zip" */ + setfiletype(out_path, "application/x-zip"); +#endif + +#ifdef MACOS + /* Set the Creator/Type of the zipfile to 'IZip' and 'ZIP ' */ + setfiletype(out_path, 'IZip', 'ZIP '); +#endif + +#ifdef RISCOS + /* Set the filetype of the zipfile to &DDC */ + setfiletype(out_path, 0xDDC); +#endif + return ZE_OK; +} + +/* rename a split + * A split has a tempfile name until it is closed, then + * here rename it as out_path the final name for the split. + * + * This is not used in zipsplit but is referenced by the generic split + * writing code. If zipsplit is made split aware (so can write splits of + * splits, if that makes sense) then this would get used. But if that + * happens these utility versions should be dropped and the main ones + * used. + */ +int rename_split(temp_name, out_path) + char *temp_name; + char *out_path; +{ + int r; + /* Replace old zip file with new zip file, leaving only the new one */ + if ((r = replace(out_path, temp_name)) != ZE_OK) + { + zipwarn("new zip file left as: ", temp_name); + free((zvoid *)tempzip); + tempzip = NULL; + ZIPERR(r, "was replacing split file"); + } + if (zip_attributes) { + setfileattr(out_path, zip_attributes); + } + return ZE_OK; +} + +void zipmessage_nl(a, nl) +ZCONST char *a; /* message string to output */ +int nl; /* 1 = add nl to end */ +/* If nl false, print a message to mesg without new line. + If nl true, print and add new line. If logfile is + open then also write message to log file. */ +{ + if (noisy) { + fprintf(mesg, "%s", a); + if (nl) { + fprintf(mesg, "\n"); + mesg_line_started = 0; + } else { + mesg_line_started = 1; + } + fflush(mesg); + } +} + +void zipmessage(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a message to mesg and flush. Also write to log file if + open. Write new line first if current line has output already. */ +{ + if (noisy) { + if (mesg_line_started) + fprintf(mesg, "\n"); + fprintf(mesg, "%s%s\n", a, b); + mesg_line_started = 0; + fflush(mesg); + } +} + +local zvoid *talloc(s) +extent s; +/* does a malloc() and saves the pointer to free later (does not check + for an overflow of the talls[] list) */ +{ + zvoid *p; + + if ((p = (zvoid *)malloc(s)) != NULL) + talls[talln++] = p; + return p; +} + + +local void tfree(p) +zvoid *p; +/* does a free() and also removes the pointer from the talloc() list */ +{ + int i; + + free(p); + i = talln; + while (i--) + if (talls[i] == p) + break; + if (i >= 0) + { + while (++i < talln) + talls[i - 1] = talls[i]; + talln--; + } +} + + +local void tfreeall() +/* free everything talloc'ed and not tfree'd */ +{ + while (talln) + free(talls[--talln]); +} + + +void ziperr(c, h) +int c; /* error code from the ZE_ class */ +ZCONST char *h; /* message about how it happened */ +/* Issue a message for the error, clean up files and memory, and exit. */ +{ + if (PERR(c)) + perror("zipsplit error"); + fprintf(mesg, "zipsplit error: %s (%s)\n", ZIPERRORS(c), h); + if (indexmade) + { + strcpy(name, INDEX); + destroy(path); + } + for (; zipsmade; zipsmade--) + { + sprintf(name, template, zipsmade); + destroy(path); + } + tfreeall(); + if (zipfile != NULL) + free((zvoid *)zipfile); + EXIT(c); +} + + +local void handler(s) +int s; /* signal number (ignored) */ +/* Upon getting a user interrupt, abort cleanly using ziperr(). */ +{ +#ifndef MSDOS + putc('\n', mesg); +#endif /* !MSDOS */ + ziperr(ZE_ABORT, "aborting"); + s++; /* keep some compilers happy */ +} + + +void zipwarn(a, b) +ZCONST char *a, *b; /* message strings juxtaposed in output */ +/* Print a warning message to mesg (usually stderr) and return. */ +{ + fprintf(mesg, "zipsplit warning: %s%s\n", a, b); +} + + +local void license() +/* Print license information to stdout. */ +{ + extent i; /* counter for copyright array */ + + for (i = 0; i < sizeof(swlicense)/sizeof(char *); i++) + puts(swlicense[i]); +} + + +local void help() +/* Print help (along with license info) to stdout. */ +{ + extent i; /* counter for help array */ + + /* help array */ + static ZCONST char *text[] = { +"", +"ZipSplit %s (%s)", +#ifdef VM_CMS +"Usage: zipsplit [-tipqs] [-n size] [-r room] [-b fm] zipfile", +#else +"Usage: zipsplit [-tipqs] [-n size] [-r room] [-b path] zipfile", +#endif +" -t report how many files it will take, but don't make them", +#ifdef RISCOS +" -i make index (" INDEX ") and count its size against first zip file", +#else +" -i make index (zipsplit.idx) and count its size against first zip file", +#endif +" -n make zip files no larger than \"size\" (default = 36000)", +" -r leave room for \"room\" bytes on the first disk (default = 0)", +#ifdef VM_CMS +" -b use \"fm\" as the filemode for the output zip files", +#else +" -b use \"path\" for the output zip files", +#endif +" -q quieter operation, suppress some informational messages", +" -p pause between output zip files", +" -s do a sequential split even if it takes more zip files", +" -h show this help -v show version info -L show software license" + }; + + for (i = 0; i < sizeof(copyright)/sizeof(char *); i++) { + printf(copyright[i], "zipsplit"); + putchar('\n'); + } + for (i = 0; i < sizeof(text)/sizeof(char *); i++) + { + printf(text[i], VERSION, REVDATE); + putchar('\n'); + } +} + + +local void version_info() +/* Print verbose info about program version and compile time options + to stdout. */ +{ + extent i; /* counter in text arrays */ + + /* Options info array */ + static ZCONST char *comp_opts[] = { +#ifdef DEBUG + "DEBUG", +#endif + NULL + }; + + for (i = 0; i < sizeof(versinfolines)/sizeof(char *); i++) + { + printf(versinfolines[i], "ZipSplit", VERSION, REVDATE); + putchar('\n'); + } + + version_local(); + + puts("ZipSplit special compilation options:"); + for (i = 0; (int)i < (int)(sizeof(comp_opts)/sizeof(char *) - 1); i++) + { + printf("\t%s\n",comp_opts[i]); + } + if (i == 0) + puts("\t[none]"); +} + + +local extent simple(a, n, c, d) +uzoff_t *a; /* items to put in bins, return value: destination bins */ +extent n; /* number of items */ +uzoff_t c; /* capacity of each bin */ +uzoff_t d; /* amount to deduct from first bin */ +/* Return the number of bins of capacity c that are needed to contain the + integers in a[0..n-1] placed sequentially into the bins. The value d + is deducted initially from the first bin (space for index). The entries + in a[] are replaced by the destination bins. */ +{ + extent k; /* current bin number */ + uzoff_t t; /* space used in current bin */ + + t = k = 0; + while (n--) + { + if (*a + t > c - (k == 0 ? d : 0)) + { + k++; + t = 0; + } + t += *a; + *(ulg huge *)a++ = k; + } + return k + 1; +} + + +local int descmp(a, b) +ZCONST zvoid *a, *b; /* pointers to pointers to ulg's to compare */ +/* Used by qsort() in greedy() to do a descending sort. */ +{ + return **(uzoff_t **)a < **(uzoff_t **)b ? 1 : + (**(uzoff_t **)a > **(uzoff_t **)b ? -1 : 0); +} + + +local extent greedy(a, n, c, d) +uzoff_t *a; /* items to put in bins, return value: destination bins */ +extent n; /* number of items */ +uzoff_t c; /* capacity of each bin */ +uzoff_t d; /* amount to deduct from first bin */ +/* Return the number of bins of capacity c that are needed to contain the + items with sizes a[0..n-1] placed non-sequentially into the bins. The + value d is deducted initially from the first bin (space for index). + The entries in a[] are replaced by the destination bins. */ +{ + uzoff_t *b; /* space left in each bin (malloc'ed for each m) */ + uzoff_t *e; /* copy of argument a[] (malloc'ed) */ + extent i; /* steps through items */ + extent j; /* steps through bins */ + extent k; /* best bin to put current item in */ + extent m; /* current number of bins */ + uzoff_t **s; /* pointers to e[], sorted descending (malloc'ed) */ + uzoff_t t; /* space left in best bin (index k) */ + + /* Algorithm: + 1. Copy a[] to e[] and sort pointers to e[0..n-1] (in s[]), in + descending order. + 2. Compute total of s[] and set m to the smallest number of bins of + capacity c that can hold the total. + 3. Allocate m bins. + 4. For each item in s[], starting with the largest, put it in the + bin with the smallest current capacity greater than or equal to the + item's size. If no bin has enough room, increment m and go to step 4. + 5. Else, all items ended up in a bin--return m. + */ + + /* Copy a[] to e[], put pointers to e[] in s[], and sort s[]. Also compute + the initial number of bins (minus 1). */ + if ((e = (uzoff_t *)malloc(n * sizeof(uzoff_t))) == NULL || + (s = (uzoff_t **)malloc(n * sizeof(uzoff_t *))) == NULL) + { + if (e != NULL) + free((zvoid *)e); + ziperr(ZE_MEM, "was trying a smart split"); + return 0; /* only to make compiler happy */ + } + memcpy((char *)e, (char *)a, n * sizeof(uzoff_t)); + for (t = i = 0; i < n; i++) + t += *(s[i] = e + i); + m = (extent)((t + c - 1) / c) - 1; /* pre-decrement for loop */ + qsort((char *)s, n, sizeof(ulg *), descmp); + + /* Stuff bins until successful */ + do { + /* Increment the number of bins, allocate and initialize bins */ + if ((b = (uzoff_t *)malloc(++m * sizeof(uzoff_t))) == NULL) + { + free((zvoid *)s); + free((zvoid *)e); + ziperr(ZE_MEM, "was trying a smart split"); + } + b[0] = c - d; /* leave space in first bin */ + for (j = 1; j < m; j++) + b[j] = c; + + /* Fill the bins greedily */ + for (i = 0; i < n; i++) + { + /* Find smallest bin that will hold item i (size s[i]) */ + t = c + 1; + for (k = j = 0; j < m; j++) + if (*s[i] <= b[j] && b[j] < t) + t = b[k = j]; + + /* If no bins big enough for *s[i], try next m */ + if (t == c + 1) + break; + + /* Diminish that bin and save where it goes */ + b[k] -= *s[i]; + a[(int)((uzoff_t huge *)(s[i]) - (uzoff_t huge *)e)] = k; + } + + /* Clean up */ + free((zvoid *)b); + + /* Do until all items put in a bin */ + } while (i < n); + + /* Done--clean up and return the number of bins needed */ + free((zvoid *)s); + free((zvoid *)e); + return m; +} + +/* keep compiler happy until implement long options - 11/4/2003 EG */ +struct option_struct far options[] = { + /* short longopt value_type negatable ID name */ + {"h", "help", o_NO_VALUE, o_NOT_NEGATABLE, 'h', "help"}, + /* the end of the list */ + {NULL, NULL, o_NO_VALUE, o_NOT_NEGATABLE, 0, NULL} /* end has option_ID = 0 */ + }; + + +local int retry() +{ + char m[10]; + fputs("Error writing to disk--redo entire disk? ", mesg); + fgets(m, 10, stdin); + return *m == 'y' || *m == 'Y'; +} + + +#ifndef USE_ZIPSPLITMAIN +int main(argc, argv) +#else +int zipsplitmain(argc, argv) +#endif + +int argc; /* number of tokens in command line */ +char **argv; /* command line tokens */ +/* Split a zip file into several zip files less than a specified size. See + the command help in help() above. */ +{ + uzoff_t *a; /* malloc'ed list of sizes, dest bins */ + extent *b; /* heads of bin linked lists (malloc'ed) */ + uzoff_t c; /* bin capacity, start of central directory */ + int d; /* if true, just report the number of disks */ + FILE *e; /* input zip file */ + FILE *f; /* output index and zip files */ + extent g; /* number of bins from greedy(), entry to write */ + int h; /* how to split--true means simple split, counter */ + zoff_t i = 0; /* size of index file plus room to leave */ + extent j; /* steps through zip entries, bins */ + int k; /* next argument type */ + extent *n = NULL; /* next item in bin list (heads in b) */ + uzoff_t *p; /* malloc'ed list of sizes, dest bins for greedy() */ + char *q; /* steps through option characters */ + int r; /* temporary variable, counter */ + extent s; /* number of bins needed */ + zoff_t t; /* total of sizes, end of central directory */ + int u; /* flag to wait for user on output files */ + struct zlist far **w; /* malloc'ed table for zfiles linked list */ + int x; /* if true, make an index file */ + struct zlist far *z; /* steps through zfiles linked list */ +#ifdef AMIGA + char tailchar; /* temporary variable used in name generation below */ +#endif + char errbuf[5000]; + +#ifdef THEOS + setlocale(LC_CTYPE, "I"); +#endif + +#ifdef UNICODE_SUPPORT +# ifdef UNIX + /* For Unix, set the locale to UTF-8. Any UTF-8 locale is + OK and they should all be the same. This allows seeing, + writing, and displaying (if the fonts are loaded) all + characters in UTF-8. */ + { + char *loc; + + /* + loc = setlocale(LC_CTYPE, NULL); + printf(" Initial language locale = '%s'\n", loc); + */ + + loc = setlocale(LC_CTYPE, "en_US.UTF-8"); + + /* + printf("langinfo %s\n", nl_langinfo(CODESET)); + */ + + if (loc != NULL) { + /* using UTF-8 character set so can set UTF-8 GPBF bit 11 */ + using_utf8 = 1; + /* + printf(" Locale set to %s\n", loc); + */ + } else { + /* + printf(" Could not set Unicode UTF-8 locale\n"); + */ + } + } +# endif +#endif + + /* If no args, show help */ + if (argc == 1) + { + help(); + EXIT(ZE_OK); + } + + /* Informational messages are written to stdout. */ + mesg = stdout; + + init_upper(); /* build case map table */ + + /* Go through args */ + signal(SIGINT, handler); +#ifdef SIGTERM /* Amiga has no SIGTERM */ + signal(SIGTERM, handler); +#endif +#ifdef SIGABRT + signal(SIGABRT, handler); +#endif +#ifdef SIGBREAK + signal(SIGBREAK, handler); +#endif +#ifdef SIGBUS + signal(SIGBUS, handler); +#endif +#ifdef SIGILL + signal(SIGILL, handler); +#endif +#ifdef SIGSEGV + signal(SIGSEGV, handler); +#endif + k = h = x = d = u = 0; + c = DEFSIZ; + for (r = 1; r < argc; r++) + if (*argv[r] == '-') + { + if (argv[r][1]) + for (q = argv[r]+1; *q; q++) + switch (*q) + { + case 'b': /* Specify path for output files */ + if (k) + ziperr(ZE_PARMS, "options are separate and precede zip file"); + else + k = 1; /* Next non-option is path */ + break; + case 'h': /* Show help */ + help(); EXIT(ZE_OK); + case 'i': /* Make an index file */ + x = 1; + break; + case 'l': case 'L': /* Show copyright and disclaimer */ + license(); EXIT(ZE_OK); + case 'n': /* Specify maximum size of resulting zip files */ + if (k) + ziperr(ZE_PARMS, "options are separate and precede zip file"); + else + k = 2; /* Next non-option is size */ + break; + case 'p': + u = 1; + break; + case 'q': /* Quiet operation, suppress info messages */ + noisy = 0; + break; + case 'r': + if (k) + ziperr(ZE_PARMS, "options are separate and precede zip file"); + else + k = 3; /* Next non-option is room to leave */ + break; + case 's': + h = 1; /* Only try simple */ + break; + case 't': /* Just report number of disks */ + d = 1; + break; + case 'v': /* Show version info */ + version_info(); EXIT(ZE_OK); + default: + ziperr(ZE_PARMS, "Use option -h for help."); + } + else + ziperr(ZE_PARMS, "zip file cannot be stdin"); + } + else + switch (k) + { + case 0: + if (zipfile == NULL) + { + if ((zipfile = ziptyp(argv[r])) == NULL) + ziperr(ZE_MEM, "was processing arguments"); + } + else + ziperr(ZE_PARMS, "can only specify one zip file"); + break; + case 1: + tempath = argv[r]; + k = 0; + break; + case 2: + if ((c = (ulg)atol(argv[r])) < 100) /* 100 is smallest zip file */ + ziperr(ZE_PARMS, "invalid size given. Use option -h for help."); + k = 0; + break; + default: /* k must be 3 */ + i = (ulg)atol(argv[r]); + k = 0; + break; + } + if (zipfile == NULL) + ziperr(ZE_PARMS, "need to specify zip file"); + + if ((in_path = malloc(strlen(zipfile) + 1)) == NULL) { + ziperr(ZE_MEM, "input"); + } + strcpy(in_path, zipfile); + + /* Read zip file */ + if ((r = readzipfile()) != ZE_OK) + ziperr(r, zipfile); + if (zfiles == NULL) + ziperr(ZE_NAME, zipfile); + + /* Make a list of sizes and check against capacity. Also compute the + size of the index file. */ + c -= ENDHEAD + 4; /* subtract overhead/zipfile */ + if ((a = (uzoff_t *)talloc(zcount * sizeof(uzoff_t))) == NULL || + (w = (struct zlist far **)talloc(zcount * sizeof(struct zlist far *))) == + NULL) + { + ziperr(ZE_MEM, "was computing split"); + return 1; + } + t = 0; + for (j = 0, z = zfiles; j < zcount; j++, z = z->nxt) + { + w[j] = z; + if (x) + i += z->nam + 6 + NL; + /* New scanzip_reg only reads central directory so use cext for ext */ + t += a[j] = 8 + LOCHEAD + CENHEAD + + 2 * (zoff_t)z->nam + 2 * (zoff_t)z->cext + z->com + z->siz; + if (a[j] > c) { + sprintf(errbuf, "Entry is larger than max split size of: %s", + zip_fzofft(c, NULL, "u")); + zipwarn(errbuf, ""); + zipwarn("use -n to set split size", ""); + ziperr(ZE_BIG, z->zname); + } + } + + /* Decide on split to use, report number of files */ + if (h) + s = simple(a, zcount, c, i); + else + { + if ((p = (uzoff_t *)talloc(zcount * sizeof(uzoff_t))) == NULL) + ziperr(ZE_MEM, "was computing split"); + memcpy((char *)p, (char *)a, zcount * sizeof(uzoff_t)); + s = simple(a, zcount, c, i); + g = greedy(p, zcount, c, i); + if (s <= g) + tfree((zvoid *)p); + else + { + tfree((zvoid *)a); + a = p; + s = g; + } + } + printf("%ld zip files w%s be made (%s%% efficiency)\n", + (ulg)s, d ? "ould" : "ill", + zip_fzofft( ((200 * ((t + c - 1)/c)) / s + 1) / 2, NULL, "d")); + if (d) + { + tfreeall(); + free((zvoid *)zipfile); + zipfile = NULL; + EXIT(ZE_OK); + } + + /* Set up path for output files */ + /* Point "name" past the path, where the filename should go */ + if ((path = (char *)talloc(tempath == NULL ? 13 : strlen(tempath) + 14)) == + NULL) + ziperr(ZE_MEM, "was making output file names"); + if (tempath == NULL) + name = path; + else + { +#ifndef VM_CMS + /* Copy the output path to the target */ + strcpy(path, tempath); +#endif +#ifdef AMIGA + tailchar = path[strlen(path) - 1]; /* last character */ + if (path[0] && (tailchar != '/') && (tailchar != ':')) + strcat(path, "/"); +#else +#ifdef RISCOS + if (path[0] && path[strlen(path) - 1] != '.') + strcat(path, "."); +#else +#ifdef QDOS + if (path[0] && path[strlen(path) - 1] != '_') + strcat(path, "_"); +#else +#ifndef VMS + if (path[0] && path[strlen(path) - 1] != '/') + strcat(path, "/"); +#endif /* !VMS */ +#endif /* ?QDOS */ +#endif /* ?RISCOS */ +#endif /* ?AMIGA */ + name = path + strlen(path); + } + + /* Make linked lists of results */ + if ((b = (extent *)talloc(s * sizeof(extent))) == NULL || + (n = (extent *)talloc(zcount * sizeof(extent))) == NULL) + ziperr(ZE_MEM, "was computing split"); + for (j = 0; j < s; j++) + b[j] = (extent)-1; + j = zcount; + while (j--) + { + g = (extent)a[j]; + n[j] = b[g]; + b[g] = j; + } + + /* Make a name template for the zip files that is eight or less characters + before the .zip, and that will not overwrite the original zip file. */ + for (k = 1, j = s; j >= 10; j /= 10) + k++; + if (k > 7) + ziperr(ZE_PARMS, "way too many zip files must be made"); +/* + * XXX, ugly .... + */ +/* Find the final "path" separator character */ +#ifdef QDOS + q = LastDir(zipfile); +#else +#ifdef VMS + if ((q = strrchr(zipfile, ']')) != NULL) +#else +#ifdef AMIGA + if (((q = strrchr(zipfile, '/')) != NULL) + || ((q = strrchr(zipfile, ':'))) != NULL) +#else +#ifdef RISCOS + if ((q = strrchr(zipfile, '.')) != NULL) +#else +#ifdef MVS + if ((q = strrchr(zipfile, '.')) != NULL) +#else + if ((q = strrchr(zipfile, '/')) != NULL) +#endif /* MVS */ +#endif /* RISCOS */ +#endif /* AMIGA */ +#endif /* VMS */ + q++; + else + q = zipfile; +#endif /* QDOS */ + + r = 0; + while ((g = *q++) != '\0' && g != ZPATH_SEP && r < 8 - k) + template[r++] = (char)g; + if (r == 0) + template[r++] = '_'; + else if (g >= '0' && g <= '9') + template[r - 1] = (char)(template[r - 1] == '_' ? '-' : '_'); + sprintf(template + r, TEMPL_FMT, k); +#ifdef VM_CMS + /* For CMS, add the "path" as the filemode at the end */ + if (tempath) + { + strcat(template,"."); + strcat(template,tempath); + } +#endif + + /* Make the zip files from the linked lists of entry numbers */ + if ((e = fopen(zipfile, FOPR)) == NULL) + ziperr(ZE_NAME, zipfile); + free((zvoid *)zipfile); + zipfile = NULL; + for (j = 0; j < s; j++) + { + /* jump here on a disk retry */ + redobin: + + current_disk = 0; + cd_start_disk = 0; + cd_entries_this_disk = 0; + + /* prompt if requested */ + if (u) + { + char m[10]; + fprintf(mesg, "Insert disk #%ld of %ld and hit return: ", + (ulg)j + 1, (ulg)s); + fgets(m, 10, stdin); + } + + /* write index file on first disk if requested */ + if (j == 0 && x) + { + strcpy(name, INDEX); + printf("creating: %s\n", path); + indexmade = 1; + if ((f = fopen(path, "w")) == NULL) + { + if (u && retry()) goto redobin; + ziperr(ZE_CREAT, path); + } + for (j = 0; j < zcount; j++) + fprintf(f, "%5s %s\n", + zip_fzofft( (a[j] + 1), NULL, "d"), w[j]->zname); + + if ((j = ferror(f)) != 0 || fclose(f)) + { + if (j) + fclose(f); + if (u && retry()) goto redobin; + ziperr(ZE_WRITE, path); + } + } + + /* create output zip file j */ + sprintf(name, template, j + 1L); + printf("creating: %s\n", path); + zipsmade = j + 1; + if ((y = f = fopen(path, FOPW)) == NULL) + { + if (u && retry()) goto redobin; + ziperr(ZE_CREAT, path); + } + bytes_this_split = 0; + tempzn = 0; + + /* write local headers and copy compressed data */ + for (g = b[j]; g != (extent)-1; g = (extent)n[g]) + { + if (zfseeko(e, w[g]->off, SEEK_SET)) + ziperr(ferror(e) ? ZE_READ : ZE_EOF, zipfile); + in_file = e; + if ((r = zipcopy(w[g])) != ZE_OK) + { + if (r == ZE_TEMP) + { + if (u && retry()) goto redobin; + ziperr(ZE_WRITE, path); + } + else + ziperr(r, zipfile); + } + } + + /* write central headers */ + if ((c = zftello(f)) == (uzoff_t)-1) + { + if (u && retry()) goto redobin; + ziperr(ZE_WRITE, path); + } + for (g = b[j], k = 0; g != (extent)-1; g = n[g], k++) + if ((r = putcentral(w[g])) != ZE_OK) + { + if (u && retry()) goto redobin; + ziperr(ZE_WRITE, path); + } + + /* write end-of-central header */ + cd_start_offset = c; + total_cd_entries = k; + if ((t = zftello(f)) == (zoff_t)-1 || + (r = putend((zoff_t)k, t - c, c, (extent)0, (char *)NULL)) != + ZE_OK || + ferror(f) || fclose(f)) + { + if (u && retry()) goto redobin; + ziperr(ZE_WRITE, path); + } +#ifdef RISCOS + /* Set the filetype to &DDC */ + setfiletype(path,0xDDC); +#endif + } + fclose(e); + + /* Done! */ + if (u) + fputs("Done.\n", mesg); + tfreeall(); + + RETURN(0); +} diff --git a/third_party/infozip/zip/zipup.c b/third_party/infozip/zip/zipup.c new file mode 100644 index 000000000..6ec5028ba --- /dev/null +++ b/third_party/infozip/zip/zipup.c @@ -0,0 +1,1928 @@ +/* clang-format off */ +/* + zipup.c - Zip 3 + + Copyright (c) 1990-2008 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2007-Mar-4 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html +*/ +/* + * zipup.c by Mark Adler and Jean-loup Gailly. + */ +#define __ZIPUP_C + +/* Found that for at least unix port zip.h has to be first or ctype.h will + define off_t and when using 64-bit file environment off_t in other files + is 8 bytes while off_t here is 4 bytes, and this makes the zlist struct + different sizes and needless to say leads to segmentation faults. Putting + zip.h first seems to fix this. 8/14/04 EG */ +#include "third_party/infozip/zip/zip.h" +#include "libc/errno.h" +#include "libc/str/str.h" +#include "libc/log/log.h" +#include "libc/sysv/consts/prot.h" +#include "libc/runtime/runtime.h" + +#ifndef UTIL /* This module contains no code for Zip Utilities */ + +#include "third_party/infozip/zip/revision.h" +#include "third_party/infozip/zip/crc32.h" +#include "third_party/infozip/zip/crypt.h" +#ifdef USE_ZLIB +# include "third_party/zlib/zlib.h" +#endif +#ifdef BZIP2_SUPPORT +# ifdef BZIP2_USEBZIP2DIR +# include "bzip2/bzlib.h" +# else +# include "bzlib.h" +# endif +#endif + +#ifdef OS2 +# include "os2/os2zip.h" +#endif + +#if defined(MMAP) +#include "libc/calls/calls.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/mremap.h" +# ifndef PAGESIZE /* used to be SYSV, what about pagesize on SVR3 ? */ +# define PAGESIZE getpagesize() +# endif +# if defined(NO_VALLOC) && !defined(valloc) +# define valloc malloc +# endif +#endif + +/* Use the raw functions for MSDOS and Unix to save on buffer space. + They're not used for VMS since it doesn't work (raw is weird on VMS). + */ + +#ifdef AMIGA +# include "amiga/zipup.h" +#endif /* AMIGA */ + +#ifdef AOSVS +# include "aosvs/zipup.h" +#endif /* AOSVS */ + +#ifdef ATARI +# include "atari/zipup.h" +#endif + +#ifdef __BEOS__ +# include "beos/zipup.h" +#endif + +#ifdef __ATHEOS__ +# include "atheos/zipup.h" +#endif /* __ATHEOS__ */ + +#ifdef __human68k__ +# include "human68k/zipup.h" +#endif /* __human68k__ */ + +#ifdef MACOS +# include "macos/zipup.h" +#endif + +#ifdef DOS +# include "msdos/zipup.h" +#endif /* DOS */ + +#ifdef NLM +# include "novell/zipup.h" +# include +#endif + +#ifdef OS2 +# include "os2/zipup.h" +#endif /* OS2 */ + +#ifdef RISCOS +# include "acorn/zipup.h" +#endif + +#ifdef TOPS20 +# include "tops20/zipup.h" +#endif + +#ifdef UNIX +# include "unix/zipup.h" +#endif + +#ifdef CMS_MVS +# include "zipup.h" +#endif /* CMS_MVS */ + +#ifdef TANDEM +# include "zipup.h" +#endif /* TANDEM */ + +#ifdef VMS +# include "vms/zipup.h" +#endif /* VMS */ + +#ifdef QDOS +# include "qdos/zipup.h" +#endif /* QDOS */ + +#ifdef WIN32 +# include "win32/zipup.h" +#endif + +#ifdef THEOS +# include "theos/zipup.h" +#endif + +/* Local functions */ +#ifndef RISCOS + local int suffixes OF((char *, char *)); +#else + local int filetypes OF((char *, char *)); +#endif +local unsigned file_read OF((char *buf, unsigned size)); +#ifdef USE_ZLIB + local int zl_deflate_init OF((int pack_level)); +#else /* !USE_ZLIB */ +# ifdef ZP_NEED_MEMCOMPR + local unsigned mem_read OF((char *buf, unsigned size)); +# endif +#endif /* ?USE_ZLIB */ + +/* zip64 support 08/29/2003 R.Nausedat */ +local zoff_t filecompress OF((struct zlist far *z_entry, int *cmpr_method)); + +#ifdef BZIP2_SUPPORT +local zoff_t bzfilecompress OF((struct zlist far *z_entry, int *cmpr_method)); +#endif + +/* Deflate "internal" global data (currently not in zip.h) */ +#if defined(MMAP) || defined(BIG_MEM) +# ifdef USE_ZLIB + local uch *window = NULL; /* Used to read all input file at once */ + local ulg window_size; /* size of said window */ +# else /* !USE_ZLIB */ + extern uch *window; /* Used to read all input file at once */ +#endif /* ?USE_ZLIB */ +#endif /* MMAP || BIG_MEM */ +#ifndef USE_ZLIB + extern ulg window_size; /* size of said window */ + + unsigned (*read_buf) OF((char *buf, unsigned size)) = file_read; + /* Current input function. Set to mem_read for in-memory compression */ +#endif /* !USE_ZLIB */ + + +/* Local data */ +local ulg crc; /* crc on uncompressed file data */ +local ftype ifile; /* file to compress */ +#if defined(MMAP) || defined(BIG_MEM) + local ulg remain; + /* window bytes not yet processed. + * special value "(ulg)-1L" reserved to signal normal reads. + */ +#endif /* MMAP || BIG_MEM */ +#ifdef USE_ZLIB + local int deflInit = FALSE; /* flag: zlib deflate is initialized */ + local z_stream zstrm; /* zlib's data interface structure */ + local char *f_ibuf = NULL; + local char *f_obuf = NULL; +#else /* !USE_ZLIB */ + local char file_outbuf[1024]; /* output buffer for compression to file */ + +# ifdef ZP_NEED_MEMCOMPR + local char *in_buf; + /* Current input buffer, in_buf is used only for in-memory compression. */ + local unsigned in_offset; + /* Current offset in input buffer. in_offset is used only for in-memory + * compression. On 16 bit machines, the buffer is limited to 64K. + */ + local unsigned in_size; /* size of current input buffer */ +# endif /* ZP_NEED_MEMCOMPR */ +#endif /* ?USE_ZLIB */ + +#ifdef BZIP2_SUPPORT + local int bzipInit; /* flag: bzip2lib is initialized */ + local bz_stream bstrm; /* zlib's data interface structure */ +# if !defined(USE_ZLIB) + local char *f_ibuf = NULL; + local char *f_obuf = NULL; +# endif /* !USE_ZLIB */ +#endif /* BZIP2_SUPPORT */ + +#ifdef DEBUG + zoff_t isize; /* input file size. global only for debugging */ +#else /* !DEBUG */ + local zoff_t isize; /* input file size. global only for debugging */ +#endif /* ?DEBUG */ + /* If file_read detects binary it sets this flag - 12/16/04 EG */ + local int file_binary = 0; /* first buf */ + local int file_binary_final = 0; /* for bzip2 for entire file. assume text until find binary */ + + +/* moved check to function 3/14/05 EG */ +int is_seekable(y) + FILE *y; +{ + zoff_t pos; + +#ifdef BROKEN_FSEEK + if (!fseekable(y)) { + return 0; + } +#endif + + pos = zftello(y); + if (zfseeko(y, pos, SEEK_SET)) { + return 0; + } + + return 1; +} + + +int percent(n, m) + uzoff_t n; + uzoff_t m; /* n is the original size, m is the new size */ +/* Return the percentage compression from n to m using only integer + operations */ +{ + zoff_t p; + +#if 0 + if (n > 0xffffffL) /* If n >= 16M */ + { /* then divide n and m by 256 */ + n += 0x80; n >>= 8; + m += 0x80; m >>= 8; + } + return n > m ? (int)(1 + (200 * (n - m)/n)) / 2 : 0; +#endif + +/* 2004-12-01 SMS. + * Changed to do big-n test only for small zoff_t. + * Changed big-n arithmetic to accomodate apparently negative values + * when a small zoff_t value exceeds 2G. + * Increased the reduction divisor from 256 to 512 to avoid the sign bit + * in a reduced intermediate, allowing signed arithmetic for the final + * result (which is no longer artificially limited to non-negative + * values). + * Note that right shifts must be on unsigned values to avoid undesired + * sign extension. + */ + +/* Handle n = 0 case and account for int maybe being 16-bit. 12/28/2004 EG + */ + +#define PC_MAX_SAFE 0x007fffffL /* 9 clear bits at high end. */ +#define PC_MAX_RND 0xffffff00L /* 8 clear bits at low end. */ + + if (sizeof(uzoff_t) < 8) /* Don't fiddle with big zoff_t. */ + { + if ((ulg)n > PC_MAX_SAFE) /* Reduce large values. (n > m) */ + { + if ((ulg)n < PC_MAX_RND) /* Divide n by 512 with rounding, */ + n = ((ulg)n + 0x100) >> 9; /* if boost won't overflow. */ + else /* Otherwise, use max value. */ + n = PC_MAX_SAFE; + + if ((ulg)m < PC_MAX_RND) /* Divide m by 512 with rounding, */ + m = ((ulg)m + 0x100) >> 9; /* if boost won't overflow. */ + else /* Otherwise, use max value. */ + m = PC_MAX_SAFE; + } + } + if (n != 0) + p = ((200 * ((zoff_t)n - (zoff_t)m) / (zoff_t)n) + 1) / 2; + else + p = 0; + return (int)p; /* Return (rounded) % reduction. */ +} + + +#ifndef RISCOS + +local int suffixes(a, s) + char *a; /* name to check suffix of */ + char *s; /* list of suffixes separated by : or ; */ +/* Return true if a ends in any of the suffixes in the list s. */ +{ + int m; /* true if suffix matches so far */ + char *p; /* pointer into special */ + char *q; /* pointer into name a */ + +#ifdef QDOS + short dlen = devlen(a); + a = a + dlen; +#endif + + m = 1; +#ifdef VMS + if( (q = strrchr(a,';')) != NULL ) /* Cut out VMS file version */ + --q; + else + q = a + strlen(a) - 1; +#else /* !VMS */ + q = a + strlen(a) - 1; +#endif /* ?VMS */ + for (p = s + strlen(s) - 1; p >= s; p--) + if (*p == ':' || *p == ';') + { + if (m) + return 1; + else + { + m = 1; +#ifdef VMS + if( (q = strrchr(a,';')) != NULL ) /* Cut out VMS file version */ + --q; + else + q = a + strlen(a) - 1; +#else /* !VMS */ + q = a + strlen(a) - 1; +#endif /* ?VMS */ + } + } + else + { + m = m && q >= a && case_map(*p) == case_map(*q); + q--; + } + return m; +} + +#else /* RISCOS */ + +local int filetypes(a, s) +char *a; /* extra field of file to check filetype of */ +char *s; /* list of filetypes separated by : or ; */ +/* Return true if a is any of the filetypes in the list s. */ +{ + char *p; /* pointer into special */ + char typestr[4]; /* filetype hex string taken from a */ + + if ((((unsigned*)a)[2] & 0xFFF00000) != 0xFFF00000) { + /* The file is not filestamped, always try to compress it */ + return 0; + } + + sprintf(typestr,"%.3X",(((unsigned*)a)[2] & 0x000FFF00) >> 8); + + for (p=s;p<=s+strlen(s)-3;p+=3) { /* p+=3 to skip 3 hex type */ + while (*p==':' || *p==';') + p++; + + if (typestr[0] == toupper(p[0]) && + typestr[1] == toupper(p[1]) && + typestr[2] == toupper(p[2])) + return 1; + } + return 0; +} +#endif /* ?RISCOS */ + + + +/* Note: a zip "entry" includes a local header (which includes the file + name), an encryption header if encrypting, the compressed data + and possibly an extended local header. */ + +int zipup(z) +struct zlist far *z; /* zip entry to compress */ +/* Compress the file z->name into the zip entry described by *z and write + it to the file *y. Encrypt if requested. Return an error code in the + ZE_ class. Also, update tempzn by the number of bytes written. */ +/* y is now global */ +{ + iztimes f_utim; /* UNIX GMT timestamps, filled by filetime() */ + ulg tim; /* time returned by filetime() */ + ulg a = 0L; /* attributes returned by filetime() */ + char *b; /* malloc'ed file buffer */ + extent k = 0; /* result of zread */ + int l = 0; /* true if this file is a symbolic link */ + int m; /* method for this entry */ + + zoff_t o = 0, p; /* offsets in zip file */ + zoff_t q = (zoff_t) -3; /* size returned by filetime */ + uzoff_t uq; /* unsigned q */ + zoff_t s = 0; /* size of compressed data */ + + int r; /* temporary variable */ + int isdir; /* set for a directory name */ + int set_type = 0; /* set if file type (ascii/binary) unknown */ + zoff_t last_o; /* used to detect wrap around */ + + ush tempext = 0; /* temp copies of extra fields */ + ush tempcext = 0; + char *tempextra = NULL; + char *tempcextra = NULL; + + +#ifdef WINDLL +# ifdef ZIP64_SUPPORT + extern _int64 filesize64; + extern unsigned long low; + extern unsigned long high; +# endif +#endif + + z->nam = strlen(z->iname); + isdir = z->iname[z->nam-1] == (char)0x2f; /* ascii[(unsigned)('/')] */ + + file_binary = -1; /* not set, set after first read */ + file_binary_final = 0; /* not set, set after first read */ + +#if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) + tim = filetimew(z->namew, &a, &q, &f_utim); + else + tim = filetime(z->name, &a, &q, &f_utim); +#else + tim = filetime(z->name, &a, &q, &f_utim); +#endif + if (tim == 0 || q == (zoff_t) -3) + return ZE_OPEN; + + /* q is set to -1 if the input file is a device, -2 for a volume label */ + if (q == (zoff_t) -2) { + isdir = 1; + q = 0; + } else if (isdir != ((a & MSDOS_DIR_ATTR) != 0)) { + /* don't overwrite a directory with a file and vice-versa */ + return ZE_MISS; + } + /* reset dot_count for each file */ + if (!display_globaldots) + dot_count = -1; + + /* display uncompressed size */ + uq = ((uzoff_t) q > (uzoff_t) -3) ? 0 : (uzoff_t) q; + if (noisy && display_usize) { + fprintf(mesg, " ("); + DisplayNumString( mesg, uq ); + fprintf(mesg, ")"); + mesg_line_started = 1; + fflush(mesg); + } + if (logall && display_usize) { + fprintf(logfile, " ("); + DisplayNumString( logfile, uq ); + fprintf(logfile, ")"); + logfile_line_started = 1; + fflush(logfile); + } + + /* initial z->len so if error later have something */ + z->len = uq; + + z->att = (ush)UNKNOWN; /* will be changed later */ + z->atx = 0; /* may be changed by set_extra_field() */ + + /* Free the old extra fields which are probably obsolete */ + /* Should probably read these and keep any we don't update. 12/30/04 EG */ + if (extra_fields == 2) { + /* If keeping extra fields, make copy before clearing for set_extra_field() + A better approach is to modify the port code, but maybe later */ + if (z->ext) { + if ((tempextra = malloc(z->ext)) == NULL) { + ZIPERR(ZE_MEM, "extra fields copy"); + } + memcpy(tempextra, z->extra, z->ext); + tempext = z->ext; + } + if (z->cext) { + if ((tempcextra = malloc(z->cext)) == NULL) { + ZIPERR(ZE_MEM, "extra fields copy"); + } + memcpy(tempcextra, z->cextra, z->cext); + tempcext = z->cext; + } + } + if (z->ext) { + free((zvoid *)(z->extra)); + } + if (z->cext && z->extra != z->cextra) { + free((zvoid *)(z->cextra)); + } + z->extra = z->cextra = NULL; + z->ext = z->cext = 0; + +#if defined(MMAP) || defined(BIG_MEM) + remain = (ulg)-1L; /* changed only for MMAP or BIG_MEM */ +#endif /* MMAP || BIG_MEM */ +#if (!defined(USE_ZLIB) || defined(MMAP) || defined(BIG_MEM)) + window_size = 0L; +#endif /* !USE_ZLIB || MMAP || BIG_MEM */ + + /* Select method based on the suffix and the global method */ +#ifndef RISCOS + m = special != NULL && suffixes(z->name, special) ? STORE : method; +#else /* RISCOS must set m after setting extra field */ + m = method; +#endif /* ?RISCOS */ + + /* For now force deflate if using descriptors. Instead zip and unzip + could check bytes read against compressed size in each data descriptor + found and skip over any that don't match. This is how at least one + other zipper does it. To be added later. Until then it + probably doesn't hurt to force deflation when streaming. 12/30/04 EG + */ + + /* Now is a good time. For now allow storing for testing. 12/16/05 EG */ + /* By release need to force deflation based on reports some inflate + streamed data to find the end of the data */ + /* Need to handle bzip2 */ +#ifdef NO_STREAMING_STORE + if (use_descriptors && m == STORE) + { + m = DEFLATE; + } +#endif + + /* Open file to zip up unless it is stdin */ + if (strcmp(z->name, "-") == 0) + { + ifile = (ftype)zstdin; +#if defined(MSDOS) || defined(__human68k__) + if (isatty(zstdin) == 0) /* keep default mode if stdin is a terminal */ + setmode(zstdin, O_BINARY); +#endif + z->tim = tim; + } + else + { +#if !(defined(VMS) && defined(VMS_PK_EXTRA)) + if (extra_fields) { + /* create extra field and change z->att and z->atx if desired */ + set_extra_field(z, &f_utim); +# ifdef QLZIP + if(qlflag) + a |= (S_IXUSR) << 16; /* Cross compilers don't set this */ +# endif +# ifdef RISCOS + m = special != NULL && filetypes(z->extra, special) ? STORE : method; +# endif /* RISCOS */ + + /* For now allow store for testing */ +#ifdef NO_STREAMING_STORE + /* For now force deflation if using data descriptors. */ + if (use_descriptors && m == STORE) + { + m = DEFLATE; + } +#endif + + } +#endif /* !(VMS && VMS_PK_EXTRA) */ + l = issymlnk(a); + if (l) { + ifile = fbad; + m = STORE; + } + else if (isdir) { /* directory */ + ifile = fbad; + m = STORE; + q = 0; + } +#ifdef THEOS + else if (((a >> 16) & S_IFMT) == S_IFLIB) { /* library */ + ifile = fbad; + m = STORE; + q = 0; + } +#endif + else { +#ifdef CMS_MVS + if (bflag) { + if ((ifile = zopen(z->name, fhowb)) == fbad) + return ZE_OPEN; + } + else +#endif /* CMS_MVS */ +#if defined(UNICODE_SUPPORT) && defined(WIN32) + if (!no_win32_wide) { + if ((ifile = zwopen(z->namew, fhow)) == fbad) + return ZE_OPEN; + } else { + if ((ifile = zopen(z->name, fhow)) == fbad) + return ZE_OPEN; + } +#else + if ((ifile = zopen(z->name, fhow)) == fbad) + return ZE_OPEN; +#endif + } + + z->tim = tim; + +#if defined(VMS) && defined(VMS_PK_EXTRA) + /* vms_get_attributes must be called after vms_open() */ + if (extra_fields) { + /* create extra field and change z->att and z->atx if desired */ + vms_get_attributes(ifile, z, &f_utim); + } +#endif /* VMS && VMS_PK_EXTRA */ + +#if defined(MMAP) || defined(BIG_MEM) + /* Map ordinary files but not devices. This code should go in fileio.c */ + if (!translate_eol && m != STORE && q != -1L && (ulg)q > 0 && + (ulg)q + MIN_LOOKAHEAD > (ulg)q) { +# ifdef MMAP + /* Map the whole input file in memory */ + if (window != NULL) + free(window); /* window can't be a mapped file here */ + window_size = (ulg)q + MIN_LOOKAHEAD; + remain = window_size & (PAGESIZE-1); + /* If we can't touch the page beyond the end of file, we must + * allocate an extra page. + */ + if (remain > MIN_LOOKAHEAD) { + window = (uch*)mmap(0, window_size, PROT_READ, MAP_PRIVATE, ifile, 0); + } else { + window = (uch*)valloc(window_size - remain + PAGESIZE); + if (window != NULL) { + window = (uch*)mmap((char*)window, window_size - remain, PROT_READ, + MAP_PRIVATE | MAP_FIXED, ifile, 0); + } else { + window = (uch*)(-1); + } + } + if (window == (uch*)(-1)) { + Trace((mesg, " mmap failure on %s\n", z->name)); + window = NULL; + window_size = 0L; + remain = (ulg)-1L; + } else { + remain = (ulg)q; + } +# else /* !MMAP, must be BIG_MEM */ + /* Read the whole input file at once */ + window_size = (ulg)q + MIN_LOOKAHEAD; + window = window ? (uch*) realloc(window, (unsigned)window_size) + : (uch*) malloc((unsigned)window_size); + /* Just use normal code if big malloc or realloc fails: */ + if (window != NULL) { + remain = (ulg)zread(ifile, (char*)window, q+1); + if (remain != (ulg)q) { + fprintf(mesg, " q=%lu, remain=%lu ", (ulg)q, remain); + error("can't read whole file at once"); + } + } else { + window_size = 0L; + } +# endif /* ?MMAP */ + } +#endif /* MMAP || BIG_MEM */ + + } /* strcmp(z->name, "-") == 0 */ + + if (extra_fields == 2) { + unsigned len; + char *p; + + /* step through old extra fields and copy over any not already + in new extra fields */ + p = copy_nondup_extra_fields(tempextra, tempext, z->extra, z->ext, &len); + free(z->extra); + z->ext = len; + z->extra = p; + p = copy_nondup_extra_fields(tempcextra, tempcext, z->cextra, z->cext, &len); + free(z->cextra); + z->cext = len; + z->cextra = p; + + if (tempext) + free(tempextra); + if (tempcext) + free(tempcextra); + } + + if (q == 0) + m = STORE; + if (m == BEST) + m = DEFLATE; + + /* Do not create STORED files with extended local headers if the + * input size is not known, because such files could not be extracted. + * So if the zip file is not seekable and the input file is not + * on disk, obey the -0 option by forcing deflation with stored block. + * Note however that using "zip -0" as filter is not very useful... + * ??? to be done. + */ + + /* An alternative used by others is to allow storing but on reading do + * a second check when a signature is found. This is simply to check + * the compressed size to the bytes read since the start of the file data. + * If this is the right signature then the compressed size should match + * the size of the compressed data to that point. If not look for the + * next signature. We should do this. 12/31/04 EG + * + * For reading and testing we should do this, but should not write + * stored streamed data unless for testing as finding the end of + * streamed deflated data can be done by inflating. 6/26/06 EG + */ + + /* Fill in header information and write local header to zip file. + * This header will later be re-written since compressed length and + * crc are not yet known. + */ + + /* (Assume ext, cext, com, and zname already filled in.) */ +#if defined(OS2) || defined(WIN32) +# ifdef WIN32_OEM + /* When creating OEM-coded names on Win32, the entries must always be marked + as "created on MSDOS" (OS_CODE = 0), because UnZip needs to handle archive + entry names just like those created by Zip's MSDOS port. + */ + z->vem = (ush)(dosify ? 20 : 0 + Z_MAJORVER * 10 + Z_MINORVER); +# else + z->vem = (ush)(z->dosflag ? (dosify ? 20 : /* Made under MSDOS by PKZIP 2.0 */ + (0 + Z_MAJORVER * 10 + Z_MINORVER)) + : OS_CODE + Z_MAJORVER * 10 + Z_MINORVER); + /* For a plain old (8+3) FAT file system, we cheat and pretend that the file + * was not made on OS2/WIN32 but under DOS. unzip is confused otherwise. + */ +# endif +#else /* !(OS2 || WIN32) */ + z->vem = (ush)(dosify ? 20 : OS_CODE + Z_MAJORVER * 10 + Z_MINORVER); +#endif /* ?(OS2 || WIN32) */ + + z->ver = (ush)(m == STORE ? 10 : 20); /* Need PKUNZIP 2.0 except for store */ +#ifdef BZIP2_SUPPORT + if (method == BZIP2) + z->ver = (ush)(m == STORE ? 10 : 46); +#endif + z->crc = 0; /* to be updated later */ + /* Assume first that we will need an extended local header: */ + if (isdir) + /* If dir then q = 0 and extended header not needed */ + z->flg = 0; + else + z->flg = 8; /* to be updated later */ +#if CRYPT + if (!isdir && key != NULL) { + z->flg |= 1; + /* Since we do not yet know the crc here, we pretend that the crc + * is the modification time: + */ + z->crc = z->tim << 16; + /* More than pretend. File is encrypted using crypt header with that. */ + } +#endif /* CRYPT */ + z->lflg = z->flg; + z->how = (ush)m; /* may be changed later */ + z->siz = (zoff_t)(m == STORE && q >= 0 ? q : 0); /* will be changed later */ + z->len = (zoff_t)(q != -1L ? q : 0); /* may be changed later */ + if (z->att == (ush)UNKNOWN) { + z->att = BINARY; /* set sensible value in header */ + set_type = 1; + } + /* Attributes from filetime(), flag bits from set_extra_field(): */ +#if defined(DOS) || defined(OS2) || defined(WIN32) + z->atx = z->dosflag ? a & 0xff : a | (z->atx & 0x0000ff00); +#else + z->atx = dosify ? a & 0xff : a | (z->atx & 0x0000ff00); +#endif /* DOS || OS2 || WIN32 */ + + if ((r = putlocal(z, PUTLOCAL_WRITE)) != ZE_OK) { + if (ifile != fbad) + zclose(ifile); + return r; + } + + /* now get split information set by bfwrite() */ + z->off = current_local_offset; + + /* disk local header was written to */ + z->dsk = current_local_disk; + + tempzn += 4 + LOCHEAD + z->nam + z->ext; + + +#if CRYPT + if (!isdir && key != NULL) { + crypthead(key, z->crc); + z->siz += RAND_HEAD_LEN; /* to be updated later */ + tempzn += RAND_HEAD_LEN; + } +#endif /* CRYPT */ + if (ferror(y)) { + if (ifile != fbad) + zclose(ifile); + ZIPERR(ZE_WRITE, "unexpected error on zip file"); + } + + last_o = o; + o = zftello(y); /* for debugging only, ftell can fail on pipes */ + if (ferror(y)) + clearerr(y); + + if (o != -1 && last_o > o) { + fprintf(mesg, "last %s o %s\n", zip_fzofft(last_o, NULL, NULL), + zip_fzofft(o, NULL, NULL)); + ZIPERR(ZE_BIG, "seek wrap - zip file too big to write"); + } + + /* Write stored or deflated file to zip file */ + isize = 0L; + crc = CRCVAL_INITIAL; + + if (isdir) { + /* nothing to write */ + } + else if (m != STORE) { + if (set_type) z->att = (ush)UNKNOWN; + /* ... is finally set in file compression routine */ +#ifdef BZIP2_SUPPORT + if (m == BZIP2) { + s = bzfilecompress(z, &m); + } + else +#endif /* BZIP2_SUPPORT */ + { + s = filecompress(z, &m); + } +#ifndef PGP + if (z->att == (ush)BINARY && translate_eol && file_binary) { + if (translate_eol == 1) + zipwarn("has binary so -l ignored", ""); + else + zipwarn("has binary so -ll ignored", ""); + } + else if (z->att == (ush)BINARY && translate_eol) { + if (translate_eol == 1) + zipwarn("-l used on binary file - corrupted?", ""); + else + zipwarn("-ll used on binary file - corrupted?", ""); + } +#endif + } + else + { + if ((b = malloc(SBSZ)) == NULL) + return ZE_MEM; + + if (l) { + k = rdsymlnk(z->name, b, SBSZ); +/* + * compute crc first because zfwrite will alter the buffer b points to !! + */ + crc = crc32(crc, (uch *) b, k); + if (zfwrite(b, 1, k) != k) + { + free((zvoid *)b); + return ZE_TEMP; + } + isize = k; + +#ifdef MINIX + q = k; +#endif /* MINIX */ + } + else + { + while ((k = file_read(b, SBSZ)) > 0 && k != (extent) EOF) + { + if (zfwrite(b, 1, k) != k) + { + if (ifile != fbad) + zclose(ifile); + free((zvoid *)b); + return ZE_TEMP; + } + if (!display_globaldots) { + if (dot_size > 0) { + /* initial space */ + if (noisy && dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + dot_count++; + } + dot_count++; + if (dot_size <= (dot_count + 1) * SBSZ) dot_count = 0; + } + if ((verbose || noisy) && dot_size && !dot_count) { +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + } + } + free((zvoid *)b); + s = isize; + } + if (ifile != fbad && zerr(ifile)) { + perror("\nzip warning"); + if (logfile) + fprintf(logfile, "\nzip warning: %s\n", strerror(errno)); + zipwarn("could not read input file: ", z->oname); + } + if (ifile != fbad) + zclose(ifile); +#ifdef MMAP + if (remain != (ulg)-1L) { + munmap((void*) window, window_size); + window = NULL; + } +#endif /*MMAP */ + + tempzn += s; + p = tempzn; /* save for future fseek() */ + +#if (!defined(MSDOS) || defined(OS2)) +#if !defined(VMS) && !defined(CMS_MVS) && !defined(__mpexl) + /* Check input size (but not in VMS -- variable record lengths mess it up) + * and not on MSDOS -- diet in TSR mode reports an incorrect file size) + */ +#ifndef TANDEM /* Tandem EOF does not match byte count unless Unstructured */ + if (!translate_eol && q != -1L && isize != q) + { + Trace((mesg, " i=%lu, q=%lu ", isize, q)); + zipwarn(" file size changed while zipping ", z->name); + } +#endif /* !TANDEM */ +#endif /* !VMS && !CMS_MVS && !__mpexl */ +#endif /* (!MSDOS || OS2) */ + + if (isdir) + { + /* A directory */ + z->siz = 0; + z->len = 0; + z->how = STORE; + z->ver = 10; + /* never encrypt directory so don't need extended local header */ + z->flg &= ~8; + z->lflg &= ~8; + } + else + { + /* Try to rewrite the local header with correct information */ + z->crc = crc; + z->siz = s; +#if CRYPT + if (!isdir && key != NULL) + z->siz += RAND_HEAD_LEN; +#endif /* CRYPT */ + z->len = isize; + /* if can seek back to local header */ +#ifdef BROKEN_FSEEK + if (use_descriptors || !fseekable(y) || zfseeko(y, z->off, SEEK_SET)) +#else + if (use_descriptors || zfseeko(y, z->off, SEEK_SET)) +#endif + { + if (z->how != (ush) m) + error("can't rewrite method"); + if (m == STORE && q < 0) + ZIPERR(ZE_PARMS, "zip -0 not supported for I/O on pipes or devices"); + if ((r = putextended(z)) != ZE_OK) + return r; + /* if Zip64 and not seekable then Zip64 data descriptor */ +#ifdef ZIP64_SUPPORT + tempzn += (zip64_entry ? 24L : 16L); +#else + tempzn += 16L; +#endif + z->flg = z->lflg; /* if z->flg modified by deflate */ + } else { + /* ftell() not as useful across splits */ + if (bytes_this_entry != (uzoff_t)(key ? s + 12 : s)) { + fprintf(mesg, " s=%s, actual=%s ", + zip_fzofft(s, NULL, NULL), zip_fzofft(bytes_this_entry, NULL, NULL)); + error("incorrect compressed size"); + } +#if 0 + /* seek ok, ftell() should work, check compressed size */ +# if !defined(VMS) && !defined(CMS_MVS) + if (p - o != s) { + fprintf(mesg, " s=%s, actual=%s ", + zip_fzofft(s, NULL, NULL), zip_fzofft(p-o, NULL, NULL)); + error("incorrect compressed size"); + } +# endif /* !VMS && !CMS_MVS */ +#endif /* 0 */ + z->how = (ush)m; + switch (m) + { + case STORE: + z->ver = 10; break; + /* Need PKUNZIP 2.0 for DEFLATE */ + case DEFLATE: + z->ver = 20; break; +#ifdef BZIP2_SUPPORT + case BZIP2: + z->ver = 46; break; +#endif + } + /* + * The encryption header needs the crc, but we don't have it + * for a new file. The file time is used instead and the encryption + * header then used to encrypt the data. The AppNote standard only + * can be applied to a file that the crc is known, so that means + * either an existing entry in an archive or get the crc before + * creating the encryption header and then encrypt the data. + */ + if ((z->flg & 1) == 0) { + /* not encrypting so don't need extended local header */ + z->flg &= ~8; + } + /* deflate may have set compression level bit markers in z->flg, + and we can't think of any reason central and local flags should + be different. */ + z->lflg = z->flg; + + /* If not using descriptors, back up and rewrite local header. */ + if (split_method == 1 && current_local_file != y) { + if (zfseeko(current_local_file, z->off, SEEK_SET)) + return ZE_READ; + } + + /* if local header in another split, putlocal will close it */ + if ((r = putlocal(z, PUTLOCAL_REWRITE)) != ZE_OK) + return r; + + if (zfseeko(y, bytes_this_split, SEEK_SET)) + return ZE_READ; + + if ((z->flg & 1) != 0) { + /* encrypted file, extended header still required */ + if ((r = putextended(z)) != ZE_OK) + return r; +#ifdef ZIP64_SUPPORT + if (zip64_entry) + tempzn += 24L; + else + tempzn += 16L; +#else + tempzn += 16L; +#endif + } + } + } /* isdir */ + /* Free the local extra field which is no longer needed */ + if (z->ext) { + if (z->extra != z->cextra) { + free((zvoid *)(z->extra)); + z->extra = NULL; + } + z->ext = 0; + } + + /* Display statistics */ + if (noisy) + { + if (verbose) { + fprintf( mesg, "\t(in=%s) (out=%s)", + zip_fzofft(isize, NULL, "u"), zip_fzofft(s, NULL, "u")); + } +#ifdef BZIP2_SUPPORT + if (m == BZIP2) + fprintf(mesg, " (bzipped %d%%)\n", percent(isize, s)); + else +#endif + if (m == DEFLATE) + fprintf(mesg, " (deflated %d%%)\n", percent(isize, s)); + else + fprintf(mesg, " (stored 0%%)\n"); + mesg_line_started = 0; + fflush(mesg); + } + if (logall) + { +#ifdef BZIP2_SUPPORT + if (m == BZIP2) + fprintf(logfile, " (bzipped %d%%)\n", percent(isize, s)); + else +#endif + if (m == DEFLATE) + fprintf(logfile, " (deflated %d%%)\n", percent(isize, s)); + else + fprintf(logfile, " (stored 0%%)\n"); + logfile_line_started = 0; + fflush(logfile); + } + +#ifdef WINDLL +# ifdef ZIP64_SUPPORT + /* The DLL api has been updated and uses a different + interface. 7/24/04 EG */ + if (lpZipUserFunctions->ServiceApplication64 != NULL) + { + if ((*lpZipUserFunctions->ServiceApplication64)(z->zname, isize)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + else + { + filesize64 = isize; + low = (unsigned long)(filesize64 & 0x00000000FFFFFFFF); + high = (unsigned long)((filesize64 >> 32) & 0x00000000FFFFFFFF); + if (lpZipUserFunctions->ServiceApplication64_No_Int64 != NULL) { + if ((*lpZipUserFunctions->ServiceApplication64_No_Int64)(z->zname, low, high)) + ZIPERR(ZE_ABORT, "User terminated operation"); + } + } +# else + if (lpZipUserFunctions->ServiceApplication != NULL) + { + if ((*lpZipUserFunctions->ServiceApplication)(z->zname, isize)) + { + ZIPERR(ZE_ABORT, "User terminated operation"); + } + } +# endif +#endif + + return ZE_OK; +} + + + + +local unsigned file_read(buf, size) + char *buf; + unsigned size; +/* Read a new buffer from the current input file, perform end-of-line + * translation, and update the crc and input file size. + * IN assertion: size >= 2 (for end-of-line translation) + */ +{ + unsigned len; + char *b; + zoff_t isize_prev; /* Previous isize. Used for overflow check. */ + +#if defined(MMAP) || defined(BIG_MEM) + if (remain == 0L) { + return 0; + } else if (remain != (ulg)-1L) { + /* The window data is already in place. We still compute the crc + * by 32K blocks instead of once on whole file to keep a certain + * locality of reference. + */ + Assert(buf == (char*)window + isize, "are you lost?"); + if ((ulg)size > remain) size = (unsigned)remain; + if (size > WSIZE) size = WSIZE; /* don't touch all pages at once */ + remain -= (ulg)size; + len = size; + } else +#endif /* MMAP || BIG_MEM */ + if (translate_eol == 0) { + len = zread(ifile, buf, size); + if (len == (unsigned)EOF || len == 0) return len; +#ifdef OS390 + b = buf; + if (aflag == ASCII) { + while (*b != '\0') { + *b = (char)ascii[(uch)*b]; + b++; + } + } +#endif + } else if (translate_eol == 1) { + /* translate_eol == 1 */ + /* Transform LF to CR LF */ + size >>= 1; + b = buf+size; + size = len = zread(ifile, b, size); + if (len == (unsigned)EOF || len == 0) return len; + + /* check buf for binary - 12/16/04 */ + if (file_binary == -1) { + /* first read */ + file_binary = is_text_buf(b, size) ? 0 : 1; + } + + if (file_binary != 1) { +#ifdef EBCDIC + if (aflag == ASCII) + { + do { + char c; + + if ((c = *b++) == '\n') { + *buf++ = CR; *buf++ = LF; len++; + } else { + *buf++ = (char)ascii[(uch)c]; + } + } while (--size != 0); + } + else +#endif /* EBCDIC */ + { + do { + if ((*buf++ = *b++) == '\n') *(buf-1) = CR, *buf++ = LF, len++; + } while (--size != 0); + } + buf -= len; + } else { /* do not translate binary */ + memcpy(buf, b, size); + } + + } else { + /* translate_eol == 2 */ + /* Transform CR LF to LF and suppress final ^Z */ + b = buf; + size = len = zread(ifile, buf, size-1); + if (len == (unsigned)EOF || len == 0) return len; + + /* check buf for binary - 12/16/04 */ + if (file_binary == -1) { + /* first read */ + file_binary = is_text_buf(b, size) ? 0 : 1; + } + + if (file_binary != 1) { + buf[len] = '\n'; /* I should check if next char is really a \n */ +#ifdef EBCDIC + if (aflag == ASCII) + { + do { + char c; + + if ((c = *b++) == '\r' && *b == '\n') { + len--; + } else { + *buf++ = (char)(c == '\n' ? LF : ascii[(uch)c]); + } + } while (--size != 0); + } + else +#endif /* EBCDIC */ + { + do { + if (( *buf++ = *b++) == CR && *b == LF) buf--, len--; + } while (--size != 0); + } + if (len == 0) { + zread(ifile, buf, 1); len = 1; /* keep single \r if EOF */ +#ifdef EBCDIC + if (aflag == ASCII) { + *buf = (char)(*buf == '\n' ? LF : ascii[(uch)(*buf)]); + } +#endif + } else { + buf -= len; + if (buf[len-1] == CTRLZ) len--; /* suppress final ^Z */ + } + } + } + crc = crc32(crc, (uch *) buf, len); + /* 2005-05-23 SMS. + Increment file size. A small-file program reading a large file may + cause isize to overflow, so complain (and abort) if it goes + negative or wraps around. Awful things happen later otherwise. + */ + isize_prev = isize; + isize += (ulg)len; + if (isize < isize_prev) { + ZIPERR(ZE_BIG, "overflow in byte count"); + } + return len; +} + + +#ifdef USE_ZLIB + +local int zl_deflate_init(pack_level) + int pack_level; +{ + unsigned i; + int windowBits; + int err = Z_OK; + int zp_err = ZE_OK; + + if (zlib_version[0] != ZLIB_VERSION[0]) { + sprintf(errbuf, "incompatible zlib version (expected %s, found %s)", + ZLIB_VERSION, zlib_version); + zp_err = ZE_LOGIC; + } else if (strcmp(zlib_version, ZLIB_VERSION) != 0) { + fprintf(mesg, + "\twarning: different zlib version (expected %s, using %s)\n", + ZLIB_VERSION, zlib_version); + } + + /* windowBits = log2(WSIZE) */ + for (i = ((unsigned)WSIZE), windowBits = 0; i != 1; i >>= 1, ++windowBits); + + zstrm.zalloc = (alloc_func)Z_NULL; + zstrm.zfree = (free_func)Z_NULL; + + Trace((stderr, "initializing deflate()\n")); + err = deflateInit2(&zstrm, pack_level, Z_DEFLATED, -windowBits, 8, 0); + + if (err == Z_MEM_ERROR) { + sprintf(errbuf, "cannot initialize zlib deflate"); + zp_err = ZE_MEM; + } else if (err != Z_OK) { + sprintf(errbuf, "zlib deflateInit failure (%d)", err); + zp_err = ZE_LOGIC; + } + + deflInit = TRUE; + return zp_err; +} + + +void zl_deflate_free() +{ + int err; + + if (f_obuf != NULL) { + free(f_obuf); + f_obuf = NULL; + } + if (f_ibuf != NULL) { + free(f_ibuf); + f_ibuf = NULL; + } + if (deflInit) { + err = deflateEnd(&zstrm); + if (err != Z_OK && err !=Z_DATA_ERROR) { + ziperr(ZE_LOGIC, "zlib deflateEnd failed"); + } + deflInit = FALSE; + } +} + +#else /* !USE_ZLIB */ + +# ifdef ZP_NEED_MEMCOMPR +/* =========================================================================== + * In-memory read function. As opposed to file_read(), this function + * does not perform end-of-line translation, and does not update the + * crc and input size. + * Note that the size of the entire input buffer is an unsigned long, + * but the size used in mem_read() is only an unsigned int. This makes a + * difference on 16 bit machines. mem_read() may be called several + * times for an in-memory compression. + */ +local unsigned mem_read(b, bsize) + char *b; + unsigned bsize; +{ + if (in_offset < in_size) { + ulg block_size = in_size - in_offset; + if (block_size > (ulg)bsize) block_size = (ulg)bsize; + memcpy(b, in_buf + in_offset, (unsigned)block_size); + in_offset += (unsigned)block_size; + return (unsigned)block_size; + } else { + return 0; /* end of input */ + } +} +# endif /* ZP_NEED_MEMCOMPR */ + + +/* =========================================================================== + * Flush the current output buffer. + */ +void flush_outbuf(o_buf, o_idx) + char *o_buf; + unsigned *o_idx; +{ + if (y == NULL) { + error("output buffer too small for in-memory compression"); + } + /* Encrypt and write the output buffer: */ + if (*o_idx != 0) { + zfwrite(o_buf, 1, (extent)*o_idx); + if (ferror(y)) ziperr(ZE_WRITE, "write error on zip file"); + } + *o_idx = 0; +} + +/* =========================================================================== + * Return true if the zip file can be seeked. This is used to check if + * the local header can be re-rewritten. This function always returns + * true for in-memory compression. + * IN assertion: the local header has already been written (ftell() > 0). + */ +int seekable() +{ + return fseekable(y); +} +#endif /* ?USE_ZLIB */ + + +/* =========================================================================== + * Compression to archive file. + */ +local zoff_t filecompress(z_entry, cmpr_method) + struct zlist far *z_entry; + int *cmpr_method; +{ +#ifdef USE_ZLIB + int err = Z_OK; + unsigned mrk_cnt = 1; + int maybe_stored = FALSE; + ulg cmpr_size; +#if defined(MMAP) || defined(BIG_MEM) + unsigned ibuf_sz = (unsigned)SBSZ; +#else +# define ibuf_sz ((unsigned)SBSZ) +#endif +#ifndef OBUF_SZ +# define OBUF_SZ ZBSZ +#endif + unsigned u; + +#if defined(MMAP) || defined(BIG_MEM) + if (remain == (ulg)-1L && f_ibuf == NULL) +#else /* !(MMAP || BIG_MEM */ + if (f_ibuf == NULL) +#endif /* MMAP || BIG_MEM */ + f_ibuf = (char *)malloc(SBSZ); + if (f_obuf == NULL) + f_obuf = (char *)malloc(OBUF_SZ); +#if defined(MMAP) || defined(BIG_MEM) + if ((remain == (ulg)-1L && f_ibuf == NULL) || f_obuf == NULL) +#else /* !(MMAP || BIG_MEM */ + if (f_ibuf == NULL || f_obuf == NULL) +#endif /* MMAP || BIG_MEM */ + ziperr(ZE_MEM, "allocating zlib file-I/O buffers"); + + if (!deflInit) { + err = zl_deflate_init(level); + if (err != ZE_OK) + ziperr(err, errbuf); + } + + if (level <= 2) { + z_entry->flg |= 4; + } else if (level >= 8) { + z_entry->flg |= 2; + } +#if defined(MMAP) || defined(BIG_MEM) + if (remain != (ulg)-1L) { + zstrm.next_in = (Bytef *)window; + ibuf_sz = (unsigned)WSIZE; + } else +#endif /* MMAP || BIG_MEM */ + { + zstrm.next_in = (Bytef *)f_ibuf; + } + zstrm.avail_in = file_read(zstrm.next_in, ibuf_sz); + if (zstrm.avail_in < ibuf_sz) { + unsigned more = file_read(zstrm.next_in + zstrm.avail_in, + (ibuf_sz - zstrm.avail_in)); + if (more == EOF || more == 0) { + maybe_stored = TRUE; + } else { + zstrm.avail_in += more; + } + } + zstrm.next_out = (Bytef *)f_obuf; + zstrm.avail_out = OBUF_SZ; + + if (!maybe_stored) while (zstrm.avail_in != 0 && zstrm.avail_in != EOF) { + err = deflate(&zstrm, Z_NO_FLUSH); + if (err != Z_OK && err != Z_STREAM_END) { + sprintf(errbuf, "unexpected zlib deflate error %d", err); + ziperr(ZE_LOGIC, errbuf); + } + if (zstrm.avail_out == 0) { + if (zfwrite(f_obuf, 1, OBUF_SZ) != OBUF_SZ) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + zstrm.next_out = (Bytef *)f_obuf; + zstrm.avail_out = OBUF_SZ; + } + if (zstrm.avail_in == 0) { + if (verbose || noisy) + while((unsigned)(zstrm.total_in / (uLong)WSIZE) > mrk_cnt) { + mrk_cnt++; + if (!display_globaldots) { + if (dot_size > 0) { + /* initial space */ + if (noisy && dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + dot_count++; + } + dot_count++; + if (dot_size <= (dot_count + 1) * WSIZE) dot_count = 0; + } + if (noisy && dot_size && !dot_count) { +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + } +#if defined(MMAP) || defined(BIG_MEM) + if (remain == (ulg)-1L) + zstrm.next_in = (Bytef *)f_ibuf; +#else + zstrm.next_in = (Bytef *)f_ibuf; +#endif + zstrm.avail_in = file_read(zstrm.next_in, ibuf_sz); + } + } + + do { + err = deflate(&zstrm, Z_FINISH); + if (maybe_stored) { + if (err == Z_STREAM_END && zstrm.total_out >= zstrm.total_in && + fseekable(zipfile)) { + /* deflation does not reduce size, switch to STORE method */ + unsigned len_out = (unsigned)zstrm.total_in; + if (zfwrite(f_ibuf, 1, len_out) != len_out) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + zstrm.total_out = (uLong)len_out; + *cmpr_method = STORE; + break; + } else { + maybe_stored = FALSE; + } + } + if (zstrm.avail_out < OBUF_SZ) { + unsigned len_out = OBUF_SZ - zstrm.avail_out; + if (zfwrite(f_obuf, 1, len_out) != len_out) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + zstrm.next_out = (Bytef *)f_obuf; + zstrm.avail_out = OBUF_SZ; + } + } while (err == Z_OK); + + if (err != Z_STREAM_END) { + sprintf(errbuf, "unexpected zlib deflate error %d", err); + ziperr(ZE_LOGIC, errbuf); + } + + if (z_entry->att == (ush)UNKNOWN) + z_entry->att = (ush)(zstrm.data_type == Z_ASCII ? ASCII : BINARY); + cmpr_size = (ulg)zstrm.total_out; + + if ((err = deflateReset(&zstrm)) != Z_OK) + ziperr(ZE_LOGIC, "zlib deflateReset failed"); + return cmpr_size; +#else /* !USE_ZLIB */ + + /* Set the defaults for file compression. */ + read_buf = file_read; + + /* Initialize deflate's internals and execute file compression. */ + bi_init(file_outbuf, sizeof(file_outbuf), TRUE); + ct_init(&z_entry->att, cmpr_method); + lm_init(level, &z_entry->flg); + return deflate(); +#endif /* ?USE_ZLIB */ +} + +#ifdef ZP_NEED_MEMCOMPR +/* =========================================================================== + * In-memory compression. This version can be used only if the entire input + * fits in one memory buffer. The compression is then done in a single + * call of memcompress(). (An extension to allow repeated calls would be + * possible but is not needed here.) + * The first two bytes of the compressed output are set to a short with the + * method used (DEFLATE or STORE). The following four bytes contain the CRC. + * The values are stored in little-endian order on all machines. + * This function returns the byte size of the compressed output, including + * the first six bytes (method and crc). + */ + +ulg memcompress(tgt, tgtsize, src, srcsize) + char *tgt, *src; /* target and source buffers */ + ulg tgtsize, srcsize; /* target and source sizes */ +{ + ulg crc; + unsigned out_total; + int method = DEFLATE; +#ifdef USE_ZLIB + int err = Z_OK; +#else + ush att = (ush)UNKNOWN; + ush flags = 0; +#endif + + if (tgtsize <= (ulg)6L) error("target buffer too small"); + out_total = 2 + 4; + +#ifdef USE_ZLIB + if (!deflInit) { + err = zl_deflate_init(level); + if (err != ZE_OK) + ziperr(err, errbuf); + } + + zstrm.next_in = (Bytef *)src; + zstrm.avail_in = (uInt)srcsize; + zstrm.next_out = (Bytef *)(tgt + out_total); + zstrm.avail_out = (uInt)tgtsize - (uInt)out_total; + + err = deflate(&zstrm, Z_FINISH); + if (err != Z_STREAM_END) + error("output buffer too small for in-memory compression"); + out_total += (unsigned)zstrm.total_out; + + if ((err = deflateReset(&zstrm)) != Z_OK) + error("zlib deflateReset failed"); +#else /* !USE_ZLIB */ + read_buf = mem_read; + in_buf = src; + in_size = (unsigned)srcsize; + in_offset = 0; + window_size = 0L; + + bi_init(tgt + (2 + 4), (unsigned)(tgtsize - (2 + 4)), FALSE); + ct_init(&att, &method); + lm_init((level != 0 ? level : 1), &flags); + out_total += (unsigned)deflate(); + window_size = 0L; /* was updated by lm_init() */ +#endif /* ?USE_ZLIB */ + + crc = CRCVAL_INITIAL; + crc = crc32(crc, (uch *)src, (extent)srcsize); + + /* For portability, force little-endian order on all machines: */ + tgt[0] = (char)(method & 0xff); + tgt[1] = (char)((method >> 8) & 0xff); + tgt[2] = (char)(crc & 0xff); + tgt[3] = (char)((crc >> 8) & 0xff); + tgt[4] = (char)((crc >> 16) & 0xff); + tgt[5] = (char)((crc >> 24) & 0xff); + + return (ulg)out_total; +} +#endif /* ZP_NEED_MEMCOMPR */ + +#ifdef BZIP2_SUPPORT + +local int bz_compress_init(pack_level) +int pack_level; +{ + int err = BZ_OK; + int zp_err = ZE_OK; + const char *bzlibVer; + + bzlibVer = BZ2_bzlibVersion(); + + /* $TODO - Check BZIP2 LIB version? */ + + bstrm.bzalloc = NULL; + bstrm.bzfree = NULL; + bstrm.opaque = NULL; + + Trace((stderr, "initializing bzlib compress()\n")); + err = BZ2_bzCompressInit(&bstrm, pack_level, 0, 30); + + if (err == BZ_MEM_ERROR) { + sprintf(errbuf, "cannot initialize bzlib compress"); + zp_err = ZE_MEM; + } else if (err != BZ_OK) { + sprintf(errbuf, "bzlib bzCompressInit failure (%d)", err); + zp_err = ZE_LOGIC; + } + + bzipInit = TRUE; + return zp_err; +} + +void bz_compress_free() +{ + int err; + + if (f_obuf != NULL) { + free(f_obuf); + f_obuf = NULL; + } + if (f_ibuf != NULL) { + free(f_ibuf); + f_ibuf = NULL; + } + if (bzipInit) { + err = BZ2_bzCompressEnd(&bstrm); + if (err != BZ_OK && err != BZ_DATA_ERROR) { + ziperr(ZE_LOGIC, "bzlib bzCompressEnd failed"); + } + bzipInit = FALSE; + } +} + +/* =========================================================================== + * BZIP2 Compression to archive file. + */ + +local zoff_t bzfilecompress(z_entry, cmpr_method) +struct zlist far *z_entry; +int *cmpr_method; +{ + FILE *zipfile = y; + + int err = BZ_OK; + unsigned mrk_cnt = 1; + int maybe_stored = FALSE; + zoff_t cmpr_size; +#if defined(MMAP) || defined(BIG_MEM) + unsigned ibuf_sz = (unsigned)SBSZ; +#else +# define ibuf_sz ((unsigned)SBSZ) +#endif +#ifndef OBUF_SZ +# define OBUF_SZ ZBSZ +#endif + +#if defined(MMAP) || defined(BIG_MEM) + if (remain == (ulg)-1L && f_ibuf == NULL) +#else /* !(MMAP || BIG_MEM */ + if (f_ibuf == NULL) +#endif /* MMAP || BIG_MEM */ + f_ibuf = (char *)malloc(SBSZ); + if (f_obuf == NULL) + f_obuf = (char *)malloc(OBUF_SZ); +#if defined(MMAP) || defined(BIG_MEM) + if ((remain == (ulg)-1L && f_ibuf == NULL) || f_obuf == NULL) +#else /* !(MMAP || BIG_MEM */ + if (f_ibuf == NULL || f_obuf == NULL) +#endif /* MMAP || BIG_MEM */ + ziperr(ZE_MEM, "allocating zlib/bzlib file-I/O buffers"); + + if (!bzipInit) { + err = bz_compress_init(level); + if (err != ZE_OK) + ziperr(err, errbuf); + } + +#if defined(MMAP) || defined(BIG_MEM) + if (remain != (ulg)-1L) { + bstrm.next_in = (Bytef *)window; + ibuf_sz = (unsigned)WSIZE; + } else +#endif /* MMAP || BIG_MEM */ + { + bstrm.next_in = (char *)f_ibuf; + } + bstrm.avail_in = file_read(bstrm.next_in, ibuf_sz); + if (file_binary_final == 0) { + /* check for binary as library does not */ + if (!is_text_buf(bstrm.next_in, ibuf_sz)) + file_binary_final = 1; + } + if (bstrm.avail_in < ibuf_sz) { + unsigned more = file_read(bstrm.next_in + bstrm.avail_in, + (ibuf_sz - bstrm.avail_in)); + if (more == (unsigned) EOF || more == 0) { + maybe_stored = TRUE; + } else { + bstrm.avail_in += more; + } + } + bstrm.next_out = (char *)f_obuf; + bstrm.avail_out = OBUF_SZ; + + if (!maybe_stored) { + while (bstrm.avail_in != 0 && bstrm.avail_in != (unsigned) EOF) { + err = BZ2_bzCompress(&bstrm, BZ_RUN); + if (err != BZ_RUN_OK && err != BZ_STREAM_END) { + sprintf(errbuf, "unexpected bzlib compress error %d", err); + ziperr(ZE_LOGIC, errbuf); + } + if (bstrm.avail_out == 0) { + if (zfwrite(f_obuf, 1, OBUF_SZ) != OBUF_SZ) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + bstrm.next_out = (char *)f_obuf; + bstrm.avail_out = OBUF_SZ; + } + /* $TODO what about high 32-bits of total-in??? */ + if (bstrm.avail_in == 0) { + if (verbose || noisy) +#ifdef LARGE_FILE_SUPPORT + while((unsigned)((bstrm.total_in_lo32 + + (((zoff_t)bstrm.total_in_hi32) << 32)) + / (zoff_t)(ulg)WSIZE) > mrk_cnt) { +#else + while((unsigned)(bstrm.total_in_lo32 / (ulg)WSIZE) > mrk_cnt) { +#endif + mrk_cnt++; + if (!display_globaldots) { + if (dot_size > 0) { + /* initial space */ + if (noisy && dot_count == -1) { +#ifndef WINDLL + putc(' ', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",' '); +#endif + dot_count++; + } + dot_count++; + if (dot_size <= (dot_count + 1) * WSIZE) dot_count = 0; + } + if (noisy && dot_size && !dot_count) { +#ifndef WINDLL + putc('.', mesg); + fflush(mesg); +#else + fprintf(stdout,"%c",'.'); +#endif + mesg_line_started = 1; + } + } + } +#if defined(MMAP) || defined(BIG_MEM) + if (remain == (ulg)-1L) + bstrm.next_in = (char *)f_ibuf; +#else + bstrm.next_in = (char *)f_ibuf; +#endif + bstrm.avail_in = file_read(bstrm.next_in, ibuf_sz); + if (file_binary_final == 0) { + /* check for binary as library does not */ + if (!is_text_buf(bstrm.next_in, ibuf_sz)) + file_binary_final = 1; + } + } + } + } + + /* binary or text */ + if (file_binary_final) + /* found binary in file */ + z_entry->att = (ush)BINARY; + else + /* text file */ + z_entry->att = (ush)ASCII; + + do { + err = BZ2_bzCompress(&bstrm, BZ_FINISH); + if (maybe_stored) { + /* This code is only executed when the complete data stream fits + into the input buffer (see above where maybe_stored gets set). + So, it is safe to assume that total_in_hi32 (and total_out_hi32) + are 0, because the input buffer size is well below the 32-bit + limit. + */ + if (err == BZ_STREAM_END + && bstrm.total_out_lo32 >= bstrm.total_in_lo32 + && fseekable(zipfile)) { + /* BZIP2 compress does not reduce size, + switch to STORE method */ + unsigned len_out = (unsigned)bstrm.total_in_lo32; + if (zfwrite(f_ibuf, 1, len_out) != len_out) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + bstrm.total_out_lo32 = (ulg)len_out; + *cmpr_method = STORE; + break; + } else { + maybe_stored = FALSE; + } + } + if (bstrm.avail_out < OBUF_SZ) { + unsigned len_out = OBUF_SZ - bstrm.avail_out; + if (zfwrite(f_obuf, 1, len_out) != len_out) { + ziperr(ZE_TEMP, "error writing to zipfile"); + } + bstrm.next_out = (char *)f_obuf; + bstrm.avail_out = OBUF_SZ; + } + } while (err == BZ_FINISH_OK); + + if (err < BZ_OK) { + sprintf(errbuf, "unexpected bzlib compress error %d", err); + ziperr(ZE_LOGIC, errbuf); + } + + if (z_entry->att == (ush)UNKNOWN) + z_entry->att = (ush)BINARY; +#ifdef LARGE_FILE_SUPPORT + cmpr_size = (zoff_t)bstrm.total_out_lo32 + + (((zoff_t)bstrm.total_out_hi32) << 32); +#else + cmpr_size = (zoff_t)bstrm.total_out_lo32; +#endif + + if ((err = BZ2_bzCompressEnd(&bstrm)) != BZ_OK) + ziperr(ZE_LOGIC, "zlib deflateReset failed"); + bzipInit = FALSE; + return cmpr_size; +} + +#endif /* BZIP2_SUPPORT */ +#endif /* !UTIL */ diff --git a/third_party/lua/luaconf.h b/third_party/lua/luaconf.h index f5578c07b..e36d0eb0c 100644 --- a/third_party/lua/luaconf.h +++ b/third_party/lua/luaconf.h @@ -219,15 +219,13 @@ #else /* }{ */ -#define LUA_ROOT "/usr/local/" -#define LUA_LDIR LUA_ROOT "share/lua/" LUA_VDIR "/" -#define LUA_CDIR LUA_ROOT "lib/lua/" LUA_VDIR "/" +#define LUA_ROOT "zip:" +#define LUA_LDIR LUA_ROOT ".lua/" +#define LUA_CDIR LUA_ROOT ".lua/" #if !defined(LUA_PATH_DEFAULT) #define LUA_PATH_DEFAULT \ - LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" \ - LUA_CDIR"?.lua;" LUA_CDIR"?/init.lua;" \ - "./?.lua;" "./?/init.lua" + LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua" #endif #if !defined(LUA_CPATH_DEFAULT) diff --git a/third_party/quickjs/quickjs-libc.c b/third_party/quickjs/quickjs-libc.c index cea3b1bad..c9bd6e3c5 100644 --- a/third_party/quickjs/quickjs-libc.c +++ b/third_party/quickjs/quickjs-libc.c @@ -1463,7 +1463,7 @@ static textstartup void js_std_error_props_init() { DEF(EBADF), #undef DEF }; - _Static_assert(sizeof(js_std_error_props) == sizeof(props)); + _Static_assert(sizeof(js_std_error_props) == sizeof(props), ""); memcpy(js_std_error_props, props, sizeof(props)); } const void *const js_std_error_props_ctor[] initarray = {js_std_error_props_init}; @@ -3592,7 +3592,7 @@ static textstartup void js_os_funcs_init() { JS_CFUNC_DEF("dup2", 2, js_os_dup2 ), #endif }; - _Static_assert(sizeof(js_os_funcs) == sizeof(funcs)); + _Static_assert(sizeof(js_os_funcs) == sizeof(funcs), ""); memcpy(js_os_funcs, funcs, sizeof(funcs)); } const void *const js_os_funcs_ctor[] initarray = {js_os_funcs_init}; diff --git a/third_party/quickjs/quickjs.c b/third_party/quickjs/quickjs.c index 1935e0cec..7a0cc47d9 100644 --- a/third_party/quickjs/quickjs.c +++ b/third_party/quickjs/quickjs.c @@ -10731,7 +10731,7 @@ static int JS_ToInt64SatFree(JSContext *ctx, int64_t *pres, JSValue val) } else { if (d < INT64_MIN) *pres = INT64_MIN; - else if (d > INT64_MAX) + else if (d > (double)INT64_MAX) *pres = INT64_MAX; else *pres = (int64_t)d; diff --git a/third_party/regex/regerror.c b/third_party/regex/regerror.c index b9762b1c1..152fb630d 100644 --- a/third_party/regex/regerror.c +++ b/third_party/regex/regerror.c @@ -47,7 +47,7 @@ static const char kRegexErrors[] = * @return number of bytes needed to hold entire string */ size_t regerror(int e, const regex_t *preg, char *buf, size_t size) { - return 1 + snprintf(buf, size, "%s", - firstnonnull(IndexDoubleNulString(kRegexErrors, e), - "Unknown error")); + return 1 + (snprintf)(buf, size, "%s", + firstnonnull(IndexDoubleNulString(kRegexErrors, e), + "Unknown error")); } diff --git a/third_party/sqlite3/alter.c b/third_party/sqlite3/alter.c new file mode 100644 index 000000000..e65a28453 --- /dev/null +++ b/third_party/sqlite3/alter.c @@ -0,0 +1,1995 @@ +/* +** 2005 February 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that used to generate VDBE code +** that implements the ALTER TABLE command. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** The code in this file only exists if we are not omitting the +** ALTER TABLE logic from the build. +*/ +#ifndef SQLITE_OMIT_ALTERTABLE + +/* +** Parameter zName is the name of a table that is about to be altered +** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). +** If the table is a system table, this function leaves an error message +** in pParse->zErr (system tables may not be altered) and returns non-zero. +** +** Or, if zName is not a system table, zero is returned. +*/ +static int isAlterableTable(Parse *pParse, Table *pTab){ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) +#ifndef SQLITE_OMIT_VIRTUALTABLE + || ( (pTab->tabFlags & TF_Shadow)!=0 + && sqlite3ReadOnlyShadowTables(pParse->db) + ) +#endif + ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); + return 1; + } + return 0; +} + +/* +** Generate code to verify that the schemas of database zDb and, if +** bTemp is not true, database "temp", can still be parsed. This is +** called at the end of the generation of an ALTER TABLE ... RENAME ... +** statement to ensure that the operation has not rendered any schema +** objects unusable. +*/ +static void renameTestSchema( + Parse *pParse, /* Parse context */ + const char *zDb, /* Name of db to verify schema of */ + int bTemp, /* True if this is the temp db */ + const char *zWhen, /* "when" part of error message */ + const char *zDropColumn /* Name of column being dropped */ +){ + pParse->colNamesSet = 1; + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM \"%w\"." DFLT_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, %d, %Q, %Q)=NULL ", + zDb, + zDb, bTemp, zWhen, zDropColumn + ); + + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM temp." DFLT_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, 1, %Q, %Q)=NULL ", + zDb, zWhen, zDropColumn + ); + } +} + +/* +** Generate code to reload the schema for database iDb. And, if iDb!=1, for +** the temp database as well. +*/ +static void renameReloadSchema(Parse *pParse, int iDb, u16 p5){ + Vdbe *v = pParse->pVdbe; + if( v ){ + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0, p5); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0, p5); + } +} + +/* +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. +*/ +void sqlite3AlterRenameTable( + Parse *pParse, /* Parser context. */ + SrcList *pSrc, /* The table to rename. */ + Token *pName /* The new table name. */ +){ + int iDb; /* Database that contains the table */ + char *zDb; /* Name of database iDb */ + Table *pTab; /* Table being renamed */ + char *zName = 0; /* NULL-terminated version of pName */ + sqlite3 *db = pParse->db; /* Database connection */ + int nTabName; /* Number of UTF-8 characters in zTabName */ + const char *zTabName; /* Original name of the table */ + Vdbe *v; + VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ + u32 savedDbFlags; /* Saved value of db->mDbFlags */ + + savedDbFlags = db->mDbFlags; + if( NEVER(db->mallocFailed) ) goto exit_rename_table; + assert( pSrc->nSrc==1 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_table; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + db->mDbFlags |= DBFLAG_PreferBuiltin; + + /* Get a NULL terminated version of the new table name. */ + zName = sqlite3NameFromToken(db, pName); + if( !zName ) goto exit_rename_table; + + /* Check that a table or index named 'zName' does not already exist + ** in database iDb. If so, this is an error. + */ + if( sqlite3FindTable(db, zName, zDb) + || sqlite3FindIndex(db, zName, zDb) + || sqlite3IsShadowTableOf(db, pTab, zName) + ){ + sqlite3ErrorMsg(pParse, + "there is already another table or index with this name: %s", zName); + goto exit_rename_table; + } + + /* Make sure it is not a system table being altered, or a reserved name + ** that the table is being renamed to. + */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ + goto exit_rename_table; + } + if( SQLITE_OK!=sqlite3CheckObjectName(pParse,zName,"table",zName) ){ + goto exit_rename_table; + } + +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_rename_table; + } + if( IsVirtual(pTab) ){ + pVTab = sqlite3GetVTable(db, pTab); + if( pVTab->pVtab->pModule->xRename==0 ){ + pVTab = 0; + } + } +#endif + + /* Begin a transaction for database iDb. Then modify the schema cookie + ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** nested SQL may raise an exception. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto exit_rename_table; + } + sqlite3MayAbort(pParse); + + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); + + /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in + ** the schema to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " + "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" + "AND name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + , zDb, zDb, zTabName, zName, (iDb==1), zTabName + ); + + /* Update the tbl_name and name columns of the sqlite_schema table + ** as required. */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE " SET " + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqliteX_autoindex%%' ESCAPE 'X' " + " AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q COLLATE nocase AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, + zName, zName, zName, + nTabName, zTabName + ); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + + /* If the table being renamed is not itself part of the temp database, + ** edit view and trigger definitions within the temp database + ** as required. */ + if( iDb!=1 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_schema SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " + "tbl_name = " + "CASE WHEN tbl_name=%Q COLLATE nocase AND " + " sqlite_rename_test(%Q, sql, type, name, 1, 'after rename',0) " + "THEN %Q ELSE tbl_name END " + "WHERE type IN ('view', 'trigger')" + , zDb, zTabName, zName, zTabName, zDb, zName); + } + + /* If this is a virtual table, invoke the xRename() function if + ** one is defined. The xRename() callback will modify the names + ** of any resources used by the v-table implementation (including other + ** SQLite tables) that are identified by the name of the virtual table. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pVTab ){ + int i = ++pParse->nMem; + sqlite3VdbeLoadString(v, i, zName); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); + } +#endif + + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iDb==1, "after rename", 0); + +exit_rename_table: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zName); + db->mDbFlags = savedDbFlags; +} + +/* +** Write code that will raise an error if the table described by +** zDb and zTab is not empty. +*/ +static void sqlite3ErrorIfNotEmpty( + Parse *pParse, /* Parsing context */ + const char *zDb, /* Schema holding the table */ + const char *zTab, /* Table to check for empty */ + const char *zErr /* Error message text */ +){ + sqlite3NestedParse(pParse, + "SELECT raise(ABORT,%Q) FROM \"%w\".\"%w\"", + zErr, zDb, zTab + ); +} + +/* +** This function is called after an "ALTER TABLE ... ADD" statement +** has been parsed. Argument pColDef contains the text of the new +** column definition. +** +** The Table structure pParse->pNewTable was extended to include +** the new column during parsing. +*/ +void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ + Table *pNew; /* Copy of pParse->pNewTable */ + Table *pTab; /* Table being altered */ + int iDb; /* Database number */ + const char *zDb; /* Database name */ + const char *zTab; /* Table name */ + char *zCol; /* Null-terminated column definition */ + Column *pCol; /* The new column */ + Expr *pDflt; /* Default value for the new column */ + sqlite3 *db; /* The database connection; */ + Vdbe *v; /* The prepared statement under construction */ + int r1; /* Temporary registers */ + + db = pParse->db; + if( pParse->nErr || db->mallocFailed ) return; + pNew = pParse->pNewTable; + assert( pNew ); + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pNew->pSchema); + zDb = db->aDb[iDb].zDbSName; + zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ + pCol = &pNew->aCol[pNew->nCol-1]; + pDflt = pCol->pDflt; + pTab = sqlite3FindTable(db, zTab, zDb); + assert( pTab ); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + return; + } +#endif + + + /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. + ** If there is a NOT NULL constraint, then the default value for the + ** column must not be NULL. + */ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); + return; + } + if( pNew->pIndex ){ + sqlite3ErrorMsg(pParse, + "Cannot add a UNIQUE column"); + return; + } + if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ + pDflt = 0; + } + if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a REFERENCES column with non-NULL default value"); + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a NOT NULL column with default value NULL"); + } + + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); + return; + } + if( !pVal ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a column with non-constant default"); + } + sqlite3ValueFree(pVal); + } + }else if( pCol->colFlags & COLFLAG_STORED ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, "cannot add a STORED column"); + } + + + /* Modify the CREATE TABLE statement. */ + zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); + if( zCol ){ + char *zEnd = &zCol[pColDef->n-1]; + u32 savedDbFlags = db->mDbFlags; + while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ + *zEnd-- = '\0'; + } + db->mDbFlags |= DBFLAG_PreferBuiltin; + /* substr() operations on characters, but addColOffset is in bytes. So we + ** have to use printf() to translate between these units: */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = printf('%%.%ds, ',sql) || %Q" + " || substr(sql,1+length(printf('%%.%ds',sql))) " + "WHERE type = 'table' AND name = %Q", + zDb, pNew->addColOffset, zCol, pNew->addColOffset, + zTab + ); + sqlite3DbFree(db, zCol); + db->mDbFlags = savedDbFlags; + } + + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. + */ + v = sqlite3GetVdbe(pParse); + if( v ){ + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); + } + + /* Reload the table definition */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); +} + +/* +** This function is called by the parser after the table-name in +** an "ALTER TABLE ADD" statement is parsed. Argument +** pSrc is the full-name of the table being altered. +** +** This routine makes a (partial) copy of the Table structure +** for the table being altered and sets Parse.pNewTable to point +** to it. Routines called by the parser as the column definition +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c +** after parsing is finished. +** +** Routine sqlite3AlterFinishAddColumn() will be called to complete +** coding the "ALTER TABLE ... ADD" statement. +*/ +void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ + Table *pNew; + Table *pTab; + int iDb; + int i; + int nAlloc; + sqlite3 *db = pParse->db; + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( db->mallocFailed ) goto exit_begin_add_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_begin_add_column; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be altered"); + goto exit_begin_add_column; + } +#endif + + /* Make sure this is not an attempt to ALTER a view. */ + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); + goto exit_begin_add_column; + } + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ + goto exit_begin_add_column; + } + + sqlite3MayAbort(pParse); + assert( pTab->addColOffset>0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + + /* Put a copy of the Table struct in Parse.pNewTable for the + ** sqlite3AddColumn() function and friends to modify. But modify + ** the name by adding an "sqlite_altertab_" prefix. By adding this + ** prefix, we insure that the name will not collide with an existing + ** table because user table are not allowed to have the "sqlite_" + ** prefix on their name. + */ + pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); + if( !pNew ) goto exit_begin_add_column; + pParse->pNewTable = pNew; + pNew->nTabRef = 1; + pNew->nCol = pTab->nCol; + assert( pNew->nCol>0 ); + nAlloc = (((pNew->nCol-1)/8)*8)+8; + assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); + if( !pNew->aCol || !pNew->zName ){ + assert( db->mallocFailed ); + goto exit_begin_add_column; + } + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + for(i=0; inCol; i++){ + Column *pCol = &pNew->aCol[i]; + pCol->zName = sqlite3DbStrDup(db, pCol->zName); + pCol->hName = sqlite3StrIHash(pCol->zName); + pCol->zColl = 0; + pCol->pDflt = 0; + } + pNew->pSchema = db->aDb[iDb].pSchema; + pNew->addColOffset = pTab->addColOffset; + pNew->nTabRef = 1; + +exit_begin_add_column: + sqlite3SrcListDelete(db, pSrc); + return; +} + +/* +** Parameter pTab is the subject of an ALTER TABLE ... RENAME COLUMN +** command. This function checks if the table is a view or virtual +** table (columns of views or virtual tables may not be renamed). If so, +** it loads an error message into pParse and returns non-zero. +** +** Or, if pTab is not a view or virtual table, zero is returned. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ + const char *zType = 0; +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + zType = "view"; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + zType = "virtual table"; + } +#endif + if( zType ){ + sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", + (bDrop ? "drop column from" : "rename columns of"), + zType, pTab->zName + ); + return 1; + } + return 0; +} +#else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +# define isRealTable(x,y,z) (0) +#endif + +/* +** Handles the following parser reduction: +** +** cmd ::= ALTER TABLE pSrc RENAME COLUMN pOld TO pNew +*/ +void sqlite3AlterRenameColumn( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Table being altered. pSrc->nSrc==1 */ + Token *pOld, /* Name of column being changed */ + Token *pNew /* New column name */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table being updated */ + int iCol; /* Index of column being renamed */ + char *zOld = 0; /* Old column name */ + char *zNew = 0; /* New column name */ + const char *zDb; /* Name of schema containing the table */ + int iSchema; /* Index of the schema */ + int bQuote; /* True to quote the new name */ + + /* Locate the table to be altered */ + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_column; + + /* Cannot alter a system table */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 0) ) goto exit_rename_column; + + /* Which schema holds the table to be altered */ + iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iSchema>=0 ); + zDb = db->aDb[iSchema].zDbSName; + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_column; + } +#endif + + /* Make sure the old name really is a column name in the table to be + ** altered. Set iCol to be the index of the column being renamed */ + zOld = sqlite3NameFromToken(db, pOld); + if( !zOld ) goto exit_rename_column; + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3StrICmp(pTab->aCol[iCol].zName, zOld) ) break; + } + if( iCol==pTab->nCol ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld); + goto exit_rename_column; + } + + /* Do the rename operation using a recursive UPDATE statement that + ** uses the sqlite_rename_column() SQL function to compute the new + ** CREATE statement text for the sqlite_schema table. + */ + sqlite3MayAbort(pParse); + zNew = sqlite3NameFromToken(db, pNew); + if( !zNew ) goto exit_rename_column; + assert( pNew->n>0 ); + bQuote = sqlite3Isquote(pNew->z[0]); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " + " AND (type != 'index' OR tbl_name = %Q)" + " AND sql NOT LIKE 'create virtual%%'", + zDb, + zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, + pTab->zName + ); + + sqlite3NestedParse(pParse, + "UPDATE temp." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + zDb, pTab->zName, iCol, zNew, bQuote + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iSchema, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iSchema==1, "after rename", 0); + + exit_rename_column: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zOld); + sqlite3DbFree(db, zNew); + return; +} + +/* +** Each RenameToken object maps an element of the parse tree into +** the token that generated that element. The parse tree element +** might be one of: +** +** * A pointer to an Expr that represents an ID +** * The name of a table column in Column.zName +** +** A list of RenameToken objects can be constructed during parsing. +** Each new object is created by sqlite3RenameTokenMap(). +** As the parse tree is transformed, the sqlite3RenameTokenRemap() +** routine is used to keep the mapping current. +** +** After the parse finishes, renameTokenFind() routine can be used +** to look up the actual token value that created some element in +** the parse tree. +*/ +struct RenameToken { + void *p; /* Parse tree element created by token t */ + Token t; /* The token that created parse tree element p */ + RenameToken *pNext; /* Next is a list of all RenameToken objects */ +}; + +/* +** The context of an ALTER TABLE RENAME COLUMN operation that gets passed +** down into the Walker. +*/ +typedef struct RenameCtx RenameCtx; +struct RenameCtx { + RenameToken *pList; /* List of tokens to overwrite */ + int nList; /* Number of tokens in pList */ + int iCol; /* Index of column being renamed */ + Table *pTab; /* Table being ALTERed */ + const char *zOld; /* Old column name */ +}; + +#ifdef SQLITE_DEBUG +/* +** This function is only for debugging. It performs two tasks: +** +** 1. Checks that pointer pPtr does not already appear in the +** rename-token list. +** +** 2. Dereferences each pointer in the rename-token list. +** +** The second is most effective when debugging under valgrind or +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking +** tool. +** +** The point of this is to prevent comparisons of invalid pointer values. +** Even though this always seems to work, it is undefined according to the +** C standard. Example of undefined comparison: +** +** sqlite3_free(x); +** if( x==y ) ... +** +** Technically, as x no longer points into a valid object or to the byte +** following a valid object, it may not be used in comparison operations. +*/ +static void renameTokenCheckAll(Parse *pParse, void *pPtr){ + if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){ + RenameToken *p; + u8 i = 0; + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p ){ + assert( p->p!=pPtr ); + i += *(u8*)(p->p); + } + } + } +} +#else +# define renameTokenCheckAll(x,y) +#endif + +/* +** Remember that the parser tree element pPtr was created using +** the token pToken. +** +** In other words, construct a new RenameToken object and add it +** to the list of RenameToken objects currently being built up +** in pParse->pRename. +** +** The pPtr argument is returned so that this routine can be used +** with tail recursion in tokenExpr() routine, for a small performance +** improvement. +*/ +void *sqlite3RenameTokenMap(Parse *pParse, void *pPtr, Token *pToken){ + RenameToken *pNew; + assert( pPtr || pParse->db->mallocFailed ); + renameTokenCheckAll(pParse, pPtr); + if( ALWAYS(pParse->eParseMode!=PARSE_MODE_UNMAP) ){ + pNew = sqlite3DbMallocZero(pParse->db, sizeof(RenameToken)); + if( pNew ){ + pNew->p = pPtr; + pNew->t = *pToken; + pNew->pNext = pParse->pRename; + pParse->pRename = pNew; + } + } + + return pPtr; +} + +/* +** It is assumed that there is already a RenameToken object associated +** with parse tree element pFrom. This function remaps the associated token +** to parse tree element pTo. +*/ +void sqlite3RenameTokenRemap(Parse *pParse, void *pTo, void *pFrom){ + RenameToken *p; + renameTokenCheckAll(pParse, pTo); + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p==pFrom ){ + p->p = pTo; + break; + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + return WRC_Continue; +} + +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + With *pWith = pSelect->pWith; + if( pWith ){ + int i; + for(i=0; inCte; i++){ + Select *p = pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pWalker->pParse; + sqlite3SelectPrep(sNC.pParse, p, &sNC); + sqlite3WalkSelect(pWalker, p); + sqlite3RenameExprlistUnmap(pWalker->pParse, pWith->a[i].pCols); + } + } +} + +/* +** Unmap all tokens in the IdList object passed as the second argument. +*/ +static void unmapColumnIdlistNames( + Parse *pParse, + IdList *pIdList +){ + if( pIdList ){ + int ii; + for(ii=0; iinId; ii++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pIdList->a[ii].zName); + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapSelectCb(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i; + if( pParse->nErr ) return WRC_Abort; + if( NEVER(p->selFlags & SF_View) ) return WRC_Prune; + if( ALWAYS(p->pEList) ){ + ExprList *pList = p->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zEName && pList->a[i].eEName==ENAME_NAME ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName); + } + } + } + if( ALWAYS(p->pSrc) ){ /* Every Select as a SrcList, even if it is empty */ + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); + if( sqlite3WalkExpr(pWalker, pSrc->a[i].pOn) ) return WRC_Abort; + unmapColumnIdlistNames(pParse, pSrc->a[i].pUsing); + } + } + + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** Remove all nodes that are part of expression pExpr from the rename list. +*/ +void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ + u8 eMode = pParse->eParseMode; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sWalker.xSelectCallback = renameUnmapSelectCb; + pParse->eParseMode = PARSE_MODE_UNMAP; + sqlite3WalkExpr(&sWalker, pExpr); + pParse->eParseMode = eMode; +} + +/* +** Remove all nodes that are part of expression-list pEList from the +** rename list. +*/ +void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ + if( pEList ){ + int i; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExprList(&sWalker, pEList); + for(i=0; inExpr; i++){ + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName); + } + } + } +} + +/* +** Free the list of RenameToken objects given in the second argument +*/ +static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ + RenameToken *pNext; + RenameToken *p; + for(p=pToken; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Search the Parse object passed as the first argument for a RenameToken +** object associated with parse tree element pPtr. If found, return a pointer +** to it. Otherwise, return NULL. +** +** If the second argument passed to this function is not NULL and a matching +** RenameToken object is found, remove it from the Parse object and add it to +** the list maintained by the RenameCtx object. +*/ +static RenameToken *renameTokenFind( + Parse *pParse, + struct RenameCtx *pCtx, + void *pPtr +){ + RenameToken **pp; + assert( pPtr!=0 ); + for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ + if( (*pp)->p==pPtr ){ + RenameToken *pToken = *pp; + if( pCtx ){ + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + } + return pToken; + } + } + return 0; +} + +/* +** This is a Walker select callback. It does nothing. It is only required +** because without a dummy callback, sqlite3WalkExpr() and similar do not +** descend into sub-select statements. +*/ +static int renameColumnSelectCb(Walker *pWalker, Select *p){ + if( p->selFlags & SF_View ) return WRC_Prune; + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** This is a Walker expression callback. +** +** For every TK_COLUMN node in the expression tree, search to see +** if the column being references is the column being renamed by an +** ALTER TABLE statement. If it is, then attach its associated +** RenameToken object to the list of RenameToken objects being +** constructed in RenameCtx object at pWalker->u.pRename. +*/ +static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol + && pWalker->pParse->pTriggerTab==p->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + }else if( pExpr->op==TK_COLUMN + && pExpr->iColumn==p->iCol + && p->pTab==pExpr->y.pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + } + return WRC_Continue; +} + +/* +** The RenameCtx contains a list of tokens that reference a column that +** is being renamed by an ALTER TABLE statement. Return the "last" +** RenameToken in the RenameCtx and remove that RenameToken from the +** RenameContext. "Last" means the last RenameToken encountered when +** the input SQL is parsed from left to right. Repeated calls to this routine +** return all column name tokens in the order that they are encountered +** in the SQL statement. +*/ +static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ + RenameToken *pBest = pCtx->pList; + RenameToken *pToken; + RenameToken **pp; + + for(pToken=pBest->pNext; pToken; pToken=pToken->pNext){ + if( pToken->t.z>pBest->t.z ) pBest = pToken; + } + for(pp=&pCtx->pList; *pp!=pBest; pp=&(*pp)->pNext); + *pp = pBest->pNext; + + return pBest; +} + +/* +** An error occured while parsing or otherwise processing a database +** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an +** ALTER TABLE RENAME COLUMN program. The error message emitted by the +** sub-routine is currently stored in pParse->zErrMsg. This function +** adds context to the error message and then stores it in pCtx. +*/ +static void renameColumnParseError( + sqlite3_context *pCtx, + const char *zWhen, + sqlite3_value *pType, + sqlite3_value *pObject, + Parse *pParse +){ + const char *zT = (const char*)sqlite3_value_text(pType); + const char *zN = (const char*)sqlite3_value_text(pObject); + char *zErr; + + zErr = sqlite3_mprintf("error in %s %s%s%s: %s", + zT, zN, (zWhen[0] ? " " : ""), zWhen, + pParse->zErrMsg + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); +} + +/* +** For each name in the the expression-list pEList (i.e. each +** pEList->a[i].zName) that matches the string in zOld, extract the +** corresponding rename-token from Parse object pParse and add it +** to the RenameCtx pCtx. +*/ +static void renameColumnElistNames( + Parse *pParse, + RenameCtx *pCtx, + ExprList *pEList, + const char *zOld +){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + char *zName = pEList->a[i].zEName; + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) + && ALWAYS(zName!=0) + && 0==sqlite3_stricmp(zName, zOld) + ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + +/* +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token +** from Parse object pParse and add it to the RenameCtx pCtx. +*/ +static void renameColumnIdlistNames( + Parse *pParse, + RenameCtx *pCtx, + IdList *pIdList, + const char *zOld +){ + if( pIdList ){ + int i; + for(i=0; inId; i++){ + char *zName = pIdList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + + +/* +** Parse the SQL statement zSql using Parse object (*p). The Parse object +** is initialized by this function before it is used. +*/ +static int renameParseSql( + Parse *p, /* Memory to use for Parse object */ + const char *zDb, /* Name of schema SQL belongs to */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL to parse */ + int bTemp, /* True if SQL is from temp schema */ + const char *zDropColumn /* Name of column being dropped */ +){ + int rc; + char *zErr = 0; + + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + if( zDropColumn ){ + db->init.bDropColumn = 1; + db->init.azInit = (char**)&zDropColumn; + } + + /* Parse the SQL statement passed as the first argument. If no error + ** occurs and the parse does not result in a new table, index or + ** trigger object, the database must be corrupt. */ + memset(p, 0, sizeof(Parse)); + p->eParseMode = PARSE_MODE_RENAME; + p->db = db; + p->nQueryLoop = 1; + rc = zSql ? sqlite3RunParser(p, zSql, &zErr) : SQLITE_NOMEM; + assert( p->zErrMsg==0 ); + assert( rc!=SQLITE_OK || zErr==0 ); + p->zErrMsg = zErr; + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK + && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + ){ + rc = SQLITE_CORRUPT_BKPT; + } + +#ifdef SQLITE_DEBUG + /* Ensure that all mappings in the Parse.pRename list really do map to + ** a part of the input string. */ + if( rc==SQLITE_OK ){ + int nSql = sqlite3Strlen30(zSql); + RenameToken *pToken; + for(pToken=p->pRename; pToken; pToken=pToken->pNext){ + assert( pToken->t.z>=zSql && &pToken->t.z[pToken->t.n]<=&zSql[nSql] ); + } + } +#endif + + db->init.iDb = 0; + db->init.bDropColumn = 0; + return rc; +} + +/* +** This function edits SQL statement zSql, replacing each token identified +** by the linked list pRename with the text of zNew. If argument bQuote is +** true, then zNew is always quoted first. If no error occurs, the result +** is loaded into context object pCtx as the result. +** +** Or, if an error occurs (i.e. an OOM condition), an error is left in +** pCtx and an SQLite error code returned. +*/ +static int renameEditSql( + sqlite3_context *pCtx, /* Return result here */ + RenameCtx *pRename, /* Rename context */ + const char *zSql, /* SQL statement to edit */ + const char *zNew, /* New token text */ + int bQuote /* True to always quote token */ +){ + int nNew = sqlite3Strlen30(zNew); + int nSql = sqlite3Strlen30(zSql); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = SQLITE_OK; + char *zQuot; + char *zOut; + int nQuot; + + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original + ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to + ** point to zQuot so that all substitutions are made using the + ** quoted version of the new column name. */ + zQuot = sqlite3MPrintf(db, "\"%w\"", zNew); + if( zQuot==0 ){ + return SQLITE_NOMEM; + }else{ + nQuot = sqlite3Strlen30(zQuot); + } + if( bQuote ){ + zNew = zQuot; + nNew = nQuot; + } + + /* At this point pRename->pList contains a list of RenameToken objects + ** corresponding to all tokens in the input SQL that must be replaced + ** with the new column name. All that remains is to construct and + ** return the edited SQL string. */ + assert( nQuot>=nNew ); + zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + if( zOut ){ + int nOut = nSql; + memcpy(zOut, zSql, nSql); + while( pRename->pList ){ + int iOff; /* Offset of token to replace in zOut */ + RenameToken *pBest = renameColumnTokenNext(pRename); + + u32 nReplace; + const char *zReplace; + if( sqlite3IsIdChar(*pBest->t.z) ){ + nReplace = nNew; + zReplace = zNew; + }else{ + nReplace = nQuot; + zReplace = zQuot; + } + + iOff = pBest->t.z - zSql; + if( pBest->t.n!=nReplace ){ + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + nOut - (iOff + pBest->t.n) + ); + nOut += nReplace - pBest->t.n; + zOut[nOut] = '\0'; + } + memcpy(&zOut[iOff], zReplace, nReplace); + sqlite3DbFree(db, pBest); + } + + sqlite3_result_text(pCtx, zOut, -1, SQLITE_TRANSIENT); + sqlite3DbFree(db, zOut); + }else{ + rc = SQLITE_NOMEM; + } + + sqlite3_free(zQuot); + return rc; +} + +/* +** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming +** it was read from the schema of database zDb. Return SQLITE_OK if +** successful. Otherwise, return an SQLite error code and leave an error +** message in the Parse object. +*/ +static int renameResolveTrigger(Parse *pParse){ + sqlite3 *db = pParse->db; + Trigger *pNew = pParse->pNewTrigger; + TriggerStep *pStep; + NameContext sNC; + int rc = SQLITE_OK; + + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + assert( pNew->pTabSchema ); + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName + ); + pParse->eTriggerOp = pNew->op; + /* ALWAYS() because if the table of the trigger does not exist, the + ** error would have been hit before this point */ + if( ALWAYS(pParse->pTriggerTab) ){ + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + } + + /* Resolve symbols in WHEN clause */ + if( rc==SQLITE_OK && pNew->pWhen ){ + rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); + } + + for(pStep=pNew->step_list; rc==SQLITE_OK && pStep; pStep=pStep->pNext){ + if( pStep->pSelect ){ + sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); + if( pParse->nErr ) rc = pParse->rc; + } + if( rc==SQLITE_OK && pStep->zTarget ){ + SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( pSrc ){ + int i; + for(i=0; inSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pSrc->a[i]; + p->iCursor = pParse->nTab++; + if( p->pSelect ){ + sqlite3SelectPrep(pParse, p->pSelect, 0); + sqlite3ExpandSubquery(pParse, p); + assert( i>0 ); + assert( pStep->pFrom->a[i-1].pSelect ); + sqlite3SelectPrep(pParse, pStep->pFrom->a[i-1].pSelect, 0); + }else{ + p->pTab = sqlite3LocateTableItem(pParse, 0, p); + if( p->pTab==0 ){ + rc = SQLITE_ERROR; + }else{ + p->pTab->nTabRef++; + rc = sqlite3ViewGetColumnNames(pParse, p->pTab); + } + } + } + sNC.pSrcList = pSrc; + if( rc==SQLITE_OK && pStep->pWhere ){ + rc = sqlite3ResolveExprNames(&sNC, pStep->pWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprListNames(&sNC, pStep->pExprList); + } + assert( !pStep->pUpsert || (!pStep->pWhere && !pStep->pExprList) ); + if( pStep->pUpsert && rc==SQLITE_OK ){ + Upsert *pUpsert = pStep->pUpsert; + pUpsert->pUpsertSrc = pSrc; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc==SQLITE_OK ){ + ExprList *pUpsertSet = pUpsert->pUpsertSet; + rc = sqlite3ResolveExprListNames(&sNC, pUpsertSet); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + } + sNC.ncFlags = 0; + } + sNC.pSrcList = 0; + sqlite3SrcListDelete(db, pSrc); + }else{ + rc = SQLITE_NOMEM; + } + } + } + return rc; +} + +/* +** Invoke sqlite3WalkExpr() or sqlite3WalkSelect() on all Select or Expr +** objects that are part of the trigger passed as the second argument. +*/ +static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ + TriggerStep *pStep; + + /* Find tokens to edit in WHEN clause */ + sqlite3WalkExpr(pWalker, pTrigger->pWhen); + + /* Find tokens to edit in trigger steps */ + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + sqlite3WalkSelect(pWalker, pStep->pSelect); + sqlite3WalkExpr(pWalker, pStep->pWhere); + sqlite3WalkExprList(pWalker, pStep->pExprList); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + sqlite3WalkExprList(pWalker, pUpsert->pUpsertTarget); + sqlite3WalkExprList(pWalker, pUpsert->pUpsertSet); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); + } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc; i++){ + sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + } + } + } +} + +/* +** Free the contents of Parse object (*pParse). Do not free the memory +** occupied by the Parse object itself. +*/ +static void renameParseCleanup(Parse *pParse){ + sqlite3 *db = pParse->db; + Index *pIdx; + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + while( (pIdx = pParse->pNewIndex)!=0 ){ + pParse->pNewIndex = pIdx->pNext; + sqlite3FreeIndex(db, pIdx); + } + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + sqlite3DbFree(db, pParse->zErrMsg); + renameTokenFree(db, pParse->pRename); + sqlite3ParserReset(pParse); +} + +/* +** SQL function: +** +** sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld) +** +** 0. zSql: SQL statement to rewrite +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3. Database: Database name (e.g. "main") +** 4. Table: Table name +** 5. iCol: Index of column to rename +** 6. zNew: New column name +** 7. bQuote: Non-zero if the new column name should be quoted. +** 8. bTemp: True if zSql comes from temp schema +** +** Do a column rename operation on the CREATE statement given in zSql. +** The iCol-th column (left-most is 0) of table zTable is renamed from zCol +** into zNew. The name should be quoted if bQuote is true. +** +** This function is used internally by the ALTER TABLE RENAME COLUMN command. +** It is only accessible to SQL created using sqlite3NestedParse(). It is +** not reachable from ordinary SQL passed into sqlite3_prepare(). +*/ +static void renameColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + RenameCtx sCtx; + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + const char *zDb = (const char*)sqlite3_value_text(argv[3]); + const char *zTable = (const char*)sqlite3_value_text(argv[4]); + int iCol = sqlite3_value_int(argv[5]); + const char *zNew = (const char*)sqlite3_value_text(argv[6]); + int bQuote = sqlite3_value_int(argv[7]); + int bTemp = sqlite3_value_int(argv[8]); + const char *zOld; + int rc; + Parse sParse; + Walker sWalker; + Index *pIdx; + int i; + Table *pTab; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zSql==0 ) return; + if( zTable==0 ) return; + if( zNew==0 ) return; + if( iCol<0 ) return; + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTable, zDb); + if( pTab==0 || iCol>=pTab->nCol ){ + sqlite3BtreeLeaveAll(db); + return; + } + zOld = pTab->aCol[iCol].zName; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.iCol = ((iCol==pTab->iPKey) ? -1 : iCol); + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + rc = renameParseSql(&sParse, zDb, db, zSql, bTemp, 0); + + /* Find tokens that need to be replaced. */ + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameColumnExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + sCtx.pTab = pTab; + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + if( sParse.pNewTable ){ + Select *pSelect = sParse.pNewTable->pSelect; + if( pSelect ){ + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + }else{ + /* A regular table */ + int bFKOnly = sqlite3_stricmp(zTable, sParse.pNewTable->zName); + FKey *pFKey; + assert( sParse.pNewTable->pSelect==0 ); + sCtx.pTab = sParse.pNewTable; + if( bFKOnly==0 ){ + renameTokenFind( + &sParse, &sCtx, (void*)sParse.pNewTable->aCol[iCol].zName + ); + if( sCtx.iCol<0 ){ + renameTokenFind(&sParse, &sCtx, (void*)&sParse.pNewTable->iPKey); + } + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); + for(pIdx=sParse.pNewTable->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + for(pIdx=sParse.pNewIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + sqlite3WalkExpr(&sWalker, sParse.pNewTable->aCol[i].pDflt); + } +#endif + } + + for(pFKey=sParse.pNewTable->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(i=0; inCol; i++){ + if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ + renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); + } + if( 0==sqlite3_stricmp(pFKey->zTo, zTable) + && 0==sqlite3_stricmp(pFKey->aCol[i].zCol, zOld) + ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->aCol[i].zCol); + } + } + } + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ + /* A trigger */ + TriggerStep *pStep; + rc = renameResolveTrigger(&sParse); + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + + for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pTarget==pTab ){ + if( pStep->pUpsert ){ + ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; + renameColumnElistNames(&sParse, &sCtx, pUpsertSet, zOld); + } + renameColumnIdlistNames(&sParse, &sCtx, pStep->pIdList, zOld); + renameColumnElistNames(&sParse, &sCtx, pStep->pExprList, zOld); + } + } + } + + + /* Find tokens to edit in UPDATE OF clause */ + if( sParse.pTriggerTab==pTab ){ + renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); + } + + /* Find tokens to edit in various expressions and selects */ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } + + assert( rc==SQLITE_OK ); + rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); + +renameColumnFunc_done: + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + sqlite3BtreeLeaveAll(db); +} + +/* +** Walker expression callback used by "RENAME TABLE". +*/ +static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_COLUMN && p->pTab==pExpr->y.pTab ){ + renameTokenFind(pWalker->pParse, p, (void*)&pExpr->y.pTab); + } + return WRC_Continue; +} + +/* +** Walker select callback used by "RENAME TABLE". +*/ +static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ + int i; + RenameCtx *p = pWalker->u.pRename; + SrcList *pSrc = pSelect->pSrc; + if( pSelect->selFlags & SF_View ) return WRC_Prune; + if( pSrc==0 ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } + for(i=0; inSrc; i++){ + SrcItem *pItem = &pSrc->a[i]; + if( pItem->pTab==p->pTab ){ + renameTokenFind(pWalker->pParse, p, pItem->zName); + } + } + renameWalkWith(pWalker, pSelect); + + return WRC_Continue; +} + + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 0: The database containing the table being renamed. +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3: The complete text of the schema statement being modified, +** 4: The old name of the table being renamed, and +** 5: The new name of the table being renamed. +** 6: True if the schema statement comes from the temp db. +** +** It returns the new schema statement. For example: +** +** sqlite_rename_table('main', 'CREATE TABLE t1(a REFERENCES t2)','t2','t3',0) +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDb = (const char*)sqlite3_value_text(argv[0]); + const char *zInput = (const char*)sqlite3_value_text(argv[3]); + const char *zOld = (const char*)sqlite3_value_text(argv[4]); + const char *zNew = (const char*)sqlite3_value_text(argv[5]); + int bTemp = sqlite3_value_int(argv[6]); + UNUSED_PARAMETER(NotUsed); + + if( zInput && zOld && zNew ){ + Parse sParse; + int rc; + int bQuote = 1; + RenameCtx sCtx; + Walker sWalker; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + memset(&sCtx, 0, sizeof(RenameCtx)); + sCtx.pTab = sqlite3FindTable(db, zOld, zDb); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameTableExprCb; + sWalker.xSelectCallback = renameTableSelectCb; + sWalker.u.pRename = &sCtx; + + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp, 0); + + if( rc==SQLITE_OK ){ + int isLegacy = (db->flags & SQLITE_LegacyAlter); + if( sParse.pNewTable ){ + Table *pTab = sParse.pNewTable; + + if( pTab->pSelect ){ + if( isLegacy==0 ){ + Select *pSelect = pTab->pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + + assert( pSelect->selFlags & SF_View ); + pSelect->selFlags &= ~SF_View; + sqlite3SelectPrep(&sParse, pTab->pSelect, &sNC); + if( sParse.nErr ){ + rc = sParse.rc; + }else{ + sqlite3WalkSelect(&sWalker, pTab->pSelect); + } + } + }else{ + /* Modify any FK definitions to point to the new table. */ +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( isLegacy==0 || (db->flags & SQLITE_ForeignKeys) ){ + FKey *pFKey; + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + if( sqlite3_stricmp(pFKey->zTo, zOld)==0 ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->zTo); + } + } + } +#endif + + /* If this is the table being altered, fix any table refs in CHECK + ** expressions. Also update the name that appears right after the + ** "CREATE [VIRTUAL] TABLE" bit. */ + if( sqlite3_stricmp(zOld, pTab->zName)==0 ){ + sCtx.pTab = pTab; + if( isLegacy==0 ){ + sqlite3WalkExprList(&sWalker, pTab->pCheck); + } + renameTokenFind(&sParse, &sCtx, pTab->zName); + } + } + } + + else if( sParse.pNewIndex ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewIndex->zName); + if( isLegacy==0 ){ + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + } + } + +#ifndef SQLITE_OMIT_TRIGGER + else{ + Trigger *pTrigger = sParse.pNewTrigger; + TriggerStep *pStep; + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + && sCtx.pTab->pSchema==pTrigger->pTabSchema + ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); + } + + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, pTrigger); + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + } + } + } + } +#endif + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); + } + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); + sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + } + + return; +} + +/* +** An SQL user function that checks that there are no parse or symbol +** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. +** After an ALTER TABLE .. RENAME operation is performed and the schema +** reloaded, this function is called on each SQL statement in the schema +** to ensure that it is still usable. +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement. +** 2: Object type ("view", "table", "trigger" or "index"). +** 3: Object name. +** 4: True if object is from temp schema. +** 5: "when" part of error message. +** 6: Name of column being dropped, or NULL. +** +** Unless it finds an error, this function normally returns NULL. However, it +** returns integer value 1 if: +** +** * the SQL argument creates a trigger, and +** * the table that the trigger is attached to is in database zDb. +*/ +static void renameTableTest( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + int bTemp = sqlite3_value_int(argv[4]); + int isLegacy = (db->flags & SQLITE_LegacyAlter); + char const *zWhen = (const char*)sqlite3_value_text(argv[5]); + char const *zDropColumn = (const char*)sqlite3_value_text(argv[6]); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zDb && zInput ){ + int rc; + Parse sParse; + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp, zDropColumn); + if( rc==SQLITE_OK ){ + if( isLegacy==0 && sParse.pNewTable && sParse.pNewTable->pSelect ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + sqlite3SelectPrep(&sParse, sParse.pNewTable->pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + } + + else if( sParse.pNewTrigger ){ + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + } + if( rc==SQLITE_OK ){ + int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); + int i2 = sqlite3FindDbName(db, zDb); + if( i1==i2 ) sqlite3_result_int(context, 1); + } + } + } + + if( rc!=SQLITE_OK && zWhen ){ + renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif +} + +/* +** The implementation of internal UDF sqlite_drop_column(). +** +** Arguments: +** +** argv[0]: An integer - the index of the schema containing the table +** argv[1]: CREATE TABLE statement to modify. +** argv[2]: An integer - the index of the column to remove. +** +** The value returned is a string containing the CREATE TABLE statement +** with column argv[2] removed. +*/ +static void dropColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + int iSchema = sqlite3_value_int(argv[0]); + const char *zSql = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + const char *zDb = db->aDb[iSchema].zDbSName; + int rc; + Parse sParse; + RenameToken *pCol; + Table *pTab; + const char *zEnd; + char *zNew = 0; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + rc = renameParseSql(&sParse, zDb, db, zSql, iSchema==1, 0); + if( rc!=SQLITE_OK ) goto drop_column_done; + pTab = sParse.pNewTable; + if( pTab==0 || pTab->nCol==1 || iCol>=pTab->nCol ){ + /* This can happen if the sqlite_schema table is corrupt */ + rc = SQLITE_CORRUPT_BKPT; + goto drop_column_done; + } + + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zName); + if( iColnCol-1 ){ + RenameToken *pEnd; + pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zName); + zEnd = (const char*)pEnd->t.z; + }else{ + zEnd = (const char*)&zSql[pTab->addColOffset]; + while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; + } + + zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); + sqlite3_result_text(context, zNew, -1, SQLITE_TRANSIENT); + sqlite3_free(zNew); + +drop_column_done: + renameParseCleanup(&sParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** This function is called by the parser upon parsing an +** +** ALTER TABLE pSrc DROP COLUMN pName +** +** statement. Argument pSrc contains the possibly qualified name of the +** table being edited, and token pName the name of the column to drop. +*/ +void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, Token *pName){ + sqlite3 *db = pParse->db; /* Database handle */ + Table *pTab; /* Table to modify */ + int iDb; /* Index of db containing pTab in aDb[] */ + const char *zDb; /* Database containing pTab ("main" etc.) */ + char *zCol = 0; /* Name of column to drop */ + int iCol; /* Index of column zCol in pTab->aCol[] */ + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( NEVER(db->mallocFailed) ) goto exit_drop_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_drop_column; + + /* Make sure this is not an attempt to ALTER a view, virtual table or + ** system table. */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_drop_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 1) ) goto exit_drop_column; + + /* Find the index of the column being dropped. */ + zCol = sqlite3NameFromToken(db, pName); + if( zCol==0 ){ + assert( db->mallocFailed ); + goto exit_drop_column; + } + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zCol); + goto exit_drop_column; + } + + /* Do not allow the user to drop a PRIMARY KEY column or a column + ** constrained by a UNIQUE constraint. */ + if( pTab->aCol[iCol].colFlags & (COLFLAG_PRIMKEY|COLFLAG_UNIQUE) ){ + sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", + (pTab->aCol[iCol].colFlags&COLFLAG_PRIMKEY) ? "PRIMARY KEY" : "UNIQUE", + zCol + ); + goto exit_drop_column; + } + + /* Do not allow the number of columns to go to zero */ + if( pTab->nCol<=1 ){ + sqlite3ErrorMsg(pParse, "cannot drop column \"%s\": no other columns exist",zCol); + goto exit_drop_column; + } + + /* Edit the sqlite_schema table */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + zDb = db->aDb[iDb].zDbSName; + renameTestSchema(pParse, zDb, iDb==1, "", 0); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_drop_column(%d, sql, %d) " + "WHERE (type=='table' AND tbl_name=%Q COLLATE nocase)" + , zDb, iDb, iCol, pTab->zName + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDrop); + renameTestSchema(pParse, zDb, iDb==1, "after drop column", zCol); + + /* Edit rows of table on disk */ + if( pParse->nErr==0 && (pTab->aCol[iCol].colFlags & COLFLAG_VIRTUAL)==0 ){ + int i; + int addr; + int reg; + int regRec; + Index *pPk = 0; + int nField = 0; /* Number of non-virtual columns after drop */ + int iCur; + Vdbe *v = sqlite3GetVdbe(pParse); + iCur = pParse->nTab++; + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); + addr = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + reg = ++pParse->nMem; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, reg); + pParse->nMem += pTab->nCol; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + pParse->nMem += pPk->nColumn; + for(i=0; inKeyCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, iCur, i, reg+i+1); + } + nField = pPk->nKeyCol; + } + regRec = ++pParse->nMem; + for(i=0; inCol; i++){ + if( i!=iCol && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + int regOut; + if( pPk ){ + int iPos = sqlite3TableColumnToIndex(pPk, i); + int iColPos = sqlite3TableColumnToIndex(pPk, iCol); + if( iPosnKeyCol ) continue; + regOut = reg+1+iPos-(iPos>iColPos); + }else{ + regOut = reg+1+nField; + } + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + } + nField++; + } + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg+1, nField, regRec); + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iCur, regRec, reg+1, pPk->nKeyCol); + }else{ + sqlite3VdbeAddOp3(v, OP_Insert, iCur, regRec, reg); + } + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + + sqlite3VdbeAddOp2(v, OP_Next, iCur, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + } + +exit_drop_column: + sqlite3DbFree(db, zCol); + sqlite3SrcListDelete(db, pSrc); +} + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +void sqlite3AlterFunctions(void){ + static FuncDef aAlterTableFuncs[] = { + INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), + INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), + INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), + }; + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); +} +#endif /* SQLITE_ALTER_TABLE */ diff --git a/third_party/sqlite3/analyze.c b/third_party/sqlite3/analyze.c new file mode 100644 index 000000000..c370b4da0 --- /dev/null +++ b/third_party/sqlite3/analyze.c @@ -0,0 +1,1944 @@ +/* +** 2005-07-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code associated with the ANALYZE command. +** +** The ANALYZE command gather statistics about the content of tables +** and indices. These statistics are made available to the query planner +** to help it make better decisions about how to perform queries. +** +** The following system tables are or have been supported: +** +** CREATE TABLE sqlite_stat1(tbl, idx, stat); +** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); +** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** CREATE TABLE sqlite_stat4(tbl, idx, nEq, nLt, nDLt, sample); +** +** Additional tables might be added in future releases of SQLite. +** The sqlite_stat2 table is not created or used unless the SQLite version +** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled +** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. +** The sqlite_stat2 table is superseded by sqlite_stat3, which is only +** created and used by SQLite versions 3.7.9 through 3.29.0 when +** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 +** is a superset of sqlite_stat2 and is also now deprecated. The +** sqlite_stat4 is an enhanced version of sqlite_stat3 and is only +** available when compiled with SQLITE_ENABLE_STAT4 and in SQLite +** versions 3.8.1 and later. STAT4 is the only variant that is still +** supported. +** +** For most applications, sqlite_stat1 provides all the statistics required +** for the query planner to make good choices. +** +** Format of sqlite_stat1: +** +** There is normally one row per index, with the index identified by the +** name in the idx column. The tbl column is the name of the table to +** which the index belongs. In each such row, the stat column will be +** a string consisting of a list of integers. The first integer in this +** list is the number of rows in the index. (This is the same as the +** number of rows in the table, except for partial indices.) The second +** integer is the average number of rows in the index that have the same +** value in the first column of the index. The third integer is the average +** number of rows in the index that have the same value for the first two +** columns. The N-th integer (for N>1) is the average number of rows in +** the index which have the same value for the first N-1 columns. For +** a K-column index, there will be K+1 integers in the stat column. If +** the index is unique, then the last integer will be 1. +** +** The list of integers in the stat column can optionally be followed +** by the keyword "unordered". The "unordered" keyword, if it is present, +** must be separated from the last integer by a single space. If the +** "unordered" keyword is present, then the query planner assumes that +** the index is unordered and will not use the index for a range query. +** +** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat +** column contains a single integer which is the (estimated) number of +** rows in the table identified by sqlite_stat1.tbl. +** +** Format of sqlite_stat2: +** +** The sqlite_stat2 is only created and is only used if SQLite is compiled +** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between +** 3.6.18 and 3.7.8. The "stat2" table contains additional information +** about the distribution of keys within an index. The index is identified by +** the "idx" column and the "tbl" column is the name of the table to which +** the index belongs. There are usually 10 rows in the sqlite_stat2 +** table for each index. +** +** The sqlite_stat2 entries for an index that have sampleno between 0 and 9 +** inclusive are samples of the left-most key value in the index taken at +** evenly spaced points along the index. Let the number of samples be S +** (10 in the standard build) and let C be the number of rows in the index. +** Then the sampled rows are given by: +** +** rownumber = (i*C*2 + C)/(S*2) +** +** For i between 0 and S-1. Conceptually, the index space is divided into +** S uniform buckets and the samples are the middle row from each bucket. +** +** The format for sqlite_stat2 is recorded here for legacy reference. This +** version of SQLite does not support sqlite_stat2. It neither reads nor +** writes the sqlite_stat2 table. This version of SQLite only supports +** sqlite_stat3. +** +** Format for sqlite_stat3: +** +** The sqlite_stat3 format is a subset of sqlite_stat4. Hence, the +** sqlite_stat4 format will be described first. Further information +** about sqlite_stat3 follows the sqlite_stat4 description. +** +** Format for sqlite_stat4: +** +** As with sqlite_stat2, the sqlite_stat4 table contains histogram data +** to aid the query planner in choosing good indices based on the values +** that indexed columns are compared against in the WHERE clauses of +** queries. +** +** The sqlite_stat4 table contains multiple entries for each index. +** The idx column names the index and the tbl column is the table of the +** index. If the idx and tbl columns are the same, then the sample is +** of the INTEGER PRIMARY KEY. The sample column is a blob which is the +** binary encoding of a key from the index. The nEq column is a +** list of integers. The first integer is the approximate number +** of entries in the index whose left-most column exactly matches +** the left-most column of the sample. The second integer in nEq +** is the approximate number of entries in the index where the +** first two columns match the first two columns of the sample. +** And so forth. nLt is another list of integers that show the approximate +** number of entries that are strictly less than the sample. The first +** integer in nLt contains the number of entries in the index where the +** left-most column is less than the left-most column of the sample. +** The K-th integer in the nLt entry is the number of index entries +** where the first K columns are less than the first K columns of the +** sample. The nDLt column is like nLt except that it contains the +** number of distinct entries in the index that are less than the +** sample. +** +** There can be an arbitrary number of sqlite_stat4 entries per index. +** The ANALYZE command will typically generate sqlite_stat4 tables +** that contain between 10 and 40 samples which are distributed across +** the key space, though not uniformly, and which include samples with +** large nEq values. +** +** Format for sqlite_stat3 redux: +** +** The sqlite_stat3 table is like sqlite_stat4 except that it only +** looks at the left-most column of the index. The sqlite_stat3.sample +** column contains the actual value of the left-most column instead +** of a blob encoding of the complete index key as is found in +** sqlite_stat4.sample. The nEq, nLt, and nDLt entries of sqlite_stat3 +** all contain just a single integer which is the same as the first +** integer in the equivalent columns in sqlite_stat4. +*/ +#ifndef SQLITE_OMIT_ANALYZE +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#if defined(SQLITE_ENABLE_STAT4) +# define IsStat4 1 +#else +# define IsStat4 0 +# undef SQLITE_STAT4_SAMPLES +# define SQLITE_STAT4_SAMPLES 1 +#endif + +/* +** This routine generates code that opens the sqlite_statN tables. +** The sqlite_stat1 table is always relevant. sqlite_stat2 is now +** obsolete. sqlite_stat3 and sqlite_stat4 are only opened when +** appropriate compile-time options are provided. +** +** If the sqlite_statN tables do not previously exist, it is created. +** +** Argument zWhere may be a pointer to a buffer containing a table name, +** or it may be a NULL pointer. If it is not NULL, then all entries in +** the sqlite_statN tables associated with the named table are deleted. +** If zWhere==0, then code is generated to delete all stat table entries. +*/ +static void openStatTable( + Parse *pParse, /* Parsing context */ + int iDb, /* The database we are looking in */ + int iStatCur, /* Open the sqlite_stat1 table on this cursor */ + const char *zWhere, /* Delete entries for this table or index */ + const char *zWhereType /* Either "tbl" or "idx" */ +){ + static const struct { + const char *zName; + const char *zCols; + } aTable[] = { + { "sqlite_stat1", "tbl,idx,stat" }, +#if defined(SQLITE_ENABLE_STAT4) + { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, +#else + { "sqlite_stat4", 0 }, +#endif + { "sqlite_stat3", 0 }, + }; + int i; + sqlite3 *db = pParse->db; + Db *pDb; + Vdbe *v = sqlite3GetVdbe(pParse); + u32 aRoot[ArraySize(aTable)]; + u8 aCreateTbl[ArraySize(aTable)]; +#ifdef SQLITE_ENABLE_STAT4 + const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1; +#else + const int nToOpen = 1; +#endif + + if( v==0 ) return; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3VdbeDb(v)==db ); + pDb = &db->aDb[iDb]; + + /* Create new statistic tables if they do not exist, or clear them + ** if they do already exist. + */ + for(i=0; izDbSName))==0 ){ + if( iregRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols + ); + aRoot[i] = (u32)pParse->regRoot; + aCreateTbl[i] = OPFLAG_P2ISREG; + } + }else{ + /* The table already exists. If zWhere is not NULL, delete all entries + ** associated with the table zWhere. If zWhere is NULL, delete the + ** entire contents of the table. */ + aRoot[i] = pStat->tnum; + sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); + if( zWhere ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + pDb->zDbSName, zTab, zWhereType, zWhere + ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + }else if( db->xPreUpdateCallback ){ + sqlite3NestedParse(pParse, "DELETE FROM %Q.%s", pDb->zDbSName, zTab); +#endif + }else{ + /* The sqlite_stat[134] table already exists. Delete all rows. */ + sqlite3VdbeAddOp2(v, OP_Clear, (int)aRoot[i], iDb); + } + } + } + + /* Open the sqlite_stat[134] tables for writing. */ + for(i=0; inRowid ){ + sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + } +} +#endif + +/* Initialize the BLOB value of a ROWID +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowid(sqlite3 *db, StatSample *p, int n, const u8 *pData){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->u.aRowid = sqlite3DbMallocRawNN(db, n); + if( p->u.aRowid ){ + p->nRowid = n; + memcpy(p->u.aRowid, pData, n); + }else{ + p->nRowid = 0; + } +} +#endif + +/* Initialize the INTEGER value of a ROWID. +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, StatSample *p, i64 iRowid){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + p->u.iRowid = iRowid; +} +#endif + + +/* +** Copy the contents of object (*pFrom) into (*pTo). +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleCopy(StatAccum *p, StatSample *pTo, StatSample *pFrom){ + pTo->isPSample = pFrom->isPSample; + pTo->iCol = pFrom->iCol; + pTo->iHash = pFrom->iHash; + memcpy(pTo->anEq, pFrom->anEq, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anLt, pFrom->anLt, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anDLt, pFrom->anDLt, sizeof(tRowcnt)*p->nCol); + if( pFrom->nRowid ){ + sampleSetRowid(p->db, pTo, pFrom->nRowid, pFrom->u.aRowid); + }else{ + sampleSetRowidInt64(p->db, pTo, pFrom->u.iRowid); + } +} +#endif + +/* +** Reclaim all memory of a StatAccum structure. +*/ +static void statAccumDestructor(void *pOld){ + StatAccum *p = (StatAccum*)pOld; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); + } +#endif + sqlite3DbFree(p->db, p); +} + +/* +** Implementation of the stat_init(N,K,C,L) SQL function. The four parameters +** are: +** N: The number of columns in the index including the rowid/pk (note 1) +** K: The number of columns in the index excluding the rowid/pk. +** C: Estimated number of rows in the index +** L: A limit on the number of rows to scan, or 0 for no-limit +** +** Note 1: In the special case of the covering index that implements a +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the +** total number of columns in the table. +** +** For indexes on ordinary rowid tables, N==K+1. But for indexes on +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the +** PRIMARY KEY of the table. The covering index that implements the +** original WITHOUT ROWID table as N==K as a special case. +** +** This routine allocates the StatAccum object in heap memory. The return +** value is a pointer to the StatAccum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the StatAccum +** object. +*/ +static void statInit( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + StatAccum *p; + int nCol; /* Number of columns in index being sampled */ + int nKeyCol; /* Number of key columns */ + int nColUp; /* nCol rounded up for alignment */ + int n; /* Bytes of space to allocate */ + sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ +#ifdef SQLITE_ENABLE_STAT4 + /* Maximum number of samples. 0 if STAT4 data is not collected */ + int mxSample = OptimizationEnabled(db,SQLITE_Stat4) ?SQLITE_STAT4_SAMPLES :0; +#endif + + /* Decode the three function arguments */ + UNUSED_PARAMETER(argc); + nCol = sqlite3_value_int(argv[0]); + assert( nCol>0 ); + nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); + + /* Allocate the space required for the StatAccum object */ + n = sizeof(*p) + + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ +#ifdef SQLITE_ENABLE_STAT4 + if( mxSample ){ + n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample); + } +#endif + db = sqlite3_context_db_handle(context); + p = sqlite3DbMallocZero(db, n); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + p->db = db; + p->nEst = sqlite3_value_int64(argv[2]); + p->nRow = 0; + p->nLimit = sqlite3_value_int64(argv[3]); + p->nCol = nCol; + p->nKeyCol = nKeyCol; + p->nSkipAhead = 0; + p->current.anDLt = (tRowcnt*)&p[1]; + p->current.anEq = &p->current.anDLt[nColUp]; + +#ifdef SQLITE_ENABLE_STAT4 + p->mxSample = p->nLimit==0 ? mxSample : 0; + if( mxSample ){ + u8 *pSpace; /* Allocated space not yet assigned */ + int i; /* Used to iterate through p->aSample[] */ + + p->iGet = -1; + p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); + p->current.anLt = &p->current.anEq[nColUp]; + p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); + + /* Set up the StatAccum.a[] and aBest[] arrays */ + p->a = (struct StatSample*)&p->current.anLt[nColUp]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ + p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + } + assert( (pSpace - (u8*)p)==n ); + + for(i=0; iaBest[i].iCol = i; + } + } +#endif + + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ + sqlite3_result_blob(context, p, sizeof(*p), statAccumDestructor); +} +static const FuncDef statInitFuncdef = { + 4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statInit, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_init", /* zName */ + {0} +}; + +#ifdef SQLITE_ENABLE_STAT4 +/* +** pNew and pOld are both candidate non-periodic samples selected for +** the same column (pNew->iCol==pOld->iCol). Ignoring this column and +** considering only any trailing columns and the sample hash value, this +** function returns true if sample pNew is to be preferred over pOld. +** In other words, if we assume that the cardinalities of the selected +** column for pNew and pOld are equal, is pNew to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. +*/ +static int sampleIsBetterPost( + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld +){ + int nCol = pAccum->nCol; + int i; + assert( pNew->iCol==pOld->iCol ); + for(i=pNew->iCol+1; ianEq[i]>pOld->anEq[i] ) return 1; + if( pNew->anEq[i]anEq[i] ) return 0; + } + if( pNew->iHash>pOld->iHash ) return 1; + return 0; +} +#endif + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Return true if pNew is to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. +*/ +static int sampleIsBetter( + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld +){ + tRowcnt nEqNew = pNew->anEq[pNew->iCol]; + tRowcnt nEqOld = pOld->anEq[pOld->iCol]; + + assert( pOld->isPSample==0 && pNew->isPSample==0 ); + assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); + + if( (nEqNew>nEqOld) ) return 1; + if( nEqNew==nEqOld ){ + if( pNew->iColiCol ) return 1; + return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); + } + return 0; +} + +/* +** Copy the contents of sample *pNew into the p->a[] array. If necessary, +** remove the least desirable sample from p->a[] to make room. +*/ +static void sampleInsert(StatAccum *p, StatSample *pNew, int nEqZero){ + StatSample *pSample = 0; + int i; + + assert( IsStat4 || nEqZero==0 ); + + /* StatAccum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in StatAccum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with StatSample.anEq[m]==0 for (m>=n). */ + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } + if( pNew->isPSample==0 ){ + StatSample *pUpgrade = 0; + assert( pNew->anEq[pNew->iCol]>0 ); + + /* This sample is being added because the prefix that ends in column + ** iCol occurs many times in the table. However, if we have already + ** added a sample that shares this prefix, there is no need to add + ** this one. Instead, upgrade the priority of the highest priority + ** existing sample that shares this prefix. */ + for(i=p->nSample-1; i>=0; i--){ + StatSample *pOld = &p->a[i]; + if( pOld->anEq[pNew->iCol]==0 ){ + if( pOld->isPSample ) return; + assert( pOld->iCol>pNew->iCol ); + assert( sampleIsBetter(p, pNew, pOld) ); + if( pUpgrade==0 || sampleIsBetter(p, pOld, pUpgrade) ){ + pUpgrade = pOld; + } + } + } + if( pUpgrade ){ + pUpgrade->iCol = pNew->iCol; + pUpgrade->anEq[pUpgrade->iCol] = pNew->anEq[pUpgrade->iCol]; + goto find_new_min; + } + } + + /* If necessary, remove sample iMin to make room for the new sample. */ + if( p->nSample>=p->mxSample ){ + StatSample *pMin = &p->a[p->iMin]; + tRowcnt *anEq = pMin->anEq; + tRowcnt *anLt = pMin->anLt; + tRowcnt *anDLt = pMin->anDLt; + sampleClear(p->db, pMin); + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-p->iMin-1)); + pSample = &p->a[p->nSample-1]; + pSample->nRowid = 0; + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; + p->nSample = p->mxSample-1; + } + + /* The "rows less-than" for the rowid column must be greater than that + ** for the last sample in the p->a[] array. Otherwise, the samples would + ** be out of order. */ + assert( p->nSample==0 + || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); + + /* Insert the new sample */ + pSample = &p->a[p->nSample]; + sampleCopy(p, pSample, pNew); + p->nSample++; + + /* Zero the first nEqZero entries in the anEq[] array. */ + memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); + +find_new_min: + if( p->nSample>=p->mxSample ){ + int iMin = -1; + for(i=0; imxSample; i++){ + if( p->a[i].isPSample ) continue; + if( iMin<0 || sampleIsBetter(p, &p->a[iMin], &p->a[i]) ){ + iMin = i; + } + } + assert( iMin>=0 ); + p->iMin = iMin; + } +} +#endif /* SQLITE_ENABLE_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Field iChng of the index being scanned has changed. So at this point +** p->current contains a sample that reflects the previous row of the +** index. The value of anEq[iChng] and subsequent anEq[] elements are +** correct at this point. +*/ +static void samplePushPrevious(StatAccum *p, int iChng){ + int i; + + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + StatSample *pBest = &p->aBest[i]; + pBest->anEq[i] = p->current.anEq[i]; + if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ + sampleInsert(p, pBest, i); + } + } + + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); + } + + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } + p->nMaxEqZero = iChng; + } +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** Implementation of the stat_push SQL function: stat_push(P,C,R) +** Arguments: +** +** P Pointer to the StatAccum object created by stat_init() +** C Index of left-most column to differ from previous row +** R Rowid for the current row. Might be a key record for +** WITHOUT ROWID tables. +** +** The purpose of this routine is to collect statistical data and/or +** samples from the index being analyzed into the StatAccum object. +** The stat_get() SQL function will be used afterwards to +** retrieve the information gathered. +** +** This SQL function usually returns NULL, but might return an integer +** if it wants the byte-code to do special processing. +** +** The R parameter is only used for STAT4 +*/ +static void statPush( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + + /* The three function arguments */ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); + int iChng = sqlite3_value_int(argv[1]); + + UNUSED_PARAMETER( argc ); + UNUSED_PARAMETER( context ); + assert( p->nCol>0 ); + assert( iChngnCol ); + + if( p->nRow==0 ){ + /* This is the first call to this function. Do initialization. */ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + }else{ + /* Second and subsequent calls get processed here */ +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) samplePushPrevious(p, iChng); +#endif + + /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply + ** to the current row of the index. */ + for(i=0; icurrent.anEq[i]++; + } + for(i=iChng; inCol; i++){ + p->current.anDLt[i]++; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; +#endif + p->current.anEq[i] = 1; + } + } + + p->nRow++; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + tRowcnt nLt; + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + + nLt = p->current.anLt[p->nCol-1]; + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ + p->current.isPSample = 1; + p->current.iCol = 0; + sampleInsert(p, &p->current, p->nCol-1); + p->current.isPSample = 0; + } + + /* Update the aBest[] array. */ + for(i=0; i<(p->nCol-1); i++){ + p->current.iCol = i; + if( i>=iChng || sampleIsBetterPost(p, &p->current, &p->aBest[i]) ){ + sampleCopy(p, &p->aBest[i], &p->current); + } + } + }else +#endif + if( p->nLimit && p->nRow>(tRowcnt)p->nLimit*(p->nSkipAhead+1) ){ + p->nSkipAhead++; + sqlite3_result_int(context, p->current.anDLt[0]>0); + } +} + +static const FuncDef statPushFuncdef = { + 2+IsStat4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statPush, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_push", /* zName */ + {0} +}; + +#define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ +#define STAT_GET_ROWID 1 /* "rowid" column of stat[34] entry */ +#define STAT_GET_NEQ 2 /* "neq" column of stat[34] entry */ +#define STAT_GET_NLT 3 /* "nlt" column of stat[34] entry */ +#define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ + +/* +** Implementation of the stat_get(P,J) SQL function. This routine is +** used to query statistical information that has been gathered into +** the StatAccum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the StatAccum object. +** The content to returned is determined by the parameter J +** which is one of the STAT_GET_xxxx values defined above. +** +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a pointer to a StatAccum object, never a +** NULL. +** +** If STAT4 is not enabled, then J is always +** STAT_GET_STAT1 and is hence omitted and this routine becomes +** a one-parameter function, stat_get(P), that always returns the +** stat1 table entry information. +*/ +static void statGet( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT4 + /* STAT4 has a parameter on this routine. */ + int eCall = sqlite3_value_int(argv[1]); + assert( argc==2 ); + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT + || eCall==STAT_GET_NDLT + ); + assert( eCall==STAT_GET_STAT1 || p->mxSample ); + if( eCall==STAT_GET_STAT1 ) +#else + assert( argc==1 ); +#endif + { + /* Return the value to store in the "stat" column of the sqlite_stat1 + ** table for this index. + ** + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list + ** for each indexed column. This additional integer is an estimate of + ** the number of rows matched by a equality query on the index using + ** a key with the corresponding number of fields. In other words, + ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** "100 10 2", then SQLite estimates that: + ** + ** * the index contains 100 rows, + ** * "WHERE a=?" matches 10 rows, and + ** * "WHERE a=? AND b=?" matches 2 rows. + ** + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is computed as: + ** + ** I = (K+D-1)/D + */ + char *z; + int i; + + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + sqlite3_snprintf(24, zRet, "%llu", + p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow); + z = zRet + sqlite3Strlen30(zRet); + for(i=0; inKeyCol; i++){ + u64 nDistinct = p->current.anDLt[i] + 1; + u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + sqlite3_snprintf(24, z, " %llu", iVal); + z += sqlite3Strlen30(z); + assert( p->current.anEq[i] ); + } + assert( z[0]=='\0' && z>zRet ); + + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } +#ifdef SQLITE_ENABLE_STAT4 + else if( eCall==STAT_GET_ROWID ){ + if( p->iGet<0 ){ + samplePushPrevious(p, 0); + p->iGet = 0; + } + if( p->iGetnSample ){ + StatSample *pS = p->a + p->iGet; + if( pS->nRowid==0 ){ + sqlite3_result_int64(context, pS->u.iRowid); + }else{ + sqlite3_result_blob(context, pS->u.aRowid, pS->nRowid, + SQLITE_TRANSIENT); + } + } + }else{ + tRowcnt *aCnt = 0; + + assert( p->iGetnSample ); + switch( eCall ){ + case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; + case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; + default: { + aCnt = p->a[p->iGet].anDLt; + p->iGet++; + break; + } + } + + { + char *zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + char *z = zRet; + for(i=0; inCol; i++){ + sqlite3_snprintf(24, z, "%llu ", (u64)aCnt[i]); + z += sqlite3Strlen30(z); + } + assert( z[0]=='\0' && z>zRet ); + z[-1] = '\0'; + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } + } + } +#endif /* SQLITE_ENABLE_STAT4 */ +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( argc ); +#endif +} +static const FuncDef statGetFuncdef = { + 1+IsStat4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statGet, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_get", /* zName */ + {0} +}; + +static void callStatGet(Parse *pParse, int regStat, int iParam, int regOut){ +#ifdef SQLITE_ENABLE_STAT4 + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat+1); +#elif SQLITE_DEBUG + assert( iParam==STAT_GET_STAT1 ); +#else + UNUSED_PARAMETER( iParam ); +#endif + assert( regOut!=regStat && regOut!=regStat+1 ); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat, regOut, 1+IsStat4, + &statGetFuncdef, 0); +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* Add a comment to the most recent VDBE opcode that is the name +** of the k-th column of the pIdx index. +*/ +static void analyzeVdbeCommentIndexWithColumnName( + Vdbe *v, /* Prepared statement under construction */ + Index *pIdx, /* Index whose column is being loaded */ + int k /* Which column index */ +){ + int i; /* Index of column in the table */ + assert( k>=0 && knColumn ); + i = pIdx->aiColumn[k]; + if( NEVER(i==XN_ROWID) ){ + VdbeComment((v,"%s.rowid",pIdx->zName)); + }else if( i==XN_EXPR ){ + VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); + }else{ + VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zName)); + } +} +#else +# define analyzeVdbeCommentIndexWithColumnName(a,b,c) +#endif /* SQLITE_DEBUG */ + +/* +** Generate code to do an analysis of all indices associated with +** a single table. +*/ +static void analyzeOneTable( + Parse *pParse, /* Parser context */ + Table *pTab, /* Table whose indices are to be analyzed */ + Index *pOnlyIdx, /* If not NULL, only analyze this one index */ + int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ + int iMem, /* Available memory locations begin here */ + int iTab /* Next available cursor */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + Index *pIdx; /* An index to being analyzed */ + int iIdxCur; /* Cursor open on index being analyzed */ + int iTabCur; /* Table cursor */ + Vdbe *v; /* The virtual machine being built up */ + int i; /* Loop counter */ + int jZeroRows = -1; /* Jump from here if number of rows is zero */ + int iDb; /* Index of database containing pTab */ + u8 needTableCnt = 1; /* True to count the table */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat = iMem++; /* Register to hold StatAccum object */ + int regChng = iMem++; /* Index of changed index field */ + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ + int regTemp = iMem++; /* Temporary use register */ + int regTemp2 = iMem++; /* Second temporary use register */ + int regTabname = iMem++; /* Register containing table name */ + int regIdxname = iMem++; /* Register containing index name */ + int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ + int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + Table *pStat1 = 0; +#endif + + pParse->nMem = MAX(pParse->nMem, iMem); + v = sqlite3GetVdbe(pParse); + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( sqlite3_strlike("sqlite\\_%", pTab->zName, '\\')==0 ){ + /* Do not gather statistics on system tables */ + return; + } + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, + db->aDb[iDb].zDbSName ) ){ + return; + } +#endif + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( db->xPreUpdateCallback ){ + pStat1 = (Table*)sqlite3DbMallocZero(db, sizeof(Table) + 13); + if( pStat1==0 ) return; + pStat1->zName = (char*)&pStat1[1]; + memcpy(pStat1->zName, "sqlite_stat1", 13); + pStat1->nCol = 3; + pStat1->iPKey = -1; + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNBLOB); + } +#endif + + /* Establish a read-lock on the table at the shared-cache level. + ** Open a read-only cursor on the table. Also allocate a cursor number + ** to use for scanning indexes (iIdxCur). No index cursor is opened at + ** this time though. */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + iTabCur = iTab++; + iIdxCur = iTab++; + pParse->nTab = MAX(pParse->nTab, iTab); + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); + + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int nCol; /* Number of columns in pIdx. "N" */ + int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrNextRow; /* Address of "next_row:" */ + const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ + + if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; + if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + nColTest = nCol - 1; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; + } + + /* Populate the register containing the index name. */ + sqlite3VdbeLoadString(v, regIdxname, zIdxName); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); + + /* + ** Pseudo-code for loop that calls stat_push(): + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto chng_addr_0; + ** + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N + ** + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + ** + ** endDistinctTest: + ** regRowid = idx(rowid) + ** stat_push(P, regChng, regRowid) + ** Next csr + ** if !eof(csr) goto next_row; + ** + ** end_of_scan: + */ + + /* Make sure there are enough memory cells allocated to accommodate + ** the regPrev array and a trailing rowid (the rowid slot is required + ** when building a record to insert into the sample column of + ** the sqlite_stat4 table. */ + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + + /* Open a read-only cursor on the index being analyzed. */ + assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + + /* Invoke the stat_init() function. The arguments are: + ** + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) estimated number of rows in the index, + */ + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); + assert( regRowid==regStat+2 ); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + }else +#endif + { + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); + } + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, + &statInitFuncdef, 0); + + /* Implementation of the following: + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto next_push_0; + ** + */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); + addrNextRow = sqlite3VdbeCurrentAddr(v); + + if( nColTest>0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(pParse); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + analyzeVdbeCommentIndexWithColumnName(v,pIdx,i); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeGoto(v, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; ipTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knColumn ); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + analyzeVdbeCommentIndexWithColumnName(v,pIdx,k); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); + } + } +#endif + assert( regChng==(regStat+1) ); + { + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, + &statPushFuncdef, 0); + if( db->nAnalysisLimit ){ + int j1, j2, j3; + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + } + } + + /* Add the entry to the stat1 table. */ + callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE); +#endif + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + + /* Add the entries to the stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){ + int regEq = regStat1; + int regLt = regStat1+1; + int regDLt = regStat1+2; + int regSample = regStat1+3; + int regCol = regStat1+4; + int regSampleRowid = regCol + nCol; + int addrNext; + int addrIsNull; + u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + + pParse->nMem = MAX(pParse->nMem, regCol+nCol); + + addrNext = sqlite3VdbeCurrentAddr(v); + callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + VdbeCoverage(v); + callStatGet(pParse, regStat, STAT_GET_NEQ, regEq); + callStatGet(pParse, regStat, STAT_GET_NLT, regLt); + callStatGet(pParse, regStat, STAT_GET_NDLT, regDLt); + sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); + VdbeCoverage(v); + for(i=0; izName)); + sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE); +#endif + sqlite3VdbeJumpHere(v, jZeroRows); + } +} + + +/* +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. +*/ +static void loadAnalysis(Parse *pParse, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + } +} + +/* +** Generate code that will do an analysis of an entire database +*/ +static void analyzeDatabase(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; + int iTab; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + } + loadAnalysis(pParse, iDb); +} + +/* +** Generate code that will do an analysis of a single table in +** a database. If pOnlyIdx is not NULL then it is a single index +** in pTab that should be analyzed. +*/ +static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ + int iDb; + int iStatCur; + + assert( pTab!=0 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + if( pOnlyIdx ){ + openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); + }else{ + openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); + } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + loadAnalysis(pParse, iDb); +} + +/* +** Generate code for the ANALYZE command. The parser calls this routine +** when it recognizes an ANALYZE command. +** +** ANALYZE -- 1 +** ANALYZE -- 2 +** ANALYZE ?.? -- 3 +** +** Form 1 causes all indices in all attached databases to be analyzed. +** Form 2 analyzes all indices the single database named. +** Form 3 analyzes all indices associated with the named table. +*/ +void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ + sqlite3 *db = pParse->db; + int iDb; + int i; + char *z, *zDb; + Table *pTab; + Index *pIdx; + Token *pTableName; + Vdbe *v; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + assert( pName2!=0 || pName1==0 ); + if( pName1==0 ){ + /* Form 1: Analyze everything */ + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); + } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); + }else{ + /* Form 3: Analyze the table or index named as an argument */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); + if( iDb>=0 ){ + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; + z = sqlite3NameFromToken(db, pTableName); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, zDb))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); + } +} + +/* +** Used to pass information from the analyzer reader through to the +** callback routine. +*/ +typedef struct analysisInfo analysisInfo; +struct analysisInfo { + sqlite3 *db; + const char *zDatabase; +}; + +/* +** The first argument points to a nul-terminated string containing a +** list of space separated integers. Read the first nOut of these into +** the array aOut[]. +*/ +static void decodeIntArray( + char *zIntArray, /* String containing int array to decode */ + int nOut, /* Number of slots in aOut[] */ + tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ + Index *pIndex /* Handle extra flags for this index, if not NULL */ +){ + char *z = zIntArray; + int c; + int i; + tRowcnt v; + +#ifdef SQLITE_ENABLE_STAT4 + if( z==0 ) z = ""; +#else + assert( z!=0 ); +#endif + for(i=0; *z && i='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; + } +#ifdef SQLITE_ENABLE_STAT4 + if( aOut ) aOut[i] = v; + if( aLog ) aLog[i] = sqlite3LogEst(v); +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); + assert( aLog!=0 ); + aLog[i] = sqlite3LogEst(v); +#endif + if( *z==' ' ) z++; + } +#ifndef SQLITE_ENABLE_STAT4 + assert( pIndex!=0 ); { +#else + if( pIndex ){ +#endif + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + int sz = sqlite3Atoi(z+3); + if( sz<2 ) sz = 2; + pIndex->szIdxRow = sqlite3LogEst(sz); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } + } +} + +/* +** This callback is invoked once for each index when reading the +** sqlite_stat1 table. +** +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. +*/ +static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ + analysisInfo *pInfo = (analysisInfo*)pData; + Index *pIndex; + Table *pTable; + const char *z; + + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + + if( argv==0 || argv[0]==0 || argv[2]==0 ){ + return 0; + } + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ + return 0; + } + if( argv[1]==0 ){ + pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + } + z = argv[2]; + + if( pIndex ){ + tRowcnt *aiRowEst = 0; + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT4 + /* Index.aiRowEst may already be set here if there are duplicate + ** sqlite_stat1 entries for this index. In that case just clobber + ** the old data with the new instead of allocating a new array. */ + if( pIndex->aiRowEst==0 ){ + pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); + if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); + } + aiRowEst = pIndex->aiRowEst; +#endif + pIndex->bUnordered = 0; + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } + }else{ + Index fakeIdx; + fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); + pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; + } + + return 0; +} + +/* +** If the Index.aSample variable is not NULL, delete the aSample[] array +** and its contents. +*/ +void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ +#ifdef SQLITE_ENABLE_STAT4 + if( pIdx->aSample ){ + int j; + for(j=0; jnSample; j++){ + IndexSample *p = &pIdx->aSample[j]; + sqlite3DbFree(db, p->p); + } + sqlite3DbFree(db, pIdx->aSample); + } + if( db && db->pnBytesFreed==0 ){ + pIdx->nSample = 0; + pIdx->aSample = 0; + } +#else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pIdx); +#endif /* SQLITE_ENABLE_STAT4 */ +} + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Populate the pIdx->aAvgEq[] array based on the samples currently +** stored in pIdx->aSample[]. +*/ +static void initAvgEq(Index *pIdx){ + if( pIdx ){ + IndexSample *aSample = pIdx->aSample; + IndexSample *pFinal = &aSample[pIdx->nSample-1]; + int iCol; + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColnSample; + int i; /* Used to iterate through samples */ + tRowcnt sumEq = 0; /* Sum of the nEq values */ + tRowcnt avgEq = 0; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } + pIdx->nRowEst0 = nRow; + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ + sumEq += aSample[i].anEq[iCol]; + nSum100 += 100; + } + } + + if( nDist100>nSum100 && sumEqaAvgEq[iCol] = avgEq; + } + } +} + +/* +** Look up an index by name. Or, if the name of a WITHOUT ROWID table +** is supplied instead, find the PRIMARY KEY index for that table. +*/ +static Index *findIndexOrPrimaryKey( + sqlite3 *db, + const char *zName, + const char *zDb +){ + Index *pIdx = sqlite3FindIndex(db, zName, zDb); + if( pIdx==0 ){ + Table *pTab = sqlite3FindTable(db, zName, zDb); + if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); + } + return pIdx; +} + +/* +** Load the content from either the sqlite_stat4 +** into the relevant Index.aSample[] arrays. +** +** Arguments zSql1 and zSql2 must point to SQL statements that return +** data equivalent to the following: +** +** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx +** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 +** +** where %Q is replaced with the database name before the SQL is executed. +*/ +static int loadStatTbl( + sqlite3 *db, /* Database handle */ + const char *zSql1, /* SQL statement 1 (see above) */ + const char *zSql2, /* SQL statement 2 (see above) */ + const char *zDb /* Database name (e.g. "main") */ +){ + int rc; /* Result codes from subroutines */ + sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ + char *zSql; /* Text of the SQL statement */ + Index *pPrevIdx = 0; /* Previous index in the loop */ + IndexSample *pSample; /* A slot in pIdx->aSample[] */ + + assert( db->lookaside.bDisable ); + zSql = sqlite3MPrintf(db, zSql1, zDb); + if( !zSql ){ + return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int nIdxCol = 1; /* Number of columns in stat4 records */ + + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + nSample = sqlite3_column_int(pStmt, 1); + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + assert( pIdx==0 || pIdx->nSample==0 ); + if( pIdx==0 ) continue; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + pIdx->nSampleCol = nIdxCol; + nByte = sizeof(IndexSample) * nSample; + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + pIdx->aSample = sqlite3DbMallocZero(db, nByte); + if( pIdx->aSample==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM_BKPT; + } + pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + pIdx->pTable->tabFlags |= TF_HasStat4; + for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; + } + assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + + zSql = sqlite3MPrintf(db, zSql2, zDb); + if( !zSql ){ + return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nCol = 1; /* Number of columns in index */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + if( pIdx==0 ) continue; + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. */ + nCol = pIdx->nSampleCol; + if( pIdx!=pPrevIdx ){ + initAvgEq(pPrevIdx); + pPrevIdx = pIdx; + } + pSample = &pIdx->aSample[pIdx->nSample]; + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); + + /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + ** This is in case the sample record is corrupted. In that case, the + ** sqlite3VdbeRecordCompare() may read up to two varints past the + ** end of the allocated buffer before it realizes it is dealing with + ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a buffer overread. */ + pSample->n = sqlite3_column_bytes(pStmt, 4); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + if( pSample->p==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM_BKPT; + } + if( pSample->n ){ + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + } + pIdx->nSample++; + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); + return rc; +} + +/* +** Load content from the sqlite_stat4 table into +** the Index.aSample[] arrays of all indices. +*/ +static int loadStat4(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; /* Result codes from subroutines */ + + assert( db->lookaside.bDisable ); + if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ + rc = loadStatTbl(db, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", + zDb + ); + } + return rc; +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The +** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] +** arrays. The contents of sqlite_stat4 are used to populate the +** Index.aSample[] arrays. +** +** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is +** read from it. +** +** If SQLITE_ENABLE_STAT4 was defined during compilation and the +** sqlite_stat4 table is not present in the database, SQLITE_ERROR is +** returned. However, in this case, data is read from the sqlite_stat1 +** table (if it is present) before returning. +** +** If an OOM error occurs, this function always sets db->mallocFailed. +** This means if the caller does not care about other errors, the return +** code may be ignored. +*/ +int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ + analysisInfo sInfo; + HashElem *i; + char *zSql; + int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + + /* Clear any prior statistics */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + pIdx->hasStat1 = 0; +#ifdef SQLITE_ENABLE_STAT4 + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; +#endif + } + + /* Load new statistics out of the sqlite_stat1 table */ + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } + } + + /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); + } + + /* Load the statistics from the sqlite_stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( rc==SQLITE_OK ){ + DisableLookaside; + rc = loadStat4(db, sInfo.zDatabase); + EnableLookaside; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3_free(pIdx->aiRowEst); + pIdx->aiRowEst = 0; + } +#endif + + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + return rc; +} + + +#endif /* SQLITE_OMIT_ANALYZE */ diff --git a/third_party/sqlite3/attach.c b/third_party/sqlite3/attach.c new file mode 100644 index 000000000..44f52c2f6 --- /dev/null +++ b/third_party/sqlite3/attach.c @@ -0,0 +1,593 @@ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the ATTACH and DETACH commands. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#ifndef SQLITE_OMIT_ATTACH +/* +** Resolve an expression that was part of an ATTACH or DETACH statement. This +** is slightly different from resolving a normal SQL expression, because simple +** identifiers are treated as strings, not possible column names or aliases. +** +** i.e. if the parser sees: +** +** ATTACH DATABASE abc AS def +** +** it treats the two expressions as literal strings 'abc' and 'def' instead of +** looking for columns of the same name. +** +** This only applies to the root node of pExpr, so the statement: +** +** ATTACH DATABASE abc||def AS 'db2' +** +** will fail because neither abc or def can be resolved. +*/ +static int resolveAttachExpr(NameContext *pName, Expr *pExpr) +{ + int rc = SQLITE_OK; + if( pExpr ){ + if( pExpr->op!=TK_ID ){ + rc = sqlite3ResolveExprNames(pName, pExpr); + }else{ + pExpr->op = TK_STRING; + } + } + return rc; +} + +/* +** Return true if zName points to a name that may be used to refer to +** database iDb attached to handle db. +*/ +int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName){ + return ( + sqlite3StrICmp(db->aDb[iDb].zDbSName, zName)==0 + || (iDb==0 && sqlite3StrICmp("main", zName)==0) + ); +} + +/* +** An SQL user-function registered to do the work of an ATTACH statement. The +** three arguments to the function come directly from an attach statement: +** +** ATTACH DATABASE x AS y KEY z +** +** SELECT sqlite_attach(x, y, z) +** +** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the +** third argument. +** +** If the db->init.reopenMemdb flags is set, then instead of attaching a +** new database, close the database on db->init.iDb and reopen it as an +** empty MemDB. +*/ +static void attachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + int i; + int rc = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zName; + const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; + Db *aNew; /* New array of Db pointers */ + Db *pNew; /* Db object for the newly attached database */ + char *zErrDyn = 0; + sqlite3_vfs *pVfs; + + UNUSED_PARAMETER(NotUsed); + zFile = (const char *)sqlite3_value_text(argv[0]); + zName = (const char *)sqlite3_value_text(argv[1]); + if( zFile==0 ) zFile = ""; + if( zName==0 ) zName = ""; + +#ifdef SQLITE_ENABLE_DESERIALIZE +# define REOPEN_AS_MEMDB(db) (db->init.reopenMemdb) +#else +# define REOPEN_AS_MEMDB(db) (0) +#endif + + if( REOPEN_AS_MEMDB(db) ){ + /* This is not a real ATTACH. Instead, this routine is being called + ** from sqlite3_deserialize() to close database db->init.iDb and + ** reopen it as a MemDB */ + pVfs = sqlite3_vfs_find("memdb"); + if( pVfs==0 ) return; + pNew = &db->aDb[db->init.iDb]; + if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt); + pNew->pBt = 0; + pNew->pSchema = 0; + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); + }else{ + /* This is a real ATTACH + ** + ** Check for the following errors: + ** + ** * Too many attached databases, + ** * Transaction currently open + ** * Specified database name already being used. + */ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ + zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", + db->aLimit[SQLITE_LIMIT_ATTACHED] + ); + goto attach_error; + } + for(i=0; inDb; i++){ + assert( zName ); + if( sqlite3DbIsNamed(db, i, zName) ){ + zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); + goto attach_error; + } + } + + /* Allocate the new entry in the db->aDb[] array and initialize the schema + ** hash tables. + */ + if( db->aDb==db->aDbStatic ){ + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; + memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + }else{ + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; + } + db->aDb = aNew; + pNew = &db->aDb[db->nDb]; + memset(pNew, 0, sizeof(*pNew)); + + /* Open the database file. If the btree is successfully opened, use + ** it to obtain the database schema. At this point the schema may + ** or may not be initialized. + */ + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); + db->nDb++; + pNew->zDbSName = sqlite3DbStrDup(db, zName); + } + db->noSharedCache = 0; + if( rc==SQLITE_CONSTRAINT ){ + rc = SQLITE_ERROR; + zErrDyn = sqlite3MPrintf(db, "database is already attached"); + }else if( rc==SQLITE_OK ){ + Pager *pPager; + pNew->pSchema = sqlite3SchemaGet(db, pNew->pBt); + if( !pNew->pSchema ){ + rc = SQLITE_NOMEM_BKPT; + }else if( pNew->pSchema->file_format && pNew->pSchema->enc!=ENC(db) ){ + zErrDyn = sqlite3MPrintf(db, + "attached databases must use the same text encoding as main database"); + rc = SQLITE_ERROR; + } + sqlite3BtreeEnter(pNew->pBt); + pPager = sqlite3BtreePager(pNew->pBt); + sqlite3PagerLockingMode(pPager, db->dfltLockMode); + sqlite3BtreeSecureDelete(pNew->pBt, + sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + sqlite3BtreeSetPagerFlags(pNew->pBt, + PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); +#endif + sqlite3BtreeLeave(pNew->pBt); + } + pNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + if( rc==SQLITE_OK && pNew->zDbSName==0 ){ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3_free_filename( zPath ); + + /* If the file was opened successfully, read the schema for the new database. + ** If this fails, or if opening the file failed, then close the file and + ** remove the entry from the db->aDb[] array. i.e. put everything back the + ** way we found it. + */ + if( rc==SQLITE_OK ){ + sqlite3BtreeEnterAll(db); + db->init.iDb = 0; + db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); + if( !REOPEN_AS_MEMDB(db) ){ + rc = sqlite3Init(db, &zErrDyn); + } + sqlite3BtreeLeaveAll(db); + assert( zErrDyn==0 || rc!=SQLITE_OK ); + } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH_USER; + } + } +#endif + if( rc ){ + if( !REOPEN_AS_MEMDB(db) ){ + int iDb = db->nDb - 1; + assert( iDb>=2 ); + if( db->aDb[iDb].pBt ){ + sqlite3BtreeClose(db->aDb[iDb].pBt); + db->aDb[iDb].pBt = 0; + db->aDb[iDb].pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); + db->nDb = iDb; + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + sqlite3DbFree(db, zErrDyn); + zErrDyn = sqlite3MPrintf(db, "out of memory"); + }else if( zErrDyn==0 ){ + zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + } + } + goto attach_error; + } + + return; + +attach_error: + /* Return an error if we get here */ + if( zErrDyn ){ + sqlite3_result_error(context, zErrDyn, -1); + sqlite3DbFree(db, zErrDyn); + } + if( rc ) sqlite3_result_error_code(context, rc); +} + +/* +** An SQL user-function registered to do the work of an DETACH statement. The +** three arguments to the function come directly from a detach statement: +** +** DETACH DATABASE x +** +** SELECT sqlite_detach(x) +*/ +static void detachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *zName = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + int i; + Db *pDb = 0; + HashElem *pEntry; + char zErr[128]; + + UNUSED_PARAMETER(NotUsed); + + if( zName==0 ) zName = ""; + for(i=0; inDb; i++){ + pDb = &db->aDb[i]; + if( pDb->pBt==0 ) continue; + if( sqlite3DbIsNamed(db, i, zName) ) break; + } + + if( i>=db->nDb ){ + sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName); + goto detach_error; + } + if( i<2 ){ + sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); + goto detach_error; + } + if( sqlite3BtreeTxnState(pDb->pBt)!=SQLITE_TXN_NONE + || sqlite3BtreeIsInBackup(pDb->pBt) + ){ + sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); + goto detach_error; + } + + /* If any TEMP triggers reference the schema being detached, move those + ** triggers to reference the TEMP schema itself. */ + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pDb->pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } + + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + sqlite3CollapseDatabaseArray(db); + return; + +detach_error: + sqlite3_result_error(context, zErr, -1); +} + +/* +** This procedure generates VDBE code for a single invocation of either the +** sqlite_detach() or sqlite_attach() SQL user functions. +*/ +static void codeAttach( + Parse *pParse, /* The parser context */ + int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ + FuncDef const *pFunc,/* FuncDef wrapper for detachFunc() or attachFunc() */ + Expr *pAuthArg, /* Expression to pass to authorization callback */ + Expr *pFilename, /* Name of database file */ + Expr *pDbname, /* Name of the database to use internally */ + Expr *pKey /* Database key for encryption extension */ +){ + int rc; + NameContext sName; + Vdbe *v; + sqlite3* db = pParse->db; + int regArgs; + + if( pParse->nErr ) goto attach_end; + memset(&sName, 0, sizeof(NameContext)); + sName.pParse = pParse; + + if( + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) + ){ + goto attach_end; + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pAuthArg ){ + char *zAuthArg; + if( pAuthArg->op==TK_STRING ){ + zAuthArg = pAuthArg->u.zToken; + }else{ + zAuthArg = 0; + } + rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); + if(rc!=SQLITE_OK ){ + goto attach_end; + } + } +#endif /* SQLITE_OMIT_AUTHORIZATION */ + + + v = sqlite3GetVdbe(pParse); + regArgs = sqlite3GetTempRange(pParse, 4); + sqlite3ExprCode(pParse, pFilename, regArgs); + sqlite3ExprCode(pParse, pDbname, regArgs+1); + sqlite3ExprCode(pParse, pKey, regArgs+2); + + assert( v || db->mallocFailed ); + if( v ){ + sqlite3VdbeAddFunctionCall(pParse, 0, regArgs+3-pFunc->nArg, regArgs+3, + pFunc->nArg, pFunc, 0); + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this + ** statement only). For DETACH, set it to false (expire all existing + ** statements). + */ + sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); + } + +attach_end: + sqlite3ExprDelete(db, pFilename); + sqlite3ExprDelete(db, pDbname); + sqlite3ExprDelete(db, pKey); +} + +/* +** Called by the parser to compile a DETACH statement. +** +** DETACH pDbname +*/ +void sqlite3Detach(Parse *pParse, Expr *pDbname){ + static const FuncDef detach_func = { + 1, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + detachFunc, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "sqlite_detach", /* zName */ + {0} + }; + codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); +} + +/* +** Called by the parser to compile an ATTACH statement. +** +** ATTACH p AS pDbname KEY pKey +*/ +void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ + static const FuncDef attach_func = { + 3, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + attachFunc, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "sqlite_attach", /* zName */ + {0} + }; + codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); +} +#endif /* SQLITE_OMIT_ATTACH */ + +/* +** Expression callback used by sqlite3FixAAAA() routines. +*/ +static int fixExprCb(Walker *p, Expr *pExpr){ + DbFixer *pFix = p->u.pFix; + if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** Select callback used by sqlite3FixAAAA() routines. +*/ +static int fixSelectCb(Walker *p, Select *pSelect){ + DbFixer *pFix = p->u.pFix; + int i; + SrcItem *pItem; + sqlite3 *db = pFix->pParse->db; + int iDb = sqlite3FindDbName(db, pFix->zDb); + SrcList *pList = pSelect->pSrc; + + if( NEVER(pList==0) ) return WRC_Continue; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pFix->bTemp==0 ){ + if( pItem->zDatabase && iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return WRC_Abort; + } + sqlite3DbFree(db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; + pItem->fg.fromDDL = 1; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( sqlite3WalkExpr(&pFix->w, pList->a[i].pOn) ) return WRC_Abort; +#endif + } + if( pSelect->pWith ){ + for(i=0; ipWith->nCte; i++){ + if( sqlite3WalkSelect(p, pSelect->pWith->a[i].pSelect) ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + +/* +** Initialize a DbFixer structure. This routine must be called prior +** to passing the structure to one of the sqliteFixAAAA() routines below. +*/ +void sqlite3FixInit( + DbFixer *pFix, /* The fixer to be initialized */ + Parse *pParse, /* Error messages will be written here */ + int iDb, /* This is the database that must be used */ + const char *zType, /* "view", "trigger", or "index" */ + const Token *pName /* Name of the view, trigger, or index */ +){ + sqlite3 *db = pParse->db; + assert( db->nDb>iDb ); + pFix->pParse = pParse; + pFix->zDb = db->aDb[iDb].zDbSName; + pFix->pSchema = db->aDb[iDb].pSchema; + pFix->zType = zType; + pFix->pName = pName; + pFix->bTemp = (iDb==1); + pFix->w.pParse = pParse; + pFix->w.xExprCallback = fixExprCb; + pFix->w.xSelectCallback = fixSelectCb; + pFix->w.xSelectCallback2 = 0; + pFix->w.walkerDepth = 0; + pFix->w.eCode = 0; + pFix->w.u.pFix = pFix; +} + +/* +** The following set of routines walk through the parse tree and assign +** a specific database to all table references where the database name +** was left unspecified in the original SQL statement. The pFix structure +** must have been initialized by a prior call to sqlite3FixInit(). +** +** These routines are used to make sure that an index, trigger, or +** view in one database does not refer to objects in a different database. +** (Exception: indices, triggers, and views in the TEMP database are +** allowed to refer to anything.) If a reference is explicitly made +** to an object in a different database, an error message is added to +** pParse->zErrMsg and these routines return non-zero. If everything +** checks out, these routines return 0. +*/ +int sqlite3FixSrcList( + DbFixer *pFix, /* Context of the fixation */ + SrcList *pList /* The Source list to check and modify */ +){ + int res = 0; + if( pList ){ + Select s; + memset(&s, 0, sizeof(s)); + s.pSrc = pList; + res = sqlite3WalkSelect(&pFix->w, &s); + } + return res; +} +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) +int sqlite3FixSelect( + DbFixer *pFix, /* Context of the fixation */ + Select *pSelect /* The SELECT statement to be fixed to one database */ +){ + return sqlite3WalkSelect(&pFix->w, pSelect); +} +int sqlite3FixExpr( + DbFixer *pFix, /* Context of the fixation */ + Expr *pExpr /* The expression to be fixed to one database */ +){ + return sqlite3WalkExpr(&pFix->w, pExpr); +} +#endif + +#ifndef SQLITE_OMIT_TRIGGER +int sqlite3FixTriggerStep( + DbFixer *pFix, /* Context of the fixation */ + TriggerStep *pStep /* The trigger step be fixed to one database */ +){ + while( pStep ){ + if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) + || sqlite3WalkExpr(&pFix->w, pStep->pWhere) + || sqlite3WalkExprList(&pFix->w, pStep->pExprList) + || sqlite3FixSrcList(pFix, pStep->pFrom) + ){ + return 1; + } +#ifndef SQLITE_OMIT_UPSERT + if( pStep->pUpsert ){ + Upsert *pUp = pStep->pUpsert; + if( sqlite3WalkExprList(&pFix->w, pUp->pUpsertTarget) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertTargetWhere) + || sqlite3WalkExprList(&pFix->w, pUp->pUpsertSet) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertWhere) + ){ + return 1; + } + } +#endif + pStep = pStep->pNext; + } + + return 0; +} +#endif diff --git a/third_party/sqlite3/auth.c b/third_party/sqlite3/auth.c new file mode 100644 index 000000000..710f8ce75 --- /dev/null +++ b/third_party/sqlite3/auth.c @@ -0,0 +1,269 @@ +/* +** 2003 January 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the sqlite3_set_authorizer() +** API. This facility is an optional feature of the library. Embedded +** systems that do not need this facility may omit it by recompiling +** the library with -DSQLITE_OMIT_AUTHORIZATION=1 +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** All of the code in this file may be omitted by defining a single +** macro. +*/ +#ifndef SQLITE_OMIT_AUTHORIZATION + +/* +** Set or clear the access authorization function. +** +** The access authorization function is be called during the compilation +** phase to verify that the user has read and/or write access permission on +** various fields of the database. The first argument to the auth function +** is a copy of the 3rd argument to this routine. The second argument +** to the auth function is one of these constants: +** +** SQLITE_CREATE_INDEX +** SQLITE_CREATE_TABLE +** SQLITE_CREATE_TEMP_INDEX +** SQLITE_CREATE_TEMP_TABLE +** SQLITE_CREATE_TEMP_TRIGGER +** SQLITE_CREATE_TEMP_VIEW +** SQLITE_CREATE_TRIGGER +** SQLITE_CREATE_VIEW +** SQLITE_DELETE +** SQLITE_DROP_INDEX +** SQLITE_DROP_TABLE +** SQLITE_DROP_TEMP_INDEX +** SQLITE_DROP_TEMP_TABLE +** SQLITE_DROP_TEMP_TRIGGER +** SQLITE_DROP_TEMP_VIEW +** SQLITE_DROP_TRIGGER +** SQLITE_DROP_VIEW +** SQLITE_INSERT +** SQLITE_PRAGMA +** SQLITE_READ +** SQLITE_SELECT +** SQLITE_TRANSACTION +** SQLITE_UPDATE +** +** The third and fourth arguments to the auth function are the name of +** the table and the column that are being accessed. The auth function +** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE. If +** SQLITE_OK is returned, it means that access is allowed. SQLITE_DENY +** means that the SQL statement will never-run - the sqlite3_exec() call +** will return with an error. SQLITE_IGNORE means that the SQL statement +** should run but attempts to read the specified column will return NULL +** and attempts to write the column will be ignored. +** +** Setting the auth function to NULL disables this hook. The default +** setting of the auth function is NULL. +*/ +int sqlite3_set_authorizer( + sqlite3 *db, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xAuth = (sqlite3_xauth)xAuth; + db->pAuthArg = pArg; + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Write an error message into pParse->zErrMsg that explains that the +** user-supplied authorization function returned an illegal value. +*/ +static void sqliteAuthBadReturnCode(Parse *pParse){ + sqlite3ErrorMsg(pParse, "authorizer malfunction"); + pParse->rc = SQLITE_ERROR; +} + +/* +** Invoke the authorization callback for permission to read column zCol from +** table zTab in database zDb. This function assumes that an authorization +** callback has been registered (i.e. that sqlite3.xAuth is not NULL). +** +** If SQLITE_IGNORE is returned and pExpr is not NULL, then pExpr is changed +** to an SQL NULL expression. Otherwise, if pExpr is NULL, then SQLITE_IGNORE +** is treated as SQLITE_DENY. In this case an error is left in pParse. +*/ +int sqlite3AuthReadCol( + Parse *pParse, /* The parser context */ + const char *zTab, /* Table name */ + const char *zCol, /* Column name */ + int iDb /* Index of containing database. */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ + + if( db->init.busy ) return SQLITE_OK; + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); + if( rc==SQLITE_DENY ){ + char *z = sqlite3_mprintf("%s.%s", zTab, zCol); + if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); + sqlite3ErrorMsg(pParse, "access to %z is prohibited", z); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** The pExpr should be a TK_COLUMN expression. The table referred to +** is in pTabList or else it is the NEW or OLD table of a trigger. +** Check to see if it is OK to read this particular column. +** +** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN +** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, +** then generate an error. +*/ +void sqlite3AuthRead( + Parse *pParse, /* The parser context */ + Expr *pExpr, /* The expression to check authorization on */ + Schema *pSchema, /* The schema of the expression */ + SrcList *pTabList /* All table that pExpr might refer to */ +){ + Table *pTab = 0; /* The table being read */ + const char *zCol; /* Name of the column of the table */ + int iSrc; /* Index in pTabList->a[] of table being read */ + int iDb; /* The index of the database the expression refers to */ + int iCol; /* Index of column in table */ + + assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + assert( !IN_RENAME_OBJECT ); + assert( pParse->db->xAuth!=0 ); + iDb = sqlite3SchemaToIndex(pParse->db, pSchema); + if( iDb<0 ){ + /* An attempt to read a column out of a subquery or other + ** temporary table. */ + return; + } + + if( pExpr->op==TK_TRIGGER ){ + pTab = pParse->pTriggerTab; + }else{ + assert( pTabList ); + for(iSrc=0; iSrcnSrc; iSrc++){ + if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ + pTab = pTabList->a[iSrc].pTab; + break; + } + } + } + iCol = pExpr->iColumn; + if( pTab==0 ) return; + + if( iCol>=0 ){ + assert( iColnCol ); + zCol = pTab->aCol[iCol].zName; + }else if( pTab->iPKey>=0 ){ + assert( pTab->iPKeynCol ); + zCol = pTab->aCol[pTab->iPKey].zName; + }else{ + zCol = "ROWID"; + } + assert( iDb>=0 && iDbdb->nDb ); + if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ + pExpr->op = TK_NULL; + } +} + +/* +** Do an authorization check using the code and arguments given. Return +** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY. If SQLITE_DENY +** is returned, then the error count and error message in pParse are +** modified appropriately. +*/ +int sqlite3AuthCheck( + Parse *pParse, + int code, + const char *zArg1, + const char *zArg2, + const char *zArg3 +){ + sqlite3 *db = pParse->db; + int rc; + + /* Don't do any authorization checks if the database is initialising + ** or if the parser is being invoked from within sqlite3_declare_vtab. + */ + assert( !IN_RENAME_OBJECT || db->xAuth==0 ); + if( db->xAuth==0 || db->init.busy || IN_SPECIAL_PARSE ){ + return SQLITE_OK; + } + + /* EVIDENCE-OF: R-43249-19882 The third through sixth parameters to the + ** callback are either NULL pointers or zero-terminated strings that + ** contain additional details about the action to be authorized. + ** + ** The following testcase() macros show that any of the 3rd through 6th + ** parameters can be either NULL or a string. */ + testcase( zArg1==0 ); + testcase( zArg2==0 ); + testcase( zArg3==0 ); + testcase( pParse->zAuthContext==0 ); + + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); + if( rc==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized"); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){ + rc = SQLITE_DENY; + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** Push an authorization context. After this routine is called, the +** zArg3 argument to authorization callbacks will be zContext until +** popped. Or if pParse==0, this routine is a no-op. +*/ +void sqlite3AuthContextPush( + Parse *pParse, + AuthContext *pContext, + const char *zContext +){ + assert( pParse ); + pContext->pParse = pParse; + pContext->zAuthContext = pParse->zAuthContext; + pParse->zAuthContext = zContext; +} + +/* +** Pop an authorization context that was previously pushed +** by sqlite3AuthContextPush +*/ +void sqlite3AuthContextPop(AuthContext *pContext){ + if( pContext->pParse ){ + pContext->pParse->zAuthContext = pContext->zAuthContext; + pContext->pParse = 0; + } +} + +#endif /* SQLITE_OMIT_AUTHORIZATION */ diff --git a/third_party/sqlite3/backup.c b/third_party/sqlite3/backup.c new file mode 100644 index 000000000..6c46bb5c4 --- /dev/null +++ b/third_party/sqlite3/backup.c @@ -0,0 +1,772 @@ +/* +** 2009 January 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_backup_XXX() +** API functions and the related features. +*/ +#include "third_party/sqlite3/btreeInt.inc" +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** Structure allocated for each backup operation. +*/ +struct sqlite3_backup { + sqlite3* pDestDb; /* Destination database handle */ + Btree *pDest; /* Destination b-tree file */ + u32 iDestSchema; /* Original schema cookie in destination */ + int bDestLocked; /* True once a write-transaction is open on pDest */ + + Pgno iNext; /* Page number of the next source page to copy */ + sqlite3* pSrcDb; /* Source database handle */ + Btree *pSrc; /* Source b-tree file */ + + int rc; /* Backup process error code */ + + /* These two variables are set by every call to backup_step(). They are + ** read by calls to backup_remaining() and backup_pagecount(). + */ + Pgno nRemaining; /* Number of pages left to copy */ + Pgno nPagecount; /* Total number of pages to copy */ + + int isAttached; /* True once backup has been registered with pager */ + sqlite3_backup *pNext; /* Next backup associated with source pager */ +}; + +/* +** THREAD SAFETY NOTES: +** +** Once it has been created using backup_init(), a single sqlite3_backup +** structure may be accessed via two groups of thread-safe entry points: +** +** * Via the sqlite3_backup_XXX() API function backup_step() and +** backup_finish(). Both these functions obtain the source database +** handle mutex and the mutex associated with the source BtShared +** structure, in that order. +** +** * Via the BackupUpdate() and BackupRestart() functions, which are +** invoked by the pager layer to report various state changes in +** the page cache associated with the source database. The mutex +** associated with the source database BtShared structure will always +** be held when either of these functions are invoked. +** +** The other sqlite3_backup_XXX() API functions, backup_remaining() and +** backup_pagecount() are not thread-safe functions. If they are called +** while some other thread is calling backup_step() or backup_finish(), +** the values returned may be invalid. There is no way for a call to +** BackupUpdate() or BackupRestart() to interfere with backup_remaining() +** or backup_pagecount(). +** +** Depending on the SQLite configuration, the database handles and/or +** the Btree objects may have their own mutexes that require locking. +** Non-sharable Btrees (in-memory databases for example), do not have +** associated mutexes. +*/ + +/* +** Return a pointer corresponding to database zDb (i.e. "main", "temp") +** in connection handle pDb. If such a database cannot be found, return +** a NULL pointer and write an error message to pErrorDb. +** +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an +** error message to pErrorDb. +*/ +static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ + int i = sqlite3FindDbName(pDb, zDb); + + if( i==1 ){ + Parse sParse; + int rc = 0; + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; + } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParserReset(&sParse); + if( rc ){ + return 0; + } + } + + if( i<0 ){ + sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + return 0; + } + + return pDb->aDb[i].pBt; +} + +/* +** Attempt to set the page size of the destination to match the page size +** of the source. +*/ +static int setDestPgsz(sqlite3_backup *p){ + int rc; + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),0,0); + return rc; +} + +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeTxnState(p)!=SQLITE_TXN_NONE ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* +** Create an sqlite3_backup process to copy the contents of zSrcDb from +** connection handle pSrcDb to zDestDb in pDestDb. If successful, return +** a pointer to the new sqlite3_backup object. +** +** If an error occurs, NULL is returned and an error code and error message +** stored in database handle pDestDb. +*/ +sqlite3_backup *sqlite3_backup_init( + sqlite3* pDestDb, /* Database to write to */ + const char *zDestDb, /* Name of database within pDestDb */ + sqlite3* pSrcDb, /* Database connection to read from */ + const char *zSrcDb /* Name of database within pSrcDb */ +){ + sqlite3_backup *p; /* Value to return */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + /* Lock the source database handle. The destination database + ** handle is not locked in this routine, but it is locked in + ** sqlite3_backup_step(). The user is required to ensure that no + ** other thread accesses the destination handle for the duration + ** of the backup operation. Any attempt to use the destination + ** database connection while a backup is in progress may cause + ** a malfunction or a deadlock. + */ + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3_mutex_enter(pDestDb->mutex); + + if( pSrcDb==pDestDb ){ + sqlite3ErrorWithMsg( + pDestDb, SQLITE_ERROR, "source and destination must be distinct" + ); + p = 0; + }else { + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); + if( !p ){ + sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); + } + } + + /* If the allocation succeeded, populate the new object. */ + if( p ){ + p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); + p->pDest = findBtree(pDestDb, pDestDb, zDestDb); + p->pDestDb = pDestDb; + p->pSrcDb = pSrcDb; + p->iNext = 1; + p->isAttached = 0; + + if( 0==p->pSrc || 0==p->pDest + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ + /* One (or both) of the named databases did not exist or an OOM + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ + sqlite3_free(p); + p = 0; + } + } + if( p ){ + p->pSrc->nBackup++; + } + + sqlite3_mutex_leave(pDestDb->mutex); + sqlite3_mutex_leave(pSrcDb->mutex); + return p; +} + +/* +** Argument rc is an SQLite error code. Return true if this error is +** considered fatal if encountered during a backup operation. All errors +** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. +*/ +static int isFatalError(int rc){ + return (rc!=SQLITE_OK && rc!=SQLITE_BUSY && ALWAYS(rc!=SQLITE_LOCKED)); +} + +/* +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the +** destination database. +*/ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); + const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; + int rc = SQLITE_OK; + i64 iOff; + + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); + assert( p->bDestLocked ); + assert( !isFatalError(p->rc) ); + assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); + assert( zSrcData ); + + /* Catch the case where the destination is an in-memory database and the + ** page sizes of the source and destination differ. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ + rc = SQLITE_READONLY; + } + + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. + */ + for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg, 0)) + && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) + ){ + const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; + u8 *zDestData = sqlite3PagerGetData(pDestPg); + u8 *zOut = &zDestData[iOff%nDestPgsz]; + + /* Copy the data from the source page into the destination page. + ** Then clear the Btree layer MemPage.isInit flag. Both this module + ** and the pager code use this trick (clearing the first byte + ** of the page 'extra' space to invalidate the Btree layers + ** cached parse of the page). MemPage.isInit is marked + ** "MUST BE FIRST" for this purpose. + */ + memcpy(zOut, zIn, nCopy); + ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } + } + sqlite3PagerUnref(pDestPg); + } + + return rc; +} + +/* +** If pFile is currently larger than iSize bytes, then truncate it to +** exactly iSize bytes. If pFile is not larger than iSize bytes, then +** this function is a no-op. +** +** Return SQLITE_OK if everything is successful, or an SQLite error +** code if an error occurs. +*/ +static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ + i64 iCurrent; + int rc = sqlite3OsFileSize(pFile, &iCurrent); + if( rc==SQLITE_OK && iCurrent>iSize ){ + rc = sqlite3OsTruncate(pFile, iSize); + } + return rc; +} + +/* +** Register this backup object with the associated source pager for +** callbacks when pages are changed or the cache invalidated. +*/ +static void attachBackupObject(sqlite3_backup *p){ + sqlite3_backup **pp; + assert( sqlite3BtreeHoldsMutex(p->pSrc) ); + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + p->pNext = *pp; + *pp = p; + p->isAttached = 1; +} + +/* +** Copy nPage pages from the source b-tree to the destination. +*/ +int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + int rc; + int destMode; /* Destination journal mode */ + int pgszSrc = 0; /* Source page size */ + int pgszDest = 0; /* Destination page size */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + rc = p->rc; + if( !isFatalError(rc) ){ + Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ + int ii; /* Iterator variable */ + int nSrcPage = -1; /* Size of source db in pages */ + int bCloseTrans = 0; /* True if src db requires unlocking */ + + /* If the source pager is currently in a write-transaction, return + ** SQLITE_BUSY immediately. + */ + if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ + rc = SQLITE_BUSY; + }else{ + rc = SQLITE_OK; + } + + /* If there is no open read-transaction on the source database, open + ** one now. If a transaction is opened here, then it will be closed + ** before this function exits. + */ + if( rc==SQLITE_OK && SQLITE_TXN_NONE==sqlite3BtreeTxnState(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0); + bCloseTrans = 1; + } + + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2, + (int*)&p->iDestSchema)) + ){ + p->bDestLocked = 1; + } + + /* Do not allow backup if the destination database is in WAL mode + ** and the page sizes are different between source and destination */ + pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); + pgszDest = sqlite3BtreeGetPageSize(p->pDest); + destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); + if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + rc = SQLITE_READONLY; + } + + /* Now that there is a read-lock on the source database, query the + ** source pager for the number of pages in the database. + */ + nSrcPage = (int)sqlite3BtreeLastPage(p->pSrc); + assert( nSrcPage>=0 ); + for(ii=0; (nPage<0 || iiiNext<=(Pgno)nSrcPage && !rc; ii++){ + const Pgno iSrcPg = p->iNext; /* Source page number */ + if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ + DbPage *pSrcPg; /* Source page object */ + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg,PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); + sqlite3PagerUnref(pSrcPg); + } + } + p->iNext++; + } + if( rc==SQLITE_OK ){ + p->nPagecount = nSrcPage; + p->nRemaining = nSrcPage+1-p->iNext; + if( p->iNext>(Pgno)nSrcPage ){ + rc = SQLITE_DONE; + }else if( !p->isAttached ){ + attachBackupObject(p); + } + } + + /* Update the schema version field in the destination database. This + ** is to make sure that the schema-version really does change in + ** the case where the source and destination databases have the + ** same schema version. + */ + if( rc==SQLITE_DONE ){ + if( nSrcPage==0 ){ + rc = sqlite3BtreeNewDb(p->pDest); + nSrcPage = 1; + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + } + if( rc==SQLITE_OK ){ + if( p->pDestDb ){ + sqlite3ResetAllSchemasOfConnection(p->pDestDb); + } + if( destMode==PAGER_JOURNALMODE_WAL ){ + rc = sqlite3BtreeSetVersion(p->pDest, 2); + } + } + if( rc==SQLITE_OK ){ + int nDestTruncate; + /* Set nDestTruncate to the final number of pages in the destination + ** database. The complication here is that the destination page + ** size may be different to the source page size. + ** + ** If the source page size is smaller than the destination page size, + ** round up. In this case the call to sqlite3OsTruncate() below will + ** fix the size of the file. However it is important to call + ** sqlite3PagerTruncateImage() here so that any pages in the + ** destination file that lie beyond the nDestTruncate page mark are + ** journalled by PagerCommitPhaseOne() before they are destroyed + ** by the file truncation. + */ + assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); + assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); + if( pgszSrcpDest->pBt) ){ + nDestTruncate--; + } + }else{ + nDestTruncate = nSrcPage * (pgszSrc/pgszDest); + } + assert( nDestTruncate>0 ); + + if( pgszSrc= iSize || ( + nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) + && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest + )); + + /* This block ensures that all data required to recreate the original + ** database has been stored in the journal for pDestPager and the + ** journal synced to disk. So at this point we may safely modify + ** the database file in any way, knowing that if a power failure + ** occurs, the original database will be reconstructed from the + ** journal file. */ + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } + + /* Write the extra pages and truncate the database file as required */ + iEnd = MIN(PENDING_BYTE + pgszDest, iSize); + for( + iOff=PENDING_BYTE+pgszSrc; + rc==SQLITE_OK && iOffpDest, 0)) + ){ + rc = SQLITE_DONE; + } + } + } + + /* If bCloseTrans is true, then this function opened a read transaction + ** on the source database. Close the read transaction here. There is + ** no need to check the return values of the btree methods here, as + ** "committing" a read-only transaction cannot fail. + */ + if( bCloseTrans ){ + TESTONLY( int rc2 ); + TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); + TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc, 0); + assert( rc2==SQLITE_OK ); + } + + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM_BKPT; + } + p->rc = rc; + } + if( p->pDestDb ){ + sqlite3_mutex_leave(p->pDestDb->mutex); + } + sqlite3BtreeLeave(p->pSrc); + sqlite3_mutex_leave(p->pSrcDb->mutex); + return rc; +} + +/* +** Release all resources associated with an sqlite3_backup* handle. +*/ +int sqlite3_backup_finish(sqlite3_backup *p){ + sqlite3_backup **pp; /* Ptr to head of pagers backup list */ + sqlite3 *pSrcDb; /* Source database connection */ + int rc; /* Value to return */ + + /* Enter the mutexes */ + if( p==0 ) return SQLITE_OK; + pSrcDb = p->pSrcDb; + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + /* Detach this backup from the source pager. */ + if( p->pDestDb ){ + p->pSrc->nBackup--; + } + if( p->isAttached ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + assert( pp!=0 ); + while( *pp!=p ){ + pp = &(*pp)->pNext; + assert( pp!=0 ); + } + *pp = p->pNext; + } + + /* If a transaction is still open on the Btree, roll it back. */ + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); + + /* Set the error code of the destination database handle. */ + rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; + if( p->pDestDb ){ + sqlite3Error(p->pDestDb, rc); + + /* Exit the mutexes and free the backup context structure. */ + sqlite3LeaveMutexAndCloseZombie(p->pDestDb); + } + sqlite3BtreeLeave(p->pSrc); + if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + sqlite3_free(p); + } + sqlite3LeaveMutexAndCloseZombie(pSrcDb); + return rc; +} + +/* +** Return the number of pages still to be backed up as of the most recent +** call to sqlite3_backup_step(). +*/ +int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return p->nRemaining; +} + +/* +** Return the total number of pages in the source database as of the most +** recent call to sqlite3_backup_step(). +*/ +int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return p->nPagecount; +} + +/* +** This function is called after the contents of page iPage of the +** source database have been modified. If page iPage has already been +** copied into the destination database, then the data written to the +** destination is now invalidated. The destination copy of iPage needs +** to be updated with the new data before the backup operation is +** complete. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +static SQLITE_NOINLINE void backupUpdate( + sqlite3_backup *p, + Pgno iPage, + const u8 *aData +){ + assert( p!=0 ); + do{ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + if( !isFatalError(p->rc) && iPageiNext ){ + /* The backup process p has already copied page iPage. But now it + ** has been modified by a transaction on the source pager. Copy + ** the new data into the backup. + */ + int rc; + assert( p->pDestDb ); + sqlite3_mutex_enter(p->pDestDb->mutex); + rc = backupOnePage(p, iPage, aData, 1); + sqlite3_mutex_leave(p->pDestDb->mutex); + assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); + if( rc!=SQLITE_OK ){ + p->rc = rc; + } + } + }while( (p = p->pNext)!=0 ); +} +void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + if( pBackup ) backupUpdate(pBackup, iPage, aData); +} + +/* +** Restart the backup process. This is called when the pager layer +** detects that the database has been modified by an external database +** connection. In this case there is no way of knowing which of the +** pages that have been copied into the destination database are still +** valid and which are not, so the entire process needs to be restarted. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +void sqlite3BackupRestart(sqlite3_backup *pBackup){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + p->iNext = 1; + } +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Copy the complete content of pBtFrom into pBtTo. A transaction +** must be active for both files. +** +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the +** transaction is committed before returning. +*/ +int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3_file *pFd; /* File descriptor for database pTo */ + sqlite3_backup b; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + + assert( sqlite3BtreeTxnState(pTo)==SQLITE_TXN_WRITE ); + pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); + if( pFd->pMethods ){ + i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); + rc = sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc ) goto copy_finished; + } + + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set + ** to 0. This is used by the implementations of sqlite3_backup_step() + ** and sqlite3_backup_finish() to detect that they are being called + ** from this function, not directly by the user. + */ + memset(&b, 0, sizeof(b)); + b.pSrcDb = pFrom->db; + b.pSrc = pFrom; + b.pDest = pTo; + b.iNext = 1; + + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** within a single call (unless an error occurs). The assert() statement + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. */ + sqlite3_backup_step(&b, 0x7FFFFFFF); + assert( b.rc!=SQLITE_OK ); + + rc = sqlite3_backup_finish(&b); + if( rc==SQLITE_OK ){ + pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; + }else{ + sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); + } + + assert( sqlite3BtreeTxnState(pTo)!=SQLITE_TXN_WRITE ); +copy_finished: + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; +} +#endif /* SQLITE_OMIT_VACUUM */ diff --git a/third_party/sqlite3/bitvec.c b/third_party/sqlite3/bitvec.c new file mode 100644 index 000000000..5ce04aee0 --- /dev/null +++ b/third_party/sqlite3/bitvec.c @@ -0,0 +1,412 @@ +/* +** 2008 February 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an object that represents a fixed-length +** bitmap. Bits are numbered starting with 1. +** +** A bitmap is used to record which pages of a database file have been +** journalled during a transaction, or which pages have the "dont-write" +** property. Usually only a few pages are meet either condition. +** So the bitmap is usually sparse and has low cardinality. +** But sometimes (for example when during a DROP of a large table) most +** or all of the pages in a database can get journalled. In those cases, +** the bitmap becomes dense with high cardinality. The algorithm needs +** to handle both cases well. +** +** The size of the bitmap is fixed when the object is created. +** +** All bits are clear when the bitmap is created. Individual bits +** may be set or cleared one at a time. +** +** Test operations are about 100 times more common that set operations. +** Clear operations are exceedingly rare. There are usually between +** 5 and 500 set operations per Bitvec object, though the number of sets can +** sometimes grow into tens of thousands or larger. The size of the +** Bitvec object is the number of pages in the database file at the +** start of a transaction, and is thus usually less than a few thousand, +** but can be as large as 2 billion for a really big database. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* Size of the Bitvec structure in bytes. */ +#define BITVEC_SZ 512 + +/* Round the union size down to the nearest pointer boundary, since that's how +** it will be aligned within the Bitvec struct. */ +#define BITVEC_USIZE \ + (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) + +/* Type of the array "element" for the bitmap representation. +** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. +** Setting this to the "natural word" size of your CPU may improve +** performance. */ +#define BITVEC_TELEM u8 +/* Size, in bits, of the bitmap element. */ +#define BITVEC_SZELEM 8 +/* Number of elements in a bitmap array. */ +#define BITVEC_NELEM (BITVEC_USIZE/sizeof(BITVEC_TELEM)) +/* Number of bits in the bitmap array. */ +#define BITVEC_NBIT (BITVEC_NELEM*BITVEC_SZELEM) + +/* Number of u32 values in hash table. */ +#define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) +/* Maximum number of entries in hash table before +** sub-dividing and re-hashing. */ +#define BITVEC_MXHASH (BITVEC_NINT/2) +/* Hashing function for the aHash representation. +** Empirical testing showed that the *37 multiplier +** (an arbitrary prime)in the hash function provided +** no fewer collisions than the no-op *1. */ +#define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) + +#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) + + +/* +** A bitmap is an instance of the following structure. +** +** This bitmap records the existence of zero or more bits +** with values between 1 and iSize, inclusive. +** +** There are three possible representations of the bitmap. +** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight +** bitmap. The least significant bit is bit 1. +** +** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is +** a hash table that will hold up to BITVEC_MXHASH distinct values. +** +** Otherwise, the value i is redirected into one of BITVEC_NPTR +** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap +** handles up to iDivisor separate values of i. apSub[0] holds +** values between 1 and iDivisor. apSub[1] holds values between +** iDivisor+1 and 2*iDivisor. apSub[N] holds values between +** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized +** to hold deal with values between 1 and iDivisor. +*/ +struct Bitvec { + u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ + u32 nSet; /* Number of bits that are set - only valid for aHash + ** element. Max is BITVEC_NINT. For BITVEC_SZ of 512, + ** this would be 125. */ + u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ + /* Should >=0 for apSub element. */ + /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ + /* For a BITVEC_SZ of 512, this would be 34,359,739. */ + union { + BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ + u32 aHash[BITVEC_NINT]; /* Hash table representation */ + Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ + } u; +}; + +/* +** Create a new bitmap object able to handle bits between 0 and iSize, +** inclusive. Return a pointer to the new object. Return NULL if +** malloc fails. +*/ +Bitvec *sqlite3BitvecCreate(u32 iSize){ + Bitvec *p; + assert( sizeof(*p)==BITVEC_SZ ); + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->iSize = iSize; + } + return p; +} + +/* +** Check to see if the i-th bit is set. Return true or false. +** If p is NULL (if the bitmap has not been created) or if +** i is out of range, then return false. +*/ +int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ + assert( p!=0 ); + i--; + if( i>=p->iSize ) return 0; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return 0; + } + } + if( p->iSize<=BITVEC_NBIT ){ + return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; + } else{ + u32 h = BITVEC_HASH(i++); + while( p->u.aHash[h] ){ + if( p->u.aHash[h]==i ) return 1; + h = (h+1) % BITVEC_NINT; + } + return 0; + } +} +int sqlite3BitvecTest(Bitvec *p, u32 i){ + return p!=0 && sqlite3BitvecTestNotNull(p,i); +} + +/* +** Set the i-th bit. Return 0 on success and an error code if +** anything goes wrong. +** +** This routine might cause sub-bitmaps to be allocated. Failing +** to get the memory needed to hold the sub-bitmap is the only +** that can go wrong with an insert, assuming p and i are valid. +** +** The calling function must ensure that p is a valid Bitvec object +** and that the value for "i" is within range of the Bitvec object. +** Otherwise the behavior is undefined. +*/ +int sqlite3BitvecSet(Bitvec *p, u32 i){ + u32 h; + if( p==0 ) return SQLITE_OK; + assert( i>0 ); + assert( i<=p->iSize ); + i--; + while((p->iSize > BITVEC_NBIT) && p->iDivisor) { + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + if( p->u.apSub[bin]==0 ){ + p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; + } + p = p->u.apSub[bin]; + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); + return SQLITE_OK; + } + h = BITVEC_HASH(i++); + /* if there wasn't a hash collision, and this doesn't */ + /* completely fill the hash, then just add it without */ + /* worring about sub-dividing and re-hashing. */ + if( !p->u.aHash[h] ){ + if (p->nSet<(BITVEC_NINT-1)) { + goto bitvec_set_end; + } else { + goto bitvec_set_rehash; + } + } + /* there was a collision, check to see if it's already */ + /* in hash, if not, try to find a spot for it */ + do { + if( p->u.aHash[h]==i ) return SQLITE_OK; + h++; + if( h>=BITVEC_NINT ) h = 0; + } while( p->u.aHash[h] ); + /* we didn't find it in the hash. h points to the first */ + /* available free spot. check to see if this is going to */ + /* make our hash too "full". */ +bitvec_set_rehash: + if( p->nSet>=BITVEC_MXHASH ){ + unsigned int j; + int rc; + u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); + if( aiValues==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.apSub, 0, sizeof(p->u.apSub)); + p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + rc = sqlite3BitvecSet(p, i); + for(j=0; jnSet++; + p->u.aHash[h] = i; + return SQLITE_OK; +} + +/* +** Clear the i-th bit. +** +** pBuf must be a pointer to at least BITVEC_SZ bytes of temporary storage +** that BitvecClear can use to rebuilt its hash table. +*/ +void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ + if( p==0 ) return; + assert( i>0 ); + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return; + } + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + }else{ + unsigned int j; + u32 *aiValues = pBuf; + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.aHash, 0, sizeof(p->u.aHash)); + p->nSet = 0; + for(j=0; jnSet++; + while( p->u.aHash[h] ){ + h++; + if( h>=BITVEC_NINT ) h = 0; + } + p->u.aHash[h] = aiValues[j]; + } + } + } +} + +/* +** Destroy a bitmap object. Reclaim all memory used. +*/ +void sqlite3BitvecDestroy(Bitvec *p){ + if( p==0 ) return; + if( p->iDivisor ){ + unsigned int i; + for(i=0; iu.apSub[i]); + } + } + sqlite3_free(p); +} + +/* +** Return the value of the iSize parameter specified when Bitvec *p +** was created. +*/ +u32 sqlite3BitvecSize(Bitvec *p){ + return p->iSize; +} + +#ifndef SQLITE_UNTESTABLE +/* +** Let V[] be an array of unsigned characters sufficient to hold +** up to N bits. Let I be an integer between 0 and N. 0<=I>3] |= (1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + +/* +** This routine runs an extensive test of the Bitvec code. +** +** The input is an array of integers that acts as a program +** to test the Bitvec. The integers are opcodes followed +** by 0, 1, or 3 operands, depending on the opcode. Another +** opcode follows immediately after the last operand. +** +** There are 6 opcodes numbered from 0 through 5. 0 is the +** "halt" opcode and causes the test to end. +** +** 0 Halt and return the number of errors +** 1 N S X Set N bits beginning with S and incrementing by X +** 2 N S X Clear N bits beginning with S and incrementing by X +** 3 N Set N randomly chosen bits +** 4 N Clear N randomly chosen bits +** 5 N S X Set N bits from S increment X in array only, not in bitvec +** +** The opcodes 1 through 4 perform set and clear operations are performed +** on both a Bitvec object and on a linear array of bits obtained from malloc. +** Opcode 5 works on the linear array only, not on the Bitvec. +** Opcode 5 is used to deliberately induce a fault in order to +** confirm that error detection works. +** +** At the conclusion of the test the linear array is compared +** against the Bitvec object. If there are any differences, +** an error is returned. If they are the same, zero is returned. +** +** If a memory allocation error occurs, return -1. +*/ +int sqlite3BitvecBuiltinTest(int sz, int *aOp){ + Bitvec *pBitvec = 0; + unsigned char *pV = 0; + int rc = -1; + int i, nx, pc, op; + void *pTmpSpace; + + /* Allocate the Bitvec to be tested and a linear array of + ** bits to act as the reference */ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + pTmpSpace = sqlite3_malloc64(BITVEC_SZ); + if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + + /* NULL pBitvec tests */ + sqlite3BitvecSet(0, 1); + sqlite3BitvecClear(0, 1, pTmpSpace); + + /* Run the program */ + pc = 0; + while( (op = aOp[pc])!=0 ){ + switch( op ){ + case 1: + case 2: + case 5: { + nx = 4; + i = aOp[pc+2] - 1; + aOp[pc+2] += aOp[pc+3]; + break; + } + case 3: + case 4: + default: { + nx = 2; + sqlite3_randomness(sizeof(i), &i); + break; + } + } + if( (--aOp[pc+1]) > 0 ) nx = 0; + pc += nx; + i = (i & 0x7fffffff)%sz; + if( (op & 1)!=0 ){ + SETBIT(pV, (i+1)); + if( op!=5 ){ + if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; + } + }else{ + CLEARBIT(pV, (i+1)); + sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); + } + } + + /* Test to make sure the linear array exactly matches the + ** Bitvec object. Start with the assumption that they do + ** match (rc==0). Change rc to non-zero if a discrepancy + ** is found. + */ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } + } + + /* Free allocated structure */ +bitvec_end: + sqlite3_free(pTmpSpace); + sqlite3_free(pV); + sqlite3BitvecDestroy(pBitvec); + return rc; +} +#endif /* SQLITE_UNTESTABLE */ diff --git a/third_party/sqlite3/btmutex.c b/third_party/sqlite3/btmutex.c new file mode 100644 index 000000000..be05f044d --- /dev/null +++ b/third_party/sqlite3/btmutex.c @@ -0,0 +1,310 @@ +/* +** 2007 August 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement mutexes on Btree objects. +** This code really belongs in btree.c. But btree.c is getting too +** big and we want to break it down some. This packaged seemed like +** a good breakout. +*/ +#include "third_party/sqlite3/btreeInt.inc" + +/* clang-format off */ +#ifndef SQLITE_OMIT_SHARED_CACHE +#if SQLITE_THREADSAFE + +/* +** Obtain the BtShared mutex associated with B-Tree handle p. Also, +** set BtShared.db to the database handle associated with p and the +** p->locked boolean to true. +*/ +static void lockBtreeMutex(Btree *p){ + assert( p->locked==0 ); + assert( sqlite3_mutex_notheld(p->pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + + sqlite3_mutex_enter(p->pBt->mutex); + p->pBt->db = p->db; + p->locked = 1; +} + +/* +** Release the BtShared mutex associated with B-Tree handle p and +** clear the p->locked boolean. +*/ +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){ + BtShared *pBt = p->pBt; + assert( p->locked==1 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + assert( p->db==pBt->db ); + + sqlite3_mutex_leave(pBt->mutex); + p->locked = 0; +} + +/* Forward reference */ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p); + +/* +** Enter a mutex on the given BTree object. +** +** If the object is not sharable, then no mutex is ever required +** and this routine is a no-op. The underlying mutex is non-recursive. +** But we keep a reference count in Btree.wantToLock so the behavior +** of this interface is recursive. +** +** To avoid deadlocks, multiple Btrees are locked in the same order +** by all database connections. The p->pNext is a list of other +** Btrees belonging to the same database connection as the p Btree +** which need to be locked after p. If we cannot get a lock on +** p, then first unlock all of the others on p->pNext, then wait +** for the lock to become available on p, then relock all of the +** subsequent Btrees that desire a lock. +*/ +void sqlite3BtreeEnter(Btree *p){ + /* Some basic sanity checking on the Btree. The list of Btrees + ** connected by pNext and pPrev should be in sorted order by + ** Btree.pBt value. All elements of the list should belong to + ** the same connection. Only shared Btrees are on the list. */ + assert( p->pNext==0 || p->pNext->pBt>p->pBt ); + assert( p->pPrev==0 || p->pPrev->pBtpBt ); + assert( p->pNext==0 || p->pNext->db==p->db ); + assert( p->pPrev==0 || p->pPrev->db==p->db ); + assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + + /* Check for locking consistency */ + assert( !p->locked || p->wantToLock>0 ); + assert( p->sharable || p->wantToLock==0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + /* Unless the database is sharable and unlocked, then BtShared.db + ** should already be set correctly. */ + assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db ); + + if( !p->sharable ) return; + p->wantToLock++; + if( p->locked ) return; + btreeLockCarefully(p); +} + +/* This is a helper function for sqlite3BtreeLock(). By moving +** complex, but seldom used logic, out of sqlite3BtreeLock() and +** into this routine, we avoid unnecessary stack pointer changes +** and thus help the sqlite3BtreeLock() routine to run much faster +** in the common case. +*/ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ + Btree *pLater; + + /* In most cases, we should be able to acquire the lock we + ** want without having to go through the ascending lock + ** procedure that follows. Just be sure not to block. + */ + if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ + p->pBt->db = p->db; + p->locked = 1; + return; + } + + /* To avoid deadlock, first release all locks with a larger + ** BtShared address. Then acquire our lock. Then reacquire + ** the other BtShared locks that we used to hold in ascending + ** order. + */ + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + assert( pLater->sharable ); + assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); + assert( !pLater->locked || pLater->wantToLock>0 ); + if( pLater->locked ){ + unlockBtreeMutex(pLater); + } + } + lockBtreeMutex(p); + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->wantToLock ){ + lockBtreeMutex(pLater); + } + } +} + + +/* +** Exit the recursive mutex on a Btree. +*/ +void sqlite3BtreeLeave(Btree *p){ + assert( sqlite3_mutex_held(p->db->mutex) ); + if( p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + unlockBtreeMutex(p); + } + } +} + +#ifndef NDEBUG +/* +** Return true if the BtShared mutex is held on the btree, or if the +** B-Tree is not marked as sharable. +** +** This routine is used only from within assert() statements. +*/ +int sqlite3BtreeHoldsMutex(Btree *p){ + assert( p->sharable==0 || p->locked==0 || p->wantToLock>0 ); + assert( p->sharable==0 || p->locked==0 || p->db==p->pBt->db ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->pBt->mutex) ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->db->mutex) ); + + return (p->sharable==0 || p->locked); +} +#endif + + +/* +** Enter the mutex on every Btree associated with a database +** connection. This is needed (for example) prior to parsing +** a statement since we will be comparing table and column names +** against all schemas and we do not want those schemas being +** reset out from under us. +** +** There is a corresponding leave-all procedures. +** +** Enter the mutexes in accending order by BtShared pointer address +** to avoid the possibility of deadlock when two threads with +** two or more btrees in common both try to lock all their btrees +** at the same instant. +*/ +static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ + int i; + int skipOk = 1; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p && p->sharable ){ + sqlite3BtreeEnter(p); + skipOk = 0; + } + } + db->noSharedCache = skipOk; +} +void sqlite3BtreeEnterAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeEnterAll(db); +} +static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeLeave(p); + } +} +void sqlite3BtreeLeaveAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeLeaveAll(db); +} + +#ifndef NDEBUG +/* +** Return true if the current thread holds the database connection +** mutex and all required BtShared mutexes. +** +** This routine is used inside assert() statements only. +*/ +int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ + int i; + if( !sqlite3_mutex_held(db->mutex) ){ + return 0; + } + for(i=0; inDb; i++){ + Btree *p; + p = db->aDb[i].pBt; + if( p && p->sharable && + (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ + return 0; + } + } + return 1; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Return true if the correct mutexes are held for accessing the +** db->aDb[iDb].pSchema structure. The mutexes required for schema +** access are: +** +** (1) The mutex on db +** (2) if iDb!=1, then the mutex on db->aDb[iDb].pBt. +** +** If pSchema is not NULL, then iDb is computed from pSchema and +** db using sqlite3SchemaToIndex(). +*/ +int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ + Btree *p; + assert( db!=0 ); + if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); + assert( iDb>=0 && iDbnDb ); + if( !sqlite3_mutex_held(db->mutex) ) return 0; + if( iDb==1 ) return 1; + p = db->aDb[iDb].pBt; + assert( p!=0 ); + return p->sharable==0 || p->locked==1; +} +#endif /* NDEBUG */ + +#else /* SQLITE_THREADSAFE>0 above. SQLITE_THREADSAFE==0 below */ +/* +** The following are special cases for mutex enter routines for use +** in single threaded applications that use shared cache. Except for +** these two routines, all mutex operations are no-ops in that case and +** are null #defines in btree.h. +** +** If shared cache is disabled, then all btree mutex routines, including +** the ones below, are no-ops and are null #defines in btree.h. +*/ + +void sqlite3BtreeEnter(Btree *p){ + p->pBt->db = p->db; +} +void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + p->pBt->db = p->db; + } + } +} +#endif /* if SQLITE_THREADSAFE */ + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter a mutex on a Btree given a cursor owned by that Btree. +** +** These entry points are used by incremental I/O only. Enter() is required +** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not +** the build is threadsafe. Leave() is only required by threadsafe builds. +*/ +void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +# if SQLITE_THREADSAFE +void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +# endif +#endif /* ifndef SQLITE_OMIT_INCRBLOB */ + +#endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ diff --git a/third_party/sqlite3/btree.c b/third_party/sqlite3/btree.c new file mode 100644 index 000000000..d5e9e0544 --- /dev/null +++ b/third_party/sqlite3/btree.c @@ -0,0 +1,10739 @@ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an external (disk-based) database using BTrees. +** See the header comment on "btreeInt.h" for additional information. +** Including a description of file format and an overview of operation. +*/ +#include "third_party/sqlite3/btreeInt.inc" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" + +/* clang-format off */ + +/* +** The header string that appears at the beginning of every +** SQLite database. +*/ +static const char zMagicHeader[] = SQLITE_FILE_HEADER; + +/* +** Set this global variable to 1 to enable tracing using the TRACE +** macro. +*/ +#if 0 +int sqlite3BtreeTrace=1; /* True to enable tracing */ +# define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} +#else +# define TRACE(X) +#endif + +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) + +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** A list of BtShared objects that are eligible for participation +** in shared cache. This variable has file scope during normal builds, +** but the test harness needs to access it so we make it global for +** test builds. +** +** Access to this variable is protected by SQLITE_MUTEX_STATIC_MAIN. +*/ +#ifdef SQLITE_TEST +BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#else +static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#endif +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Enable or disable the shared pager and schema features. +** +** This routine has no effect on existing database connections. +** The shared cache setting effects only future calls to +** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). +*/ +int sqlite3_enable_shared_cache(int enable){ + sqlite3GlobalConfig.sharedCacheEnabled = enable; + return SQLITE_OK; +} +#endif + + + +#ifdef SQLITE_OMIT_SHARED_CACHE + /* + ** The functions querySharedCacheTableLock(), setSharedCacheTableLock(), + ** and clearAllSharedCacheTableLocks() + ** manipulate entries in the BtShared.pLock linked list used to store + ** shared-cache table level locks. If the library is compiled with the + ** shared-cache feature disabled, then there is only ever one user + ** of each BtShared structure and so this locking is not necessary. + ** So define the lock related functions as no-ops. + */ + #define querySharedCacheTableLock(a,b,c) SQLITE_OK + #define setSharedCacheTableLock(a,b,c) SQLITE_OK + #define clearAllSharedCacheTableLocks(a) + #define downgradeAllSharedCacheTableLocks(a) + #define hasSharedCacheTableLock(a,b,c,d) 1 + #define hasReadConflicts(a, b) 0 +#endif + +#ifdef SQLITE_DEBUG +/* +** Return and reset the seek counter for a Btree object. +*/ +sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ + u64 n = pBt->nSeek; + pBt->nSeek = 0; + return n; +} +#endif + +/* +** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single +** (MemPage*) as an argument. The (MemPage*) must not be NULL. +** +** If SQLITE_DEBUG is not defined, then this macro is equivalent to +** SQLITE_CORRUPT_BKPT. Or, if SQLITE_DEBUG is set, then the log message +** normally produced as a side-effect of SQLITE_CORRUPT_BKPT is augmented +** with the page number and filename associated with the (MemPage*). +*/ +#ifdef SQLITE_DEBUG +int corruptPageError(int lineno, MemPage *p){ + char *zMsg; + sqlite3BeginBenignMalloc(); + zMsg = sqlite3_mprintf("database corruption page %d of %s", + (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + ); + sqlite3EndBenignMalloc(); + if( zMsg ){ + sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); + } + sqlite3_free(zMsg); + return SQLITE_CORRUPT_BKPT; +} +# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) +#else +# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE + +#ifdef SQLITE_DEBUG +/* +**** This function is only used as part of an assert() statement. *** +** +** Check to see if pBtree holds the required locks to read or write to the +** table with root page iRoot. Return 1 if it does and 0 if not. +** +** For example, when writing to a table with root-page iRoot via +** Btree connection pBtree: +** +** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); +** +** When writing to an index that resides in a sharable database, the +** caller should have first obtained a lock specifying the root page of +** the corresponding table. This makes things a bit more complicated, +** as this module treats each table as a separate structure. To determine +** the table corresponding to the index being written, this +** function has to search through the database schema. +** +** Instead of a lock on the table/index rooted at page iRoot, the caller may +** hold a write-lock on the schema table (root page 1). This is also +** acceptable. +*/ +static int hasSharedCacheTableLock( + Btree *pBtree, /* Handle that must hold lock */ + Pgno iRoot, /* Root page of b-tree */ + int isIndex, /* True if iRoot is the root of an index b-tree */ + int eLockType /* Required lock type (READ_LOCK or WRITE_LOCK) */ +){ + Schema *pSchema = (Schema *)pBtree->pBt->pSchema; + Pgno iTab = 0; + BtLock *pLock; + + /* If this database is not shareable, or if the client is reading + ** and has the read-uncommitted flag set, then no lock is required. + ** Return true immediately. + */ + if( (pBtree->sharable==0) + || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommit)) + ){ + return 1; + } + + /* If the client is reading or writing an index and the schema is + ** not loaded, then it is too difficult to actually check to see if + ** the correct locks are held. So do not bother - just return true. + ** This case does not come up very often anyhow. + */ + if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){ + return 1; + } + + /* Figure out the root-page that the lock should be held on. For table + ** b-trees, this is just the root page of the b-tree being read or + ** written. For index b-trees, it is the root page of the associated + ** table. */ + if( isIndex ){ + HashElem *p; + int bSeen = 0; + for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ + Index *pIdx = (Index *)sqliteHashData(p); + if( pIdx->tnum==(int)iRoot ){ + if( bSeen ){ + /* Two or more indexes share the same root page. There must + ** be imposter tables. So just return true. The assert is not + ** useful in that case. */ + return 1; + } + iTab = pIdx->pTable->tnum; + bSeen = 1; + } + } + }else{ + iTab = iRoot; + } + + /* Search for the required lock. Either a write-lock on root-page iTab, a + ** write-lock on the schema table, or (if the client is reading) a + ** read-lock on iTab will suffice. Return 1 if any of these are found. */ + for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ + if( pLock->pBtree==pBtree + && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) + && pLock->eLock>=eLockType + ){ + return 1; + } + } + + /* Failed to find the required lock. */ + return 0; +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +**** This function may be used as part of assert() statements only. **** +** +** Return true if it would be illegal for pBtree to write into the +** table or index rooted at iRoot because other shared connections are +** simultaneously reading that same table or index. +** +** It is illegal for pBtree to write if some other Btree object that +** shares the same BtShared object is currently reading or writing +** the iRoot table. Except, if the other Btree object has the +** read-uncommitted flag set, then it is OK for the other object to +** have a read cursor. +** +** For example, before writing to any part of the table or index +** rooted at page iRoot, one should call: +** +** assert( !hasReadConflicts(pBtree, iRoot) ); +*/ +static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ + BtCursor *p; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( p->pgnoRoot==iRoot + && p->pBtree!=pBtree + && 0==(p->pBtree->db->flags & SQLITE_ReadUncommit) + ){ + return 1; + } + } + return 0; +} +#endif /* #ifdef SQLITE_DEBUG */ + +/* +** Query to see if Btree handle p may obtain a lock of type eLock +** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return +** SQLITE_OK if the lock may be obtained (by calling +** setSharedCacheTableLock()), or SQLITE_LOCKED if not. +*/ +static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + assert( !(p->db->flags&SQLITE_ReadUncommit)||eLock==WRITE_LOCK||iTab==1 ); + + /* If requesting a write-lock, then the Btree must have an open write + ** transaction on this file. And, obviously, for this to be so there + ** must be an open write transaction on the file itself. + */ + assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); + assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); + + /* This routine is a no-op if the shared-cache is not enabled */ + if( !p->sharable ){ + return SQLITE_OK; + } + + /* If some other connection is holding an exclusive lock, the + ** requested lock may not be obtained. + */ + if( pBt->pWriter!=p && (pBt->btsFlags & BTS_EXCLUSIVE)!=0 ){ + sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); + return SQLITE_LOCKED_SHAREDCACHE; + } + + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + /* The condition (pIter->eLock!=eLock) in the following if(...) + ** statement is a simplification of: + ** + ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) + ** + ** since we know that if eLock==WRITE_LOCK, then no other connection + ** may hold a WRITE_LOCK on any table in this file (since there can + ** only be a single writer). + */ + assert( pIter->eLock==READ_LOCK || pIter->eLock==WRITE_LOCK ); + assert( eLock==READ_LOCK || pIter->pBtree==p || pIter->eLock==READ_LOCK); + if( pIter->pBtree!=p && pIter->iTable==iTab && pIter->eLock!=eLock ){ + sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); + if( eLock==WRITE_LOCK ){ + assert( p==pBt->pWriter ); + pBt->btsFlags |= BTS_PENDING; + } + return SQLITE_LOCKED_SHAREDCACHE; + } + } + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Add a lock on the table with root-page iTable to the shared-btree used +** by Btree handle p. Parameter eLock must be either READ_LOCK or +** WRITE_LOCK. +** +** This function assumes the following: +** +** (a) The specified Btree object p is connected to a sharable +** database (one with the BtShared.sharable flag set), and +** +** (b) No other Btree objects hold a lock that conflicts +** with the requested lock (i.e. querySharedCacheTableLock() has +** already been called and returned SQLITE_OK). +** +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** is returned if a malloc attempt fails. +*/ +static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pLock = 0; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + + /* A connection with the read-uncommitted flag set will never try to + ** obtain a read-lock using this function. The only read-lock obtained + ** by a connection in read-uncommitted mode is on the sqlite_schema + ** table, and that lock is obtained in BtreeBeginTrans(). */ + assert( 0==(p->db->flags&SQLITE_ReadUncommit) || eLock==WRITE_LOCK ); + + /* This function should only be called on a sharable b-tree after it + ** has been determined that no other b-tree holds a conflicting lock. */ + assert( p->sharable ); + assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); + + /* First search the list for an existing lock on this table. */ + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->iTable==iTable && pIter->pBtree==p ){ + pLock = pIter; + break; + } + } + + /* If the above search did not find a BtLock struct associating Btree p + ** with table iTable, allocate one and link it into the list. + */ + if( !pLock ){ + pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); + if( !pLock ){ + return SQLITE_NOMEM_BKPT; + } + pLock->iTable = iTable; + pLock->pBtree = p; + pLock->pNext = pBt->pLock; + pBt->pLock = pLock; + } + + /* Set the BtLock.eLock variable to the maximum of the current lock + ** and the requested lock. This means if a write-lock was already held + ** and a read-lock requested, we don't incorrectly downgrade the lock. + */ + assert( WRITE_LOCK>READ_LOCK ); + if( eLock>pLock->eLock ){ + pLock->eLock = eLock; + } + + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Release all the table locks (locks obtained via calls to +** the setSharedCacheTableLock() procedure) held by Btree object p. +** +** This function assumes that Btree p has an open read or write +** transaction. If it does not, then the BTS_PENDING flag +** may be incorrectly cleared. +*/ +static void clearAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + BtLock **ppIter = &pBt->pLock; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->sharable || 0==*ppIter ); + assert( p->inTrans>0 ); + + while( *ppIter ){ + BtLock *pLock = *ppIter; + assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); + assert( pLock->pBtree->inTrans>=pLock->eLock ); + if( pLock->pBtree==p ){ + *ppIter = pLock->pNext; + assert( pLock->iTable!=1 || pLock==&p->lock ); + if( pLock->iTable!=1 ){ + sqlite3_free(pLock); + } + }else{ + ppIter = &pLock->pNext; + } + } + + assert( (pBt->btsFlags & BTS_PENDING)==0 || pBt->pWriter ); + if( pBt->pWriter==p ){ + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + }else if( pBt->nTransaction==2 ){ + /* This function is called when Btree p is concluding its + ** transaction. If there currently exists a writer, and p is not + ** that writer, then the number of locks held by connections other + ** than the writer must be about to drop to zero. In this case + ** set the BTS_PENDING flag to 0. + ** + ** If there is not currently a writer, then BTS_PENDING must + ** be zero already. So this next line is harmless in that case. + */ + pBt->btsFlags &= ~BTS_PENDING; + } +} + +/* +** This function changes all write-locks held by Btree p into read-locks. +*/ +static void downgradeAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->pWriter==p ){ + BtLock *pLock; + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); + pLock->eLock = READ_LOCK; + } + } +} + +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePageOne(MemPage *pPage); /* Forward reference */ +static void releasePageNotNull(MemPage *pPage); /* Forward reference */ + +/* +***** This routine is used inside of assert() only **** +** +** Verify that the cursor holds the mutex on its BtShared +*/ +#ifdef SQLITE_DEBUG +static int cursorHoldsMutex(BtCursor *p){ + return sqlite3_mutex_held(p->pBt->mutex); +} + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ +static int cursorOwnsBtShared(BtCursor *p){ + assert( cursorHoldsMutex(p) ); + return (p->pBtree->db==p->pBt->db); +} +#endif + +/* +** Invalidate the overflow cache of the cursor passed as the first argument. +** on the shared btree structure pBt. +*/ +#define invalidateOverflowCache(pCur) (pCur->curFlags &= ~BTCF_ValidOvfl) + +/* +** Invalidate the overflow page-list cache for all cursors opened +** on the shared btree structure pBt. +*/ +static void invalidateAllOverflowCache(BtShared *pBt){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + for(p=pBt->pCursor; p; p=p->pNext){ + invalidateOverflowCache(p); + } +} + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** This function is called before modifying the contents of a table +** to invalidate any incrblob cursors that are open on the +** row or one of the rows being modified. +** +** If argument isClearTable is true, then the entire contents of the +** table is about to be deleted. In this case invalidate all incrblob +** cursors open on any row within the table with root-page pgnoRoot. +** +** Otherwise, if argument isClearTable is false, then the row with +** rowid iRow is being replaced or deleted. In this case invalidate +** only those incrblob cursors open on that specific row. +*/ +static void invalidateIncrblobCursors( + Btree *pBtree, /* The database file to check */ + Pgno pgnoRoot, /* The table that might be changing */ + i64 iRow, /* The rowid that might be changing */ + int isClearTable /* True if all rows are being deleted */ +){ + BtCursor *p; + if( pBtree->hasIncrblobCur==0 ) return; + assert( sqlite3BtreeHoldsMutex(pBtree) ); + pBtree->hasIncrblobCur = 0; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 ){ + pBtree->hasIncrblobCur = 1; + if( p->pgnoRoot==pgnoRoot && (isClearTable || p->info.nKey==iRow) ){ + p->eState = CURSOR_INVALID; + } + } + } +} + +#else + /* Stub function when INCRBLOB is omitted */ + #define invalidateIncrblobCursors(w,x,y,z) +#endif /* SQLITE_OMIT_INCRBLOB */ + +/* +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf +** page. +** +** The BtShared.pHasContent bitvec exists to work around an obscure +** bug caused by the interaction of two useful IO optimizations surrounding +** free-list leaf pages: +** +** 1) When all data is deleted from a page and the page becomes +** a free-list leaf page, the page is not written to the database +** (as free-list leaf pages contain no meaningful data). Sometimes +** such a page is not even journalled (as it will not be modified, +** why bother journalling it?). +** +** 2) When a free-list leaf page is reused, its content is not read +** from the database or written to the journal file (why should it +** be, if it is not at all meaningful?). +** +** By themselves, these optimizations work fine and provide a handy +** performance boost to bulk delete or insert operations. However, if +** a page is moved to the free-list and then reused within the same +** transaction, a problem comes up. If the page is not journalled when +** it is moved to the free-list and it is also not journalled when it +** is extracted from the free-list and reused, then the original data +** may be lost. In the event of a rollback, it may not be possible +** to restore the database to its original configuration. +** +** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** moved to become a free-list leaf page, the corresponding bit is +** set in the bitvec. Whenever a leaf page is extracted from the free-list, +** optimization 2 above is omitted if the corresponding bit is already +** set in BtShared.pHasContent. The contents of the bitvec are cleared +** at the end of every transaction. +*/ +static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ + int rc = SQLITE_OK; + if( !pBt->pHasContent ){ + assert( pgno<=pBt->nPage ); + pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); + if( !pBt->pHasContent ){ + rc = SQLITE_NOMEM_BKPT; + } + } + if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ + rc = sqlite3BitvecSet(pBt->pHasContent, pgno); + } + return rc; +} + +/* +** Query the BtShared.pHasContent vector. +** +** This function is called when a free-list leaf page is removed from the +** free-list for reuse. It returns false if it is safe to retrieve the +** page from the pager layer with the 'no-content' flag set. True otherwise. +*/ +static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ + Bitvec *p = pBt->pHasContent; + return p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTestNotNull(p, pgno)); +} + +/* +** Clear (destroy) the BtShared.pHasContent bitvec. This should be +** invoked at the conclusion of each write-transaction. +*/ +static void btreeClearHasContent(BtShared *pBt){ + sqlite3BitvecDestroy(pBt->pHasContent); + pBt->pHasContent = 0; +} + +/* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + if( pCur->iPage>=0 ){ + for(i=0; iiPage; i++){ + releasePageNotNull(pCur->apPage[i]); + } + releasePageNotNull(pCur->pPage); + pCur->iPage = -1; + } +} + +/* +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. +** +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. +*/ +static int saveCursorKey(BtCursor *pCur){ + int rc = SQLITE_OK; + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content. It is possible + ** that the current key is corrupt. In that case, it is possible that + ** the sqlite3VdbeRecordUnpack() function may overread the buffer by + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated + ** below. */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); + if( pKey ){ + rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + memset(((u8*)pKey)+pCur->nKey, 0, 9+8); + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + } + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->curFlags & BTCF_Pinned ){ + return SQLITE_CONSTRAINT_PINNED; + } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; + } + + rc = saveCursorKey(pCur); + if( rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + pCur->eState = CURSOR_REQUIRESEEK; + } + + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); + return rc; +} + +/* Forward reference */ +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); + +/* +** Save the positions of all cursors (except pExcept) that are open on +** the table with root-page iRoot. "Saving the cursor position" means that +** the location in the btree is remembered in such a way that it can be +** moved back to the same spot after the btree has been modified. This +** routine is called just before cursor pExcept is used to modify the +** table, for example in BtreeDelete() or BtreeInsert(). +** +** If there are two or more cursors on the same btree, then all such +** cursors should have their BTCF_Multiple flag set. The btreeCursor() +** routine enforces that rule. This routine only needs to be called in +** the uncommon case when pExpect has the BTCF_Multiple flag set. +** +** If pExpect!=NULL and if no other cursors are found on the same root-page, +** then the BTCF_Multiple flag on pExpect is cleared, to avoid another +** pointless call to this routine. +** +** Implementation note: This routine merely checks to see if any cursors +** need to be saved. It calls out to saveCursorsOnList() in the (unusual) +** event that cursors are in need to being saved. +*/ +static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pExcept==0 || pExcept->pBt==pBt ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; + } + if( p ) return saveCursorsOnList(p, iRoot, pExcept); + if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; + return SQLITE_OK; +} + +/* This helper routine to saveAllCursors does the actual work of saving +** the cursors if and when a cursor is found that actually requires saving. +** The common case is that no cursors need to be saved, so this routine is +** broken out from its caller to avoid unnecessary stack pointer movement. +*/ +static int SQLITE_NOINLINE saveCursorsOnList( + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ +){ + do{ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>=0 ); + btreeReleaseAllCursorPages(p); + } + } + p = p->pNext; + }while( p ); + return SQLITE_OK; +} + +/* +** Clear the current cursor position. +*/ +void sqlite3BtreeClearCursor(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + pCur->eState = CURSOR_INVALID; +} + +/* +** In this version of BtreeMoveto, pKey is a packed index record +** such as is generated by the OP_MakeRecord opcode. Unpack the +** record and then call BtreeMovetoUnpacked() to do the work. +*/ +static int btreeMoveto( + BtCursor *pCur, /* Cursor open on the btree to be searched */ + const void *pKey, /* Packed key if the btree is an index */ + i64 nKey, /* Integer key for tables. Size of pKey for indices */ + int bias, /* Bias search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; /* Status code */ + UnpackedRecord *pIdxKey; /* Unpacked index key */ + + if( pKey ){ + KeyInfo *pKeyInfo = pCur->pKeyInfo; + assert( nKey==(i64)(int)nKey ); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ + rc = SQLITE_CORRUPT_BKPT; + goto moveto_done; + } + }else{ + pIdxKey = 0; + } + rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); +moveto_done: + if( pIdxKey ){ + sqlite3DbFree(pCur->pKeyInfo->db, pIdxKey); + } + return rc; +} + +/* +** Restore the cursor to the position it was in (or as close to as possible) +** when saveCursorPosition() was called. Note that this call deletes the +** saved position info stored by saveCursorPosition(), so there can be +** at most one effective restoreCursorPosition() call after each +** saveCursorPosition(). +*/ +static int btreeRestoreCursorPosition(BtCursor *pCur){ + int rc; + int skipNext = 0; + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skipNext; + } + pCur->eState = CURSOR_INVALID; + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } + if( rc==SQLITE_OK ){ + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); + if( skipNext ) pCur->skipNext = skipNext; + if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ + pCur->eState = CURSOR_SKIPNEXT; + } + } + return rc; +} + +#define restoreCursorPosition(p) \ + (p->eState>=CURSOR_REQUIRESEEK ? \ + btreeRestoreCursorPosition(p) : \ + SQLITE_OK) + +/* +** Determine whether or not a cursor has moved from the position where +** it was last placed, or has been invalidated for any other reason. +** Cursors can move when the row they are pointing at is deleted out +** from under them, for example. Cursor might also move if a btree +** is rebalanced. +** +** Calling this routine with a NULL cursor pointer returns false. +** +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor +** back to where it ought to be if this routine returns true. +*/ +int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ + assert( EIGHT_BYTE_ALIGNMENT(pCur) + || pCur==sqlite3BtreeFakeValidCursor() ); + assert( offsetof(BtCursor, eState)==0 ); + assert( sizeof(pCur->eState)==1 ); + return CURSOR_VALID != *(u8*)pCur; +} + +/* +** Return a pointer to a fake BtCursor object that will always answer +** false to the sqlite3BtreeCursorHasMoved() routine above. The fake +** cursor returned must not be used with any other Btree interface. +*/ +BtCursor *sqlite3BtreeFakeValidCursor(void){ + static u8 fakeCursor = CURSOR_VALID; + assert( offsetof(BtCursor, eState)==0 ); + return (BtCursor*)&fakeCursor; +} + +/* +** This routine restores a cursor back to its original position after it +** has been moved by some outside activity (such as a btree rebalance or +** a row having been deleted out from under the cursor). +** +** On success, the *pDifferentRow parameter is false if the cursor is left +** pointing at exactly the same row. *pDifferntRow is the row the cursor +** was pointing to has been deleted, forcing the cursor to point to some +** nearby row. +** +** This routine should only be called for a cursor that just returned +** TRUE from sqlite3BtreeCursorHasMoved(). +*/ +int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ + int rc; + + assert( pCur!=0 ); + assert( pCur->eState!=CURSOR_VALID ); + rc = restoreCursorPosition(pCur); + if( rc ){ + *pDifferentRow = 1; + return rc; + } + if( pCur->eState!=CURSOR_VALID ){ + *pDifferentRow = 1; + }else{ + *pDifferentRow = 0; + } + return SQLITE_OK; +} + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Provide hints to the cursor. The particular hint given (and the type +** and number of the varargs parameters) is determined by the eHintType +** parameter. See the definitions of the BTREE_HINT_* macros for details. +*/ +void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ + /* Used only by system that substitute their own storage engine */ +} +#endif + +/* +** Provide flag hints to the cursor. +*/ +void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ + assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); + pCur->hints = x; +} + + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Given a page number of a regular database page, return the page +** number for the pointer-map page that contains the entry for the +** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. +*/ +static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ + int nPagesPerMapPage; + Pgno iPtrMap, ret; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; + nPagesPerMapPage = (pBt->usableSize/5)+1; + iPtrMap = (pgno-2)/nPagesPerMapPage; + ret = (iPtrMap*nPagesPerMapPage) + 2; + if( ret==PENDING_BYTE_PAGE(pBt) ){ + ret++; + } + return ret; +} + +/* +** Write an entry into the pointer map. +** +** This routine updates the pointer map entry for page number 'key' +** so that it maps to type 'eType' and parent page number 'pgno'. +** +** If *pRC is initially non-zero (non-SQLITE_OK) then this routine is +** a no-op. If an error occurs, the appropriate error code is written +** into *pRC. +*/ +static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + DbPage *pDbPage; /* The pointer map page */ + u8 *pPtrmap; /* The pointer map data */ + Pgno iPtrmap; /* The pointer map page number */ + int offset; /* Offset in pointer map page */ + int rc; /* Return code from subfunctions */ + + if( *pRC ) return; + + assert( sqlite3_mutex_held(pBt->mutex) ); + /* The super-journal page number must never be used as a pointer map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); + } + } + +ptrmap_exit: + sqlite3PagerUnref(pDbPage); +} + +/* +** Read an entry from the pointer map. +** +** This routine retrieves the pointer map entry for page 'key', writing +** the type and parent page number to *pEType and *pPgno respectively. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. +*/ +static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + DbPage *pDbPage; /* The pointer map page */ + int iPtrmap; /* Pointer map page index */ + u8 *pPtrmap; /* Pointer map page data */ + int offset; /* Offset of entry in pointer map */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + if( rc!=0 ){ + return rc; + } + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; + } + assert( offset <= (int)pBt->usableSize-5 ); + assert( pEType!=0 ); + *pEType = pPtrmap[offset]; + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + + sqlite3PagerUnref(pDbPage); + if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap); + return SQLITE_OK; +} + +#else /* if defined SQLITE_OMIT_AUTOVACUUM */ + #define ptrmapPut(w,x,y,z,rc) + #define ptrmapGet(w,x,y,z) SQLITE_OK + #define ptrmapPutOvflPtr(x, y, z, rc) +#endif + +/* +** Given a btree page and a cell index (0 means the first cell on +** the page, 1 means the second cell, and so forth) return a pointer +** to the cell content. +** +** findCellPastPtr() does the same except it skips past the initial +** 4-byte child pointer found on interior pages, if there is one. +** +** This routine works only for pages that do not contain overflow cells. +*/ +#define findCell(P,I) \ + ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) +#define findCellPastPtr(P,I) \ + ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) + + +/* +** This is common tail processing for btreeParseCellPtr() and +** btreeParseCellPtrIndex() for the case when the cell does not fit entirely +** on a single B-tree page. Make necessary adjustments to the CellInfo +** structure. +*/ +static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; + } + pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; +} + +/* +** Given a record with nPayload bytes of payload stored within btree +** page pPage, return the number of bytes of payload stored locally. +*/ +static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ + int maxLocal; /* Maximum amount of payload held locally */ + maxLocal = pPage->maxLocal; + if( nPayload<=maxLocal ){ + return nPayload; + }else{ + int minLocal; /* Minimum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + minLocal = pPage->minLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); + return ( surplus <= maxLocal ) ? surplus : minLocal; + } +} + +/* +** The following routines are implementations of the MemPage.xParseCell() +** method. +** +** Parse a cell content block and fill in the CellInfo structure. +** +** btreeParseCellPtr() => table btree leaf nodes +** btreeParseCellNoPayload() => table btree internal nodes +** btreeParseCellPtrIndex() => index btree nodes +** +** There is also a wrapper function btreeParseCell() that works for +** all MemPage types and that references the cell by index rather than +** by pointer. +*/ +static void btreeParseCellPtrNoPayload( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 ); + assert( pPage->childPtrSize==4 ); +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER(pPage); +#endif + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->pPayload = 0; + return; +} +static void btreeParseCellPtr( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + u64 iKey; /* Extracted Key value */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf ); + assert( pPage->childPtrSize==0 ); + pIter = pCell; + + /* The next block of code is equivalent to: + ** + ** pIter += getVarint32(pIter, nPayload); + ** + ** The code is inlined to avoid a function call. + */ + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( (*pIter)>=0x80 && pIternKey); + ** + ** The code is inlined to avoid a function call. + */ + iKey = *pIter; + if( iKey>=0x80 ){ + u8 *pEnd = &pIter[7]; + iKey &= 0x7f; + while(1){ + iKey = (iKey<<7) | (*++pIter & 0x7f); + if( (*pIter)<0x80 ) break; + if( pIter>=pEnd ){ + iKey = (iKey<<8) | *++pIter; + break; + } + } + } + pIter++; + + pInfo->nKey = *(i64*)&iKey; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + }else{ + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + } +} +static void btreeParseCellPtrIndex( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf==0 ); + pIter = pCell + pPage->childPtrSize; + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIternKey = nPayload; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + }else{ + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + } +} +static void btreeParseCell( + MemPage *pPage, /* Page containing the cell */ + int iCell, /* The cell index. First cell is 0 */ + CellInfo *pInfo /* Fill in this structure */ +){ + pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); +} + +/* +** The following routines are implementations of the MemPage.xCellSize +** method. +** +** Compute the total number of bytes that a Cell needs in the cell +** data area of the btree-page. The return number includes the cell +** data header and the local payload, but not any overflow page or +** the space used by the cell pointer. +** +** cellSizePtrNoPayload() => table internal nodes +** cellSizePtr() => all index nodes & table leaf nodes +*/ +static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ + /* pIter now points at the 64-bit integer key value, a variable length + ** integer. The following block moves pIter to point at the first byte + ** past the end of the key value. */ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pItermaxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#else + UNUSED_PARAMETER(pPage); +#endif + + assert( pPage->childPtrSize==4 ); + pEnd = pIter + 9; + while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); +} +#endif + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** The cell pCell is currently part of page pSrc but will ultimately be part +** of pPage. (pSrc and pPager are often the same.) If pCell contains a +** pointer to an overflow page, insert an entry into the pointer-map for +** the overflow page that will be valid after pCell has been moved to pPage. +*/ +static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ + CellInfo info; + if( *pRC ) return; + assert( pCell!=0 ); + pPage->xParseCell(pPage, pCell, &info); + if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); + ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); + } +} +#endif + + +/* +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. +*/ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ + int i; /* Loop counter */ + int pc; /* Address of the i-th cell */ + int hdr; /* Offset to the page header */ + int size; /* Size of a cell */ + int usableSize; /* Number of usable bytes on a page */ + int cellOffset; /* Offset to the cell pointer array */ + int cbrk; /* Offset to the cell content area */ + int nCell; /* Number of cells on the page */ + unsigned char *data; /* The page data */ + unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ + int iCellFirst; /* First allowable cell index */ + int iCellLast; /* Last possible cell index */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt!=0 ); + assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + temp = 0; + src = data = pPage->aData; + hdr = pPage->hdrOffset; + cellOffset = pPage->cellOffset; + nCell = pPage->nCell; + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); + iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( top>=iFree ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); + sz2 = get2byte(&data[iFree2+2]); + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + }else if( NEVER(iFree+sz>usableSize) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + + cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddriCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=iCellFirst && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); + } + data[hdr+7] = 0; + + defragment_out: + assert( pPage->nFree>=0 ); + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk>=iCellFirst ); + put2byte(&data[hdr+5], cbrk); + data[hdr+1] = 0; + data[hdr+2] = 0; + memset(&data[iCellFirst], 0, cbrk-iCellFirst); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + return SQLITE_OK; +} + +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** +** Slots on the free list that are between 1 and 3 bytes larger than nByte +** will be ignored if adding the extra space to the fragmentation count +** causes the fragmentation count to exceed 60. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ + + assert( pc>0 ); + while( pc<=maxPC ){ + /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each + ** freeblock form a big-endian integer which is the size of the freeblock + ** in bytes, including the 4-byte header. */ + size = get2byte(&aData[pc+2]); + if( (x = size - nByte)>=0 ){ + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ + if( aData[hdr+7]>57 ) return 0; + + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + iAddr = pc; + pc = get2byte(&aData[pc]); + if( pc<=iAddr+size ){ + if( pc ){ + /* The next slot in the chain is not past the end of the current slot */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; + } + } + if( pc>maxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; +} + +/* +** Allocate nByte bytes of space from within the B-Tree page passed +** as the first argument. Write into *pIdx the index into pPage->aData[] +** of the first byte of allocated space. Return either SQLITE_OK or +** an error code (usually SQLITE_CORRUPT). +** +** The caller guarantees that there is sufficient space to make the +** allocation. This routine might need to defragment in order to bring +** all the space together, however. This routine will avoid using +** the first two bytes past the cell pointer area since presumably this +** allocation is being made in order to insert a new cell, so we will +** also end up needing a new cell pointer. +*/ +static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ + u8 * const data = pPage->aData; /* Local cache of pPage->aData */ + int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ + int gap; /* First byte of gap between cell pointers and cell content */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nByte>=0 ); /* Minimum cell size is 4 */ + assert( pPage->nFree>=nByte ); + assert( pPage->nOverflow==0 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); + + assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); + gap = pPage->cellOffset + 2*pPage->nCell; + assert( gap<=65536 ); + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byte(&data[hdr+5]); + assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ + if( gap>top ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; + }else{ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. + */ + testcase( gap+2==top ); + testcase( gap+1==top ); + testcase( gap==top ); + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( pSpace ){ + int g2; + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + *pIdx = g2 = (int)(pSpace-data); + if( NEVER(g2<=gap) ){ + return SQLITE_CORRUPT_PAGE(pPage); + }else{ + return SQLITE_OK; + } + }else if( rc ){ + return rc; + } + } + + /* The request could not be fulfilled using a freelist slot. Check + ** to see if defragmentation is necessary. + */ + testcase( gap+2+nByte==top ); + if( gap+2+nByte>top ){ + assert( pPage->nCell>0 || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + assert( gap+2+nByte<=top ); + } + + + /* Allocate memory from the gap in between the cell pointer array + ** and the cell content area. The btreeComputeFreeSpace() call has already + ** validated the freelist. Given that the freelist is valid, there + ** is no way that the allocation can extend off the end of the page. + ** The assert() below verifies the previous sentence. + */ + top -= nByte; + put2byte(&data[hdr+5], top); + assert( top+nByte <= (int)pPage->pBt->usableSize ); + *pIdx = top; + return SQLITE_OK; +} + +/* +** Return a section of the pPage->aData to the freelist. +** The first byte of the new free block is pPage->aData[iStart] +** and the size of the block is iSize bytes. +** +** Adjacent freeblocks are coalesced. +** +** Even though the freeblock list was checked by btreeComputeFreeSpace(), +** that routine will not detect overlap between cells or freeblocks. Nor +** does it detect cells or freeblocks that encrouch into the reserved bytes +** at the end of the page. So do additional corruption checks inside this +** routine and return SQLITE_CORRUPT if any problems are found. +*/ +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + u16 iPtr; /* Address of ptr to next freeblock */ + u16 iFreeBlk; /* Address of the next freeblock */ + u8 hdr; /* Page header size. 0 or 100 */ + u8 nFrag = 0; /* Reduction in fragmentation */ + u16 iOrigSize = iSize; /* Original value of iSize */ + u16 x; /* Offset to cell content area */ + u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + unsigned char *data = pPage->aData; /* Page content */ + + assert( pPage->pBt!=0 ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( iSize>=4 ); /* Minimum cell size is 4 */ + assert( iStart<=pPage->pBt->usableSize-4 ); + + /* The list of freeblocks must be in ascending order. Find the + ** spot on the list where iStart should be inserted. + */ + hdr = pPage->hdrOffset; + iPtr = hdr + 1; + if( data[iPtr+1]==0 && data[iPtr]==0 ){ + iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ + }else{ + while( (iFreeBlk = get2byte(&data[iPtr]))pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer to iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; + } + } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); + data[hdr+7] -= nFrag; + } + x = get2byte(&data[hdr+5]); + if( iStart<=x ){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ + if( iStartpBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); + pPage->nFree += iOrigSize; + return SQLITE_OK; +} + +/* +** Decode the flags byte (the first byte of the header) for a page +** and initialize fields of the MemPage structure accordingly. +** +** Only the following combinations are supported. Anything different +** indicates a corrupt database files: +** +** PTF_ZERODATA +** PTF_ZERODATA | PTF_LEAF +** PTF_LEAFDATA | PTF_INTKEY +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +*/ +static int decodeFlags(MemPage *pPage, int flagByte){ + BtShared *pBt; /* A copy of pPage->pBt */ + + assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); + flagByte &= ~PTF_LEAF; + pPage->childPtrSize = 4-4*pPage->leaf; + pPage->xCellSize = cellSizePtr; + pBt = pPage->pBt; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an + ** interior table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a + ** leaf table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); + pPage->intKey = 1; + if( pPage->leaf ){ + pPage->intKeyLeaf = 1; + pPage->xParseCell = btreeParseCellPtr; + }else{ + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrNoPayload; + pPage->xParseCell = btreeParseCellPtrNoPayload; + } + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an + ** interior index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a + ** leaf index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ + return SQLITE_CORRUPT_PAGE(pPage); + } + pPage->max1bytePayload = pBt->max1bytePayload; + return SQLITE_OK; +} + +/* +** Compute the amount of freespace on the page. In other words, fill +** in the pPage->nFree field. +*/ +static int btreeComputeFreeSpace(MemPage *pPage){ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Amount of usable space on each page */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); + + usableSize = pPage->pBt->usableSize; + hdr = pPage->hdrOffset; + data = pPage->aData; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ + top = get2byteNotZero(&data[hdr+5]); + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; + iCellLast = usableSize - 4; + + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ + if( pc>0 ){ + u32 next, size; + if( pciCellLast ){ + /* Freeblock off the end of the page */ + return SQLITE_CORRUPT_PAGE(pPage); + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + nFree = nFree + size; + if( next<=pc+size+3 ) break; + pc = next; + } + if( next>0 ){ + /* Freeblock not in ascending order */ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( pc+size>(unsigned int)usableSize ){ + /* Last freeblock extends past page end */ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + + /* At this point, nFree contains the sum of the offset to the start + ** of the cell-content area plus the number of free bytes within + ** the cell-content area. If this is greater than the usable-size + ** of the page, then the page must be corrupted. This check also + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize || nFreenFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ + pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); + } + return SQLITE_OK; +} + +/* +** Set up a raw page so that it looks like a database page holding +** no entries. +*/ +static void zeroPage(MemPage *pPage, int flags){ + unsigned char *data = pPage->aData; + BtShared *pBt = pPage->pBt; + u8 hdr = pPage->hdrOffset; + u16 first; + + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage) == data ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->btsFlags & BTS_FAST_SECURE ){ + memset(&data[hdr], 0, pBt->usableSize - hdr); + } + data[hdr] = (char)flags; + first = hdr + ((flags&PTF_LEAF)==0 ? 12 : 8); + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pBt->usableSize); + pPage->nFree = (u16)(pBt->usableSize - first); + decodeFlags(pPage, flags); + pPage->cellOffset = first; + pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aCellIdx = &data[first]; + pPage->aDataOfst = &data[pPage->childPtrSize]; + pPage->nOverflow = 0; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nCell = 0; + pPage->isInit = 1; +} + + +/* +** Convert a DbPage obtained from the pager into a MemPage used by +** the btree layer. +*/ +static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ + MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pgno!=pPage->pgno ){ + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pgno==1 ? 100 : 0; + } + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + return pPage; +} + +/* +** Get a page from the pager. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). +** +** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care +** about the content of the page at this time. So do not go to the disk +** to fetch the content. Just fill in the content with zeros for now. +** If in the future we call sqlite3PagerWrite() on this page, that +** means we have started to be concerned about content and the disk +** read should occur at that point. +*/ +static int btreeGetPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc; + DbPage *pDbPage; + + assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); + assert( sqlite3_mutex_held(pBt->mutex) ); + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + if( rc ) return rc; + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + return SQLITE_OK; +} + +/* +** Retrieve a page from the pager cache. If the requested page is not +** already in the pager cache return NULL. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. +*/ +static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); + if( pDbPage ){ + return btreePageFromDbPage(pDbPage, pgno, pBt); + } + return 0; +} + +/* +** Return the size of the database file in pages. If there is any kind of +** error, return ((unsigned int)-1). +*/ +static Pgno btreePagecount(BtShared *pBt){ + return pBt->nPage; +} +Pgno sqlite3BtreeLastPage(Btree *p){ + assert( sqlite3BtreeHoldsMutex(p) ); + return btreePagecount(p->pBt); +} + +/* +** Get a page from the pager and initialize it. +** +** If pCur!=0 then the page is being fetched as part of a moveToChild() +** call. Do additional sanity checking on the page in this case. +** And if the fetch fails, this routine must decrement pCur->iPage. +** +** The page is fetched as read-write unless pCur is not NULL and is +** a read-only cursor. +** +** If an error occurs, then *ppPage is undefined. It +** may remain unchanged, or it may be set to an invalid value. +*/ +static int getAndInitPage( + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + BtCursor *pCur, /* Cursor to receive the page, or NULL */ + int bReadOnly /* True for a read-only page */ +){ + int rc; + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pCur==0 || ppPage==&pCur->pPage ); + assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); + assert( pCur==0 || pCur->iPage>0 ); + + if( pgno>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + goto getAndInitPage_error1; + } + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ + goto getAndInitPage_error1; + } + *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( (*ppPage)->isInit==0 ){ + btreePageFromDbPage(pDbPage, pgno, pBt); + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + goto getAndInitPage_error2; + } + } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); + + /* If obtaining a child page for a cursor, we must verify that the page is + ** compatible with the root page. */ + if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ + rc = SQLITE_CORRUPT_PGNO(pgno); + goto getAndInitPage_error2; + } + return SQLITE_OK; + +getAndInitPage_error2: + releasePage(*ppPage); +getAndInitPage_error1: + if( pCur ){ + pCur->iPage--; + pCur->pPage = pCur->apPage[pCur->iPage]; + } + testcase( pgno==0 ); + assert( pgno!=0 || rc==SQLITE_CORRUPT ); + return rc; +} + +/* +** Release a MemPage. This should be called once for each prior +** call to btreeGetPage. +** +** Page1 is a special case and must be released using releasePageOne(). +*/ +static void releasePageNotNull(MemPage *pPage){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); +} +static void releasePage(MemPage *pPage){ + if( pPage ) releasePageNotNull(pPage); +} +static void releasePageOne(MemPage *pPage){ + assert( pPage!=0 ); + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefPageOne(pPage->pDbPage); +} + +/* +** Get an unused page. +** +** This works just like btreeGetPage() with the addition: +** +** * If the page is already in use for some other purpose, immediately +** release it and return an SQLITE_CURRUPT error. +** * Make sure the isInit flag is clear +*/ +static int btreeGetUnusedPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc = btreeGetPage(pBt, pgno, ppPage, flags); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; + } + return rc; +} + + +/* +** During a rollback, when the pager reloads information into the cache +** so that the cache is restored to its original state at the start of +** the transaction, for each page restored this routine is called. +** +** This routine needs to reset the extra data section at the end of the +** page to agree with the restored data. +*/ +static void pageReinit(DbPage *pData){ + MemPage *pPage; + pPage = (MemPage *)sqlite3PagerGetExtra(pData); + assert( sqlite3PagerPageRefcount(pData)>0 ); + if( pPage->isInit ){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->isInit = 0; + if( sqlite3PagerPageRefcount(pData)>1 ){ + /* pPage might not be a btree page; it might be an overflow page + ** or ptrmap page or a free page. In those cases, the following + ** call to btreeInitPage() will likely return SQLITE_CORRUPT. + ** But no harm is done by this. And it is very important that + ** btreeInitPage() be called on every btree page so we make + ** the call for every page that comes in for re-initing. */ + btreeInitPage(pPage); + } + } +} + +/* +** Invoke the busy handler for a btree. +*/ +static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); +} + +/* +** Open a database file. +** +** zFilename is the name of the database file. If zFilename is NULL +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** +** If zFilename is ":memory:" then an in-memory database is created +** that is automatically destroyed when it is closed. +** +** The "flags" parameter is a bitmask that might contain bits like +** BTREE_OMIT_JOURNAL and/or BTREE_MEMORY. +** +** If the database is already opened in the same database connection +** and we are in shared cache mode, then the open will fail with an +** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared +** objects in the same database connection since doing so will lead +** to problems with locking. +*/ +int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ + const char *zFilename, /* Name of the file containing the BTree database */ + sqlite3 *db, /* Associated database handle */ + Btree **ppBtree, /* Pointer to new Btree object written here */ + int flags, /* Options */ + int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ +){ + BtShared *pBt = 0; /* Shared part of btree structure */ + Btree *p; /* Handle to return */ + sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ + int rc = SQLITE_OK; /* Result code from this function */ + u8 nReserve; /* Byte of unused space on each page */ + unsigned char zDbHeader[100]; /* Database header content */ + + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; + + /* Set the variable isMemdb to true for an in-memory database, or + ** false for a file-based database. + */ +#ifdef SQLITE_OMIT_MEMORYDB + const int isMemdb = 0; +#else + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)) + || (vfsFlags & SQLITE_OPEN_MEMORY)!=0; +#endif + + assert( db!=0 ); + assert( pVfs!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } + p = sqlite3MallocZero(sizeof(Btree)); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + p->inTrans = TRANS_NONE; + p->db = db; +#ifndef SQLITE_OMIT_SHARED_CACHE + p->lock.pBtree = p; + p->lock.iTable = 1; +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* + ** If this Btree is a candidate for shared cache, try to find an + ** existing BtShared object that we can share with + */ + if( isTempDb==0 && (isMemdb==0 || (vfsFlags&SQLITE_OPEN_URI)!=0) ){ + if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFilename = sqlite3Strlen30(zFilename)+1; + int nFullPathname = pVfs->mxPathname+1; + char *zFullPathname = sqlite3Malloc(MAX(nFullPathname,nFilename)); + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + + p->sharable = 1; + if( !zFullPathname ){ + sqlite3_free(p); + return SQLITE_NOMEM_BKPT; + } + if( isMemdb ){ + memcpy(zFullPathname, zFilename, nFilename); + }else{ + rc = sqlite3OsFullPathname(pVfs, zFilename, + nFullPathname, zFullPathname); + if( rc ){ + if( rc==SQLITE_OK_SYMLINK ){ + rc = SQLITE_OK; + }else{ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } + } + } +#if SQLITE_THREADSAFE + mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); + sqlite3_mutex_enter(mutexOpen); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); + sqlite3_mutex_enter(mutexShared); +#endif + for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ + assert( pBt->nRef>0 ); + if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager, 0)) + && sqlite3PagerVfs(pBt->pPager)==pVfs ){ + int iDb; + for(iDb=db->nDb-1; iDb>=0; iDb--){ + Btree *pExisting = db->aDb[iDb].pBt; + if( pExisting && pExisting->pBt==pBt ){ + sqlite3_mutex_leave(mutexShared); + sqlite3_mutex_leave(mutexOpen); + sqlite3_free(zFullPathname); + sqlite3_free(p); + return SQLITE_CONSTRAINT; + } + } + p->pBt = pBt; + pBt->nRef++; + break; + } + } + sqlite3_mutex_leave(mutexShared); + sqlite3_free(zFullPathname); + } +#ifdef SQLITE_DEBUG + else{ + /* In debug mode, we mark all persistent databases as sharable + ** even when they are not. This exercises the locking code and + ** gives more opportunity for asserts(sqlite3_mutex_held()) + ** statements to find locking problems. + */ + p->sharable = 1; + } +#endif + } +#endif + if( pBt==0 ){ + /* + ** The following asserts make sure that structures used by the btree are + ** the right size. This is to guard against size changes that result + ** when compiling on a different architecture. + */ + assert( sizeof(i64)==8 ); + assert( sizeof(u64)==8 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(Pgno)==4 ); + + pBt = sqlite3MallocZero( sizeof(*pBt) ); + if( pBt==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto btree_open_out; + } + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + sizeof(MemPage), flags, vfsFlags, pageReinit); + if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); + rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); + } + if( rc!=SQLITE_OK ){ + goto btree_open_out; + } + pBt->openFlags = (u8)flags; + pBt->db = db; + sqlite3PagerSetBusyHandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + p->pBt = pBt; + + pBt->pCursor = 0; + pBt->pPage1 = 0; + if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; +#if defined(SQLITE_SECURE_DELETE) + pBt->btsFlags |= BTS_SECURE_DELETE; +#elif defined(SQLITE_FAST_SECURE_DELETE) + pBt->btsFlags |= BTS_OVERWRITE; +#endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); + if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE + || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ + pBt->pageSize = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the magic name ":memory:" will create an in-memory database, then + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. + */ + if( zFilename && !isMemdb ){ + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); + } +#endif + nReserve = 0; + }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ + nReserve = zDbHeader[20]; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); +#endif + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + if( rc ) goto btree_open_out; + pBt->usableSize = pBt->pageSize - nReserve; + assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* Add the new BtShared object to the linked list sharable BtShareds. + */ + pBt->nRef = 1; + if( p->sharable ){ + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN);) + if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ + pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); + if( pBt->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto btree_open_out; + } + } + sqlite3_mutex_enter(mutexShared); + pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; + sqlite3_mutex_leave(mutexShared); + } +#endif + } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* If the new Btree uses a sharable pBtShared, then link the new + ** Btree into the list of all sharable Btrees for the same connection. + ** The list is kept in ascending order by pBt address. + */ + if( p->sharable ){ + int i; + Btree *pSib; + for(i=0; inDb; i++){ + if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ + while( pSib->pPrev ){ pSib = pSib->pPrev; } + if( (uptr)p->pBt<(uptr)pSib->pBt ){ + p->pNext = pSib; + p->pPrev = 0; + pSib->pPrev = p; + }else{ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ + pSib = pSib->pNext; + } + p->pNext = pSib->pNext; + p->pPrev = pSib; + if( p->pNext ){ + p->pNext->pPrev = p; + } + pSib->pNext = p; + } + break; + } + } + } +#endif + *ppBtree = p; + +btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ + sqlite3PagerClose(pBt->pPager, 0); + } + sqlite3_free(pBt); + sqlite3_free(p); + *ppBtree = 0; + }else{ + sqlite3_file *pFile; + + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3BtreeSetCacheSize(p, SQLITE_DEFAULT_CACHE_SIZE); + } + + pFile = sqlite3PagerFile(pBt->pPager); + if( pFile->pMethods ){ + sqlite3OsFileControlHint(pFile, SQLITE_FCNTL_PDB, (void*)&pBt->db); + } + } + if( mutexOpen ){ + assert( sqlite3_mutex_held(mutexOpen) ); + sqlite3_mutex_leave(mutexOpen); + } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); + return rc; +} + +/* +** Decrement the BtShared.nRef counter. When it reaches zero, +** remove the BtShared structure from the sharing list. Return +** true if the BtShared.nRef counter reaches zero and return +** false if it is still positive. +*/ +static int removeFromSharingList(BtShared *pBt){ +#ifndef SQLITE_OMIT_SHARED_CACHE + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) + BtShared *pList; + int removed = 0; + + assert( sqlite3_mutex_notheld(pBt->mutex) ); + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); + pBt->nRef--; + if( pBt->nRef<=0 ){ + if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; + }else{ + pList = GLOBAL(BtShared*,sqlite3SharedCacheList); + while( ALWAYS(pList) && pList->pNext!=pBt ){ + pList=pList->pNext; + } + if( ALWAYS(pList) ){ + pList->pNext = pBt->pNext; + } + } + if( SQLITE_THREADSAFE ){ + sqlite3_mutex_free(pBt->mutex); + } + removed = 1; + } + sqlite3_mutex_leave(pMainMtx); + return removed; +#else + return 1; +#endif +} + +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + + /* One of the uses of pBt->pTmpSpace is to format cells before + ** inserting them into a leaf page (function fillInCell()). If + ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes + ** by the various routines that manipulate binary cells. Which + ** can mean that fillInCell() only initializes the first 2 or 3 + ** bytes of pTmpSpace, but that the first 4 bytes are copied from + ** it into a database page. This is not actually a problem, but it + ** does cause a valgrind error when the 1 or 2 bytes of unitialized + ** data is passed to system call write(). So to avoid this error, + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } + } +} + +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } +} + +/* +** Close an open database and invalidate all cursors. +*/ +int sqlite3BtreeClose(Btree *p){ + BtShared *pBt = p->pBt; + + /* Close all cursors opened via this handle. */ + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + + /* Verify that no other cursors have this Btree open */ +#ifdef SQLITE_DEBUG + { + BtCursor *pCur = pBt->pCursor; + while( pCur ){ + BtCursor *pTmp = pCur; + pCur = pCur->pNext; + assert( pTmp->pBtree!=p ); + + } + } +#endif + + /* Rollback any active transaction and free the handle structure. + ** The call to sqlite3BtreeRollback() drops any table-locks held by + ** this handle. + */ + sqlite3BtreeRollback(p, SQLITE_OK, 0); + sqlite3BtreeLeave(p); + + /* If there are still other outstanding references to the shared-btree + ** structure, return now. The remainder of this procedure cleans + ** up the shared-btree. + */ + assert( p->wantToLock==0 && p->locked==0 ); + if( !p->sharable || removeFromSharingList(pBt) ){ + /* The pBt is no longer on the sharing list, so we can access + ** it without having to hold the mutex. + ** + ** Clean out and delete the BtShared object. + */ + assert( !pBt->pCursor ); + sqlite3PagerClose(pBt->pPager, p->db); + if( pBt->xFreeSchema && pBt->pSchema ){ + pBt->xFreeSchema(pBt->pSchema); + } + sqlite3DbFree(0, pBt->pSchema); + freeTempSpace(pBt); + sqlite3_free(pBt); + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( p->wantToLock==0 ); + assert( p->locked==0 ); + if( p->pPrev ) p->pPrev->pNext = p->pNext; + if( p->pNext ) p->pNext->pPrev = p->pPrev; +#endif + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Change the "soft" limit on the number of pages in the cache. +** Unused and unmodified pages will be recycled when the number of +** pages in the cache exceeds this soft limit. But the size of the +** cache is allowed to grow larger than this limit if it contains +** dirty pages or pages still in active use. +*/ +int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetCachesize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Change the "spill" limit on the number of pages in the cache. +** If the number of pages exceeds this limit during a write transaction, +** the pager might attempt to "spill" pages to the journal early in +** order to free up memory. +** +** The value returned is the current spill size. If zero is passed +** as an argument, no changes are made to the spill size setting, so +** using mxPage of 0 is a way to query the current spill size. +*/ +int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + int res; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return res; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** Change the way data is synced to disk in order to increase or decrease +** how well the database resists damage due to OS crashes and power +** failures. Level 1 is the same as asynchronous (no syncs() occur and +** there is a high probability of damage) Level 2 is the default. There +** is a very low but non-zero probability of damage. Level 3 reduces the +** probability of damage to near zero but with a write performance reduction. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +int sqlite3BtreeSetPagerFlags( + Btree *p, /* The btree to set the safety level on */ + unsigned pgFlags /* Various PAGER_* flags */ +){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetFlags(pBt->pPager, pgFlags); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif + +/* +** Change the default pages size and the number of reserved bytes per page. +** Or, if the page size has already been fixed, return SQLITE_READONLY +** without changing anything. +** +** The page size must be a power of 2 between 512 and 65536. If the page +** size supplied does not meet this constraint then the page size is not +** changed. +** +** Page sizes are constrained to be a power of two so that the region +** of the database file used for locking (beginning at PENDING_BYTE, +** the first byte past the 1GB boundary, 0x40000000) needs to occur +** at the beginning of a page. +** +** If parameter nReserve is less than zero, then the number of reserved +** bytes per page is left unchanged. +** +** If the iFix!=0 then the BTS_PAGESIZE_FIXED flag is set so that the page size +** and autovacuum mode can no longer be changed. +*/ +int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ + int rc = SQLITE_OK; + int x; + BtShared *pBt = p->pBt; + assert( nReserve>=0 && nReserve<=255 ); + sqlite3BtreeEnter(p); + pBt->nReserveWanted = nReserve; + x = pBt->pageSize - pBt->usableSize; + if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ + sqlite3BtreeLeave(p); + return SQLITE_READONLY; + } + assert( nReserve>=0 && nReserve<=255 ); + if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && + ((pageSize-1)&pageSize)==0 ){ + assert( (pageSize & 7)==0 ); + assert( !pBt->pCursor ); + if( nReserve>32 && pageSize==512 ) pageSize = 1024; + pBt->pageSize = (u32)pageSize; + freeTempSpace(pBt); + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + pBt->usableSize = pBt->pageSize - (u16)nReserve; + if( iFix ) pBt->btsFlags |= BTS_PAGESIZE_FIXED; + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Return the currently defined page size +*/ +int sqlite3BtreeGetPageSize(Btree *p){ + return p->pBt->pageSize; +} + +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behavior. +*/ +int sqlite3BtreeGetReserveNoMutex(Btree *p){ + int n; + assert( sqlite3_mutex_held(p->pBt->mutex) ); + n = p->pBt->pageSize - p->pBt->usableSize; + return n; +} + +/* +** Return the number of bytes of space at the end of every page that +** are intentually left unused. This is the "reserved" space that is +** sometimes used by extensions. +** +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. +*/ +int sqlite3BtreeGetRequestedReserve(Btree *p){ + int n1, n2; + sqlite3BtreeEnter(p); + n1 = (int)p->pBt->nReserveWanted; + n2 = sqlite3BtreeGetReserveNoMutex(p); + sqlite3BtreeLeave(p); + return n1>n2 ? n1 : n2; +} + + +/* +** Set the maximum page count for a database if mxPage is positive. +** No changes are made if mxPage is 0 or negative. +** Regardless of the value of mxPage, return the maximum page count. +*/ +Pgno sqlite3BtreeMaxPageCount(Btree *p, Pgno mxPage){ + Pgno n; + sqlite3BtreeEnter(p); + n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return n; +} + +/* +** Change the values for the BTS_SECURE_DELETE and BTS_OVERWRITE flags: +** +** newFlag==0 Both BTS_SECURE_DELETE and BTS_OVERWRITE are cleared +** newFlag==1 BTS_SECURE_DELETE set and BTS_OVERWRITE is cleared +** newFlag==2 BTS_SECURE_DELETE cleared and BTS_OVERWRITE is set +** newFlag==(-1) No changes +** +** This routine acts as a query if newFlag is less than zero +** +** With BTS_OVERWRITE set, deleted content is overwritten by zeros, but +** freelist leaf pages are not written back to the database. Thus in-page +** deleted content is cleared, but freelist deleted content is not. +** +** With BTS_SECURE_DELETE, operation is like BTS_OVERWRITE with the addition +** that freelist leaf pages are written back into the database, increasing +** the amount of disk I/O. +*/ +int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ + int b; + if( p==0 ) return 0; + sqlite3BtreeEnter(p); + assert( BTS_OVERWRITE==BTS_SECURE_DELETE*2 ); + assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); + if( newFlag>=0 ){ + p->pBt->btsFlags &= ~BTS_FAST_SECURE; + p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + } + b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; + sqlite3BtreeLeave(p); + return b; +} + +/* +** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' +** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it +** is disabled. The default value for the auto-vacuum property is +** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. +*/ +int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return SQLITE_READONLY; +#else + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; + u8 av = (u8)autoVacuum; + + sqlite3BtreeEnter(p); + if( (pBt->btsFlags & BTS_PAGESIZE_FIXED)!=0 && (av ?1:0)!=pBt->autoVacuum ){ + rc = SQLITE_READONLY; + }else{ + pBt->autoVacuum = av ?1:0; + pBt->incrVacuum = av==2 ?1:0; + } + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** enabled 1 is returned. Otherwise 0. +*/ +int sqlite3BtreeGetAutoVacuum(Btree *p){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return BTREE_AUTOVACUUM_NONE; +#else + int rc; + sqlite3BtreeEnter(p); + rc = ( + (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: + (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: + BTREE_AUTOVACUUM_INCR + ); + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS \ + && !defined(SQLITE_OMIT_WAL) +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt,safety_level) +#endif + +/* Forward declaration */ +static int newDatabase(BtShared*); + + +/* +** Get a reference to pPage1 of the database file. This will +** also acquire a readlock on that file. +** +** SQLITE_OK is returned on success. If the file is not a +** well-formed database file, then SQLITE_CORRUPT is returned. +** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM +** is returned if we run out of memory. +*/ +static int lockBtree(BtShared *pBt){ + int rc; /* Result code from subfunctions */ + MemPage *pPage1; /* Page 1 of the database file */ + u32 nPage; /* Number of pages in the database */ + u32 nPageFile = 0; /* Number of pages in the database file */ + u32 nPageHeader; /* Number of pages in the database according to hdr */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pBt->pPage1==0 ); + rc = sqlite3PagerSharedLock(pBt->pPager); + if( rc!=SQLITE_OK ) return rc; + rc = btreeGetPage(pBt, 1, &pPage1, 0); + if( rc!=SQLITE_OK ) return rc; + + /* Do some checking to help insure the file we opened really is + ** a valid database file. + */ + nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); + sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); + if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ + nPage = nPageFile; + } + if( (pBt->db->flags & SQLITE_ResetDatabase)!=0 ){ + nPage = 0; + } + if( nPage>0 ){ + u32 pageSize; + u32 usableSize; + u8 *page1 = pPage1->aData; + rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ + if( memcmp(page1, zMagicHeader, 16)!=0 ){ + goto page1_init_failed; + } + +#ifdef SQLITE_OMIT_WAL + if( page1[18]>1 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>1 ){ + goto page1_init_failed; + } +#else + if( page1[18]>2 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>2 ){ + goto page1_init_failed; + } + + /* If the write version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then + ** return SQLITE_OK and return without populating BtShared.pPage1. + ** The caller detects this and calls this function again. This is + ** required as the version of page 1 currently in the page1 buffer + ** may not be the latest version - there may be a newer one in the log + ** file. + */ + if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ + int isOpen = 0; + rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); + if( isOpen==0 ){ + releasePageOne(pPage1); + return SQLITE_OK; + } + } + rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); + } +#endif + + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** + ** The original design allowed these amounts to vary, but as of + ** version 3.6.0, we require them to be fixed. + */ + if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ + goto page1_init_failed; + } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ + pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 + ){ + goto page1_init_failed; + } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; + assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ + usableSize = pageSize - page1[20]; + if( (u32)pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePageOne(pPage1); + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; + freeTempSpace(pBt); + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, + pageSize-usableSize); + return rc; + } + if( sqlite3WritableSchema(pBt->db)==0 && nPage>nPageFile ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ + if( usableSize<480 ){ + goto page1_init_failed; + } + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); +#endif + } + + /* maxLocal is the maximum amount of payload to store locally for + ** a cell. Make sure it is small enough so that at least minFanout + ** cells can will fit on one page. We assume a 10-byte page header. + ** Besides the payload, the cell must store: + ** 2-byte pointer to the cell + ** 4-byte child pointer + ** 9-byte nKey value + ** 4-byte nData value + ** 4-byte overflow page pointer + ** So a cell consists of a 2-byte pointer, a header which is as much as + ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow + ** page pointer. + */ + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); + if( pBt->maxLocal>127 ){ + pBt->max1bytePayload = 127; + }else{ + pBt->max1bytePayload = (u8)pBt->maxLocal; + } + assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); + pBt->pPage1 = pPage1; + pBt->nPage = nPage; + return SQLITE_OK; + +page1_init_failed: + releasePageOne(pPage1); + pBt->pPage1 = 0; + return rc; +} + +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the database. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) + && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + +/* +** If there are no outstanding cursors and we are not in the middle +** of a transaction but there is a read lock on the database, then +** this routine unrefs the first page of the database file which +** has the effect of releasing the read lock. +** +** If there is a transaction in progress, this routine is a no-op. +*/ +static void unlockBtreeIfUnused(BtShared *pBt){ + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); + if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); + assert( sqlite3PagerRefcount(pBt->pPager)==1 ); + pBt->pPage1 = 0; + releasePageOne(pPage1); + } +} + +/* +** If pBt points to an empty file then convert that empty file +** into a new empty database by initializing the first page of +** the database. +*/ +static int newDatabase(BtShared *pBt){ + MemPage *pP1; + unsigned char *data; + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->nPage>0 ){ + return SQLITE_OK; + } + pP1 = pBt->pPage1; + assert( pP1!=0 ); + data = pP1->aData; + rc = sqlite3PagerWrite(pP1->pDbPage); + if( rc ) return rc; + memcpy(data, zMagicHeader, sizeof(zMagicHeader)); + assert( sizeof(zMagicHeader)==16 ); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); + data[18] = 1; + data[19] = 1; + assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); + data[20] = (u8)(pBt->pageSize - pBt->usableSize); + data[21] = 64; + data[22] = 32; + data[23] = 32; + memset(&data[24], 0, 100-24); + zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); + assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); + put4byte(&data[36 + 4*4], pBt->autoVacuum); + put4byte(&data[36 + 7*4], pBt->incrVacuum); +#endif + pBt->nPage = 1; + data[31] = 1; + return SQLITE_OK; +} + +/* +** Initialize the first page of the database file (creating a database +** consisting of a single page and no schema objects). Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +int sqlite3BtreeNewDb(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->nPage = 0; + rc = newDatabase(p->pBt); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Attempt to start a new transaction. A write-transaction +** is started if the second argument is nonzero, otherwise a read- +** transaction. If the second argument is 2 or more and exclusive +** transaction is started, meaning that no other process is allowed +** to access the database. A preexisting transaction may not be +** upgraded to exclusive by calling this routine a second time - the +** exclusivity flag only works for a new transaction. +** +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines +** will work unless a transaction is started first: +** +** sqlite3BtreeCreateTable() +** sqlite3BtreeCreateIndex() +** sqlite3BtreeClearTable() +** sqlite3BtreeDropTable() +** sqlite3BtreeInsert() +** sqlite3BtreeDelete() +** sqlite3BtreeUpdateMeta() +** +** If an initial attempt to acquire the lock fails because of lock contention +** and the database was previously unlocked, then invoke the busy handler +** if there is one. But if there was previously a read-lock, do not +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** returned when there is already a read-lock in order to avoid a deadlock. +** +** Suppose there are two processes A and B. A has a read lock and B has +** a reserved lock. B tries to promote to exclusive but is blocked because +** of A's read lock. A tries to promote to reserved but is blocked by B. +** One or the other of the two processes must give way or there can be +** no progress. By returning SQLITE_BUSY and not invoking the busy callback +** when A already has a read lock, we encourage A to give up and let B +** proceed. +*/ +int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt = p->pBt; + Pager *pPager = pBt->pPager; + int rc = SQLITE_OK; + + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the btree is already in a write-transaction, or it + ** is already in a read-transaction and a read-transaction + ** is requested, this is a no-op. + */ + if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ + goto trans_begun; + } + assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } + + /* Write transactions are not possible on a read-only database */ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ + rc = SQLITE_READONLY; + goto trans_begun; + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + { + sqlite3 *pBlock = 0; + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } + } + } + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; + } + } +#endif + + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock + ** on page 1, the transaction cannot be opened. */ + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); + if( SQLITE_OK!=rc ) goto trans_begun; + + pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; + if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; + do { + sqlite3PagerWalDb(pPager, p->db); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If transitioning from no transaction directly to a write transaction, + ** block for the WRITER lock first if possible. */ + if( pBt->pPage1==0 && wrflag ){ + assert( pBt->inTransaction==TRANS_NONE ); + rc = sqlite3PagerWalWriteLock(pPager, 1); + if( rc!=SQLITE_BUSY && rc!=SQLITE_OK ) break; + } +#endif + + /* Call lockBtree() until either pBt->pPage1 is populated or + ** lockBtree() returns something other than SQLITE_OK. lockBtree() + ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after + ** reading page 1 it discovers that the page-size of the database + ** file is not pBt->pageSize. In this case lockBtree() will update + ** pBt->pageSize to the page-size of the file on disk. + */ + while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); + + if( rc==SQLITE_OK && wrflag ){ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ + rc = SQLITE_READONLY; + }else{ + rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ + /* if there was no transaction opened when this function was + ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error + ** code to SQLITE_BUSY. */ + rc = SQLITE_BUSY; + } + } + } + + if( rc!=SQLITE_OK ){ + (void)sqlite3PagerWalWriteLock(pPager, 0); + unlockBtreeIfUnused(pBt); + } + }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && + btreeInvokeBusyHandler(pBt) ); + sqlite3PagerWalDb(pPager, 0); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif + + if( rc==SQLITE_OK ){ + if( p->inTrans==TRANS_NONE ){ + pBt->nTransaction++; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( p->sharable ){ + assert( p->lock.pBtree==p && p->lock.iTable==1 ); + p->lock.eLock = READ_LOCK; + p->lock.pNext = pBt->pLock; + pBt->pLock = &p->lock; + } +#endif + } + p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); + if( p->inTrans>pBt->inTransaction ){ + pBt->inTransaction = p->inTrans; + } + if( wrflag ){ + MemPage *pPage1 = pBt->pPage1; +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( !pBt->pWriter ); + pBt->pWriter = p; + pBt->btsFlags &= ~BTS_EXCLUSIVE; + if( wrflag>1 ) pBt->btsFlags |= BTS_EXCLUSIVE; +#endif + + /* If the db-size header field is incorrect (as it may be if an old + ** client has been writing the database file), update it now. Doing + ** this sooner rather than later means the database size can safely + ** re-read the database size from page 1 if a savepoint or transaction + ** rollback occurs within the transaction. + */ + if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pPage1->aData[28], pBt->nPage); + } + } + } + } + +trans_begun: + if( rc==SQLITE_OK ){ + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } + } + + btreeIntegrity(p); + sqlite3BtreeLeave(p); + return rc; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM + +/* +** Set the pointer-map entries for all children of page pPage. Also, if +** pPage contains cells that point to overflow pages, set the pointer +** map entries for the overflow pages as well. +*/ +static int setChildPtrmaps(MemPage *pPage){ + int i; /* Counter variable */ + int nCell; /* Number of cells in page pPage */ + int rc; /* Return code */ + BtShared *pBt = pPage->pBt; + Pgno pgno = pPage->pgno; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); + if( rc!=SQLITE_OK ) return rc; + nCell = pPage->nCell; + + for(i=0; ileaf ){ + Pgno childPgno = get4byte(pCell); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + } + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + + return rc; +} + +/* +** Somewhere on pPage is a pointer to page iFrom. Modify this pointer so +** that it points to iTo. Parameter eType describes the type of pointer to +** be modified, as follows: +** +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** page of pPage. +** +** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow +** page pointed to by one of the cells on pPage. +** +** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next +** overflow page in the list. +*/ +static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( eType==PTRMAP_OVERFLOW2 ){ + /* The pointer is always the first 4 bytes of the page in this case. */ + if( get4byte(pPage->aData)!=iFrom ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + put4byte(pPage->aData, iTo); + }else{ + int i; + int nCell; + int rc; + + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); + if( rc ) return rc; + nCell = pPage->nCell; + + for(i=0; ixParseCell(pPage, pCell, &info); + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } + } + }else{ + if( get4byte(pCell)==iFrom ){ + put4byte(pCell, iTo); + break; + } + } + } + + if( i==nCell ){ + if( eType!=PTRMAP_BTREE || + get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); + } + } + return SQLITE_OK; +} + + +/* +** Move the open database page pDbPage to location iFreePage in the +** database. The pDbPage reference remains valid. +** +** The isCommit flag indicates that there is no need to remember that +** the journal needs to be sync()ed before database page pDbPage->pgno +** can be written to. The caller has already promised not to write to that +** page. +*/ +static int relocatePage( + BtShared *pBt, /* Btree */ + MemPage *pDbPage, /* Open page to move */ + u8 eType, /* Pointer map 'type' entry for pDbPage */ + Pgno iPtrPage, /* Pointer map 'page-no' entry for pDbPage */ + Pgno iFreePage, /* The location to move pDbPage to */ + int isCommit /* isCommit flag passed to sqlite3PagerMovepage */ +){ + MemPage *pPtrPage; /* The page that contains a pointer to pDbPage */ + Pgno iDbPage = pDbPage->pgno; + Pager *pPager = pBt->pPager; + int rc; + + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; + + /* Move page iDbPage from its current location to page number iFreePage */ + TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + iDbPage, iFreePage, iPtrPage, eType)); + rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); + if( rc!=SQLITE_OK ){ + return rc; + } + pDbPage->pgno = iFreePage; + + /* If pDbPage was a btree-page, then it may have child pages and/or cells + ** that point to overflow pages. The pointer map entries for all these + ** pages need to be changed. + ** + ** If pDbPage is an overflow page, then the first 4 bytes may store a + ** pointer to a subsequent overflow page. If this is the case, then + ** the pointer map needs to be updated for the subsequent overflow page. + */ + if( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ){ + rc = setChildPtrmaps(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + Pgno nextOvfl = get4byte(pDbPage->aData); + if( nextOvfl!=0 ){ + ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage, &rc); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + + /* Fix the database pointer on page iPtrPage that pointed at iDbPage so + ** that it points at iFreePage. Also fix the pointer map entry for + ** iPtrPage. + */ + if( eType!=PTRMAP_ROOTPAGE ){ + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pPtrPage->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pPtrPage); + return rc; + } + rc = modifyPagePointer(pPtrPage, iDbPage, iFreePage, eType); + releasePage(pPtrPage); + if( rc==SQLITE_OK ){ + ptrmapPut(pBt, iFreePage, eType, iPtrPage, &rc); + } + } + return rc; +} + +/* Forward declaration required by incrVacuumStep(). */ +static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); + +/* +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. +** +** More specifically, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. +** +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commit +** operation, or false for an incremental vacuum. +*/ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ + Pgno nFreeList; /* Number of pages still on the free-list */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iLastPg>nFin ); + + if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ + u8 eType; + Pgno iPtrPage; + + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 ){ + return SQLITE_DONE; + } + + rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); + if( rc!=SQLITE_OK ){ + return rc; + } + if( eType==PTRMAP_ROOTPAGE ){ + return SQLITE_CORRUPT_BKPT; + } + + if( eType==PTRMAP_FREEPAGE ){ + if( bCommit==0 ){ + /* Remove the page from the files free-list. This is not required + ** if bCommit is non-zero. In that case, the free-list will be + ** truncated to zero after this function returns, so it doesn't + ** matter if it still contains some garbage entries. + */ + Pgno iFreePg; + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( iFreePg==iLastPg ); + releasePage(pFreePg); + } + } else { + Pgno iFreePg; /* Index of free page to move pLastPg to */ + MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ + + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If bCommit is zero, this loop runs exactly once and page pLastPg + ** is swapped with the first free page pulled off the free list. + ** + ** On the other hand, if bCommit is greater than zero, then keep + ** looping until a free-page located within the first nFin pages + ** of the file is found. + */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } + do { + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); + if( rc!=SQLITE_OK ){ + releasePage(pLastPg); + return rc; + } + releasePage(pFreePg); + }while( bCommit && iFreePg>nFin ); + assert( iFreePgbDoTruncate = 1; + pBt->nPage = iLastPg; + } + return SQLITE_OK; +} + +/* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinpBt; + + sqlite3BtreeEnter(p); + assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); + if( !pBt->autoVacuum ){ + rc = SQLITE_DONE; + }else{ + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig=nOrig ){ + rc = SQLITE_CORRUPT_BKPT; + }else if( nFree>0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; + } + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine is called prior to sqlite3PagerCommit when a transaction +** is committed for an auto-vacuum database. +** +** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages +** the database file should be truncated to during the commit process. +** i.e. the database has been reorganized so that only the first *pnTrunc +** pages are in use. +*/ +static int autoVacuumCommit(BtShared *pBt){ + int rc = SQLITE_OK; + Pager *pPager = pBt->pPager; + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) + + assert( sqlite3_mutex_held(pBt->mutex) ); + invalidateAllOverflowCache(pBt); + assert(pBt->autoVacuum); + if( !pBt->incrVacuum ){ + Pgno nFin; /* Number of pages in database after autovacuuming */ + Pgno nFree; /* Number of pages on the freelist initially */ + Pgno iFree; /* The next page to be freed */ + Pgno nOrig; /* Database size before freeing */ + + nOrig = btreePagecount(pBt); + if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ + /* It is not possible to create a database for which the final page + ** is either a pointer-map page or the pending-byte page. If one + ** is encountered, this indicates corruption. + */ + return SQLITE_CORRUPT_BKPT; + } + + nFree = get4byte(&pBt->pPage1->aData[36]); + nFin = finalDbSize(pBt, nOrig, nFree); + if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; + if( nFinnFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree, 1); + } + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + put4byte(&pBt->pPage1->aData[28], nFin); + pBt->bDoTruncate = 1; + pBt->nPage = nFin; + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); + } + } + + assert( nRef>=sqlite3PagerRefcount(pPager) ); + return rc; +} + +#else /* ifndef SQLITE_OMIT_AUTOVACUUM */ +# define setChildPtrmaps(x) SQLITE_OK +#endif + +/* +** This routine does the first phase of a two-phase commit. This routine +** causes a rollback journal to be created (if it does not already exist) +** and populated with enough information so that if a power loss occurs +** the database can be restored to its original state by playing back +** the journal. Then the contents of the journal are flushed out to +** the disk. After the journal is safely on oxide, the changes to the +** database are written into the database file and flushed to oxide. +** At the end of this call, the rollback journal still exists on the +** disk and we are still holding all locks, so the transaction has not +** committed. See sqlite3BtreeCommitPhaseTwo() for the second phase of the +** commit process. +** +** This call is a no-op if no write-transaction is currently active on pBt. +** +** Otherwise, sync the database file for the btree pBt. zSuperJrnl points to +** the name of a super-journal file that should be written into the +** individual journal file, or is NULL, indicating no super-journal file +** (single database transaction). +** +** When this is called, the super-journal should already have been +** created, populated with this journal pointer and synced to disk. +** +** Once this is routine has returned, the only thing required to commit +** the write-transaction for this database file is to delete the journal. +*/ +int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ + int rc = SQLITE_OK; + if( p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + rc = autoVacuumCommit(pBt); + if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); + return rc; + } + } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } +#endif + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback() +** at the conclusion of a transaction. +*/ +static void btreeEndTransaction(Btree *p){ + BtShared *pBt = p->pBt; + sqlite3 *db = p->db; + assert( sqlite3BtreeHoldsMutex(p) ); + +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif + if( p->inTrans>TRANS_NONE && db->nVdbeRead>1 ){ + /* If there are other active statements that belong to this database + ** handle, downgrade to a read-only transaction. The other statements + ** may still be reading from the database. */ + downgradeAllSharedCacheTableLocks(p); + p->inTrans = TRANS_READ; + }else{ + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count + ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() + ** call below will unlock the pager. */ + if( p->inTrans!=TRANS_NONE ){ + clearAllSharedCacheTableLocks(p); + pBt->nTransaction--; + if( 0==pBt->nTransaction ){ + pBt->inTransaction = TRANS_NONE; + } + } + + /* Set the current transaction state to TRANS_NONE and unlock the + ** pager if this call closed the only read or write transaction. */ + p->inTrans = TRANS_NONE; + unlockBtreeIfUnused(pBt); + } + + btreeIntegrity(p); +} + +/* +** Commit the transaction currently in progress. +** +** This routine implements the second phase of a 2-phase commit. The +** sqlite3BtreeCommitPhaseOne() routine does the first phase and should +** be invoked prior to calling this routine. The sqlite3BtreeCommitPhaseOne() +** routine did all the work of writing information out to disk and flushing the +** contents so that they are written onto the disk platter. All this +** routine has to do is delete or truncate or zero the header in the +** the rollback journal (which causes the transaction to commit) and +** drop locks. +** +** Normally, if an error occurs while the pager layer is attempting to +** finalize the underlying journal file, this function returns an error and +** the upper layer will attempt a rollback. However, if the second argument +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a super-journal file) and the caller will ignore this +** functions return code. So, even if an error occurs in the pager layer, +** reset the b-tree objects internal state to indicate that the write +** transaction has been closed. This is quite safe, as the pager will have +** transitioned to the error state. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ + + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the handle has a write-transaction open, commit the shared-btrees + ** transaction and set the shared state to TRANS_READ. + */ + if( p->inTrans==TRANS_WRITE ){ + int rc; + BtShared *pBt = p->pBt; + assert( pBt->inTransaction==TRANS_WRITE ); + assert( pBt->nTransaction>0 ); + rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); + if( rc!=SQLITE_OK && bCleanup==0 ){ + sqlite3BtreeLeave(p); + return rc; + } + p->iBDataVersion--; /* Compensate for pPager->iDataVersion++; */ + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Do both phases of a commit. +*/ +int sqlite3BtreeCommit(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + rc = sqlite3BtreeCommitPhaseOne(p, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeCommitPhaseTwo(p, 0); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine sets the state to CURSOR_FAULT and the error +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. +** +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. +** +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. +*/ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ + BtCursor *p; + int rc = SQLITE_OK; + + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ + rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + btreeReleaseAllCursorPages(p); + } + sqlite3BtreeLeave(pBtree); + } + return rc; +} + +/* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; +} + +/* +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ + int rc; + BtShared *pBt = p->pBt; + MemPage *pPage1; + + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); + sqlite3BtreeEnter(p); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; + } + btreeIntegrity(p); + + if( p->inTrans==TRANS_WRITE ){ + int rc2; + + assert( TRANS_WRITE==pBt->inTransaction ); + rc2 = sqlite3PagerRollback(pBt->pPager); + if( rc2!=SQLITE_OK ){ + rc = rc2; + } + + /* The rollback may have destroyed the pPage1->aData value. So + ** call btreeGetPage() on page 1 again to make + ** sure pPage1->aData is set correctly. */ + if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + btreeSetNPage(pBt, pPage1); + releasePageOne(pPage1); + } + assert( countValidCursors(pBt, 1)==0 ); + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Start a statement subtransaction. The subtransaction can be rolled +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically +** if the main transaction commits or rolls back. +** +** Statement subtransactions are used around individual SQL statements +** that are contained within a BEGIN...COMMIT block. If a constraint +** error occurs within the statement, the effect of that one statement +** can be rolled back without having to rollback the entire transaction. +** +** A statement sub-transaction is implemented as an anonymous savepoint. The +** value passed as the second parameter is the total number of savepoints, +** including the new anonymous savepoint, open on the B-Tree. i.e. if there +** are no active savepoints and no other statement-transactions open, +** iStatement is 1. This anonymous savepoint can be released or rolled back +** using the sqlite3BtreeSavepoint() function. +*/ +int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( iStatement>0 ); + assert( iStatement>p->db->nSavepoint ); + assert( pBt->inTransaction==TRANS_WRITE ); + /* At the pager level, a statement transaction is a savepoint with + ** an index greater than all savepoints created explicitly using + ** SQL statements. It is illegal to open, release or rollback any + ** such savepoints while the statement transaction savepoint is active. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** The second argument to this function, op, is always SAVEPOINT_ROLLBACK +** or SAVEPOINT_RELEASE. This function either releases or rolls back the +** savepoint identified by parameter iSavepoint, depending on the value +** of op. +** +** Normally, iSavepoint is greater than or equal to zero. However, if op is +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** contents of the entire transaction are rolled back. This is different +** from a normal transaction rollback, as no locks are released and the +** transaction remains open. +*/ +int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ + int rc = SQLITE_OK; + if( p && p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); + sqlite3BtreeEnter(p); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } + if( rc==SQLITE_OK ){ + if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ + pBt->nPage = 0; + } + rc = newDatabase(pBt); + btreeSetNPage(pBt, pBt->pPage1); + + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); + } + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** Create a new cursor for the BTree whose root is on the page +** iTable. If a read-only cursor is requested, it is assumed that +** the caller already has at least a read-only transaction open +** on the database already. If a write-cursor is requested, then +** the caller is assumed to have an open write transaction. +** +** If the BTREE_WRCSR bit of wrFlag is clear, then the cursor can only +** be used for reading. If the BTREE_WRCSR bit is set, then the cursor +** can be used for reading or for writing if other conditions for writing +** are also met. These are the conditions that must be met in order +** for writing to be allowed: +** +** 1: The cursor must have been opened with wrFlag containing BTREE_WRCSR +** +** 2: Other database connections that share the same pager cache +** but which are not in the READ_UNCOMMITTED state may not have +** cursors open with wrFlag==0 on the same table. Otherwise +** the changes made by this write cursor would be visible to +** the read cursors in the other database connection. +** +** 3: The database must be writable (not on read-only media) +** +** 4: There must be an active transaction. +** +** The BTREE_FORDELETE bit of wrFlag may optionally be set if BTREE_WRCSR +** is set. If FORDELETE is set, that is a hint to the implementation that +** this cursor will only be used to seek to and delete entries of an index +** as part of a larger DELETE statement. The FORDELETE hint is not used by +** this implementation. But in a hypothetical alternative storage engine +** in which index entries are automatically deleted when corresponding table +** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE +** operations on this cursor can be no-ops and all READ operations can +** return a null row (2-bytes: 0x01 0x00). +** +** No checking is done to make sure that page iTable really is the +** root page of a b-tree. If it is not, then the cursor acquired +** will not work correctly. +** +** It is assumed that the sqlite3BtreeCursorZero() has been called +** on pCur to initialize the memory space prior to invoking this routine. +*/ +static int btreeCursor( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + BtShared *pBt = p->pBt; /* Shared b-tree handle */ + BtCursor *pX; /* Looping over other all cursors */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + ); + + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with + ** this lock. The iTable<1 term disables the check for corrupt schemas. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) + || iTable<1 ); + assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); + + /* Assert that the caller has opened the required transaction. */ + assert( p->inTrans>TRANS_NONE ); + assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1 && pBt->pPage1->aData ); + assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); + + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; + } + if( iTable<=1 ){ + if( iTable<1 ){ + return SQLITE_CORRUPT_BKPT; + }else if( btreePagecount(pBt)==0 ){ + assert( wrFlag==0 ); + iTable = 0; + } + } + + /* Now that no other errors can occur, finish filling in the BtCursor + ** variables and link the cursor into the BtShared list. */ + pCur->pgnoRoot = iTable; + pCur->iPage = -1; + pCur->pKeyInfo = pKeyInfo; + pCur->pBtree = p; + pCur->pBt = pBt; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; + pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + /* If there are two or more cursors on the same btree, then all such + ** cursors *must* have the BTCF_Multiple flag set. */ + for(pX=pBt->pCursor; pX; pX=pX->pNext){ + if( pX->pgnoRoot==iTable ){ + pX->curFlags |= BTCF_Multiple; + pCur->curFlags |= BTCF_Multiple; + } + } + pCur->pNext = pBt->pCursor; + pBt->pCursor = pCur; + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; +} +static int btreeCursorWithLock( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} +int sqlite3BtreeCursor( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ +){ + if( p->sharable ){ + return btreeCursorWithLock(p, iTable, wrFlag, pKeyInfo, pCur); + }else{ + return btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + } +} + +/* +** Return the size of a BtCursor object in bytes. +** +** This interfaces is needed so that users of cursors can preallocate +** sufficient storage to hold a cursor. The BtCursor object is opaque +** to users so they cannot do the sizeof() themselves - they must call +** this routine. +*/ +int sqlite3BtreeCursorSize(void){ + return ROUND8(sizeof(BtCursor)); +} + +/* +** Initialize memory that will be converted into a BtCursor object. +** +** The simple approach here would be to memset() the entire object +** to zero. But it turns out that the apPage[] and aiIdx[] arrays +** do not need to be zeroed and they are large, so we can save a lot +** of run-time by skipping the initialization of those elements. +*/ +void sqlite3BtreeCursorZero(BtCursor *p){ + memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); +} + +/* +** Close a cursor. The read lock on the database file is released +** when the last cursor is closed. +*/ +int sqlite3BtreeCloseCursor(BtCursor *pCur){ + Btree *pBtree = pCur->pBtree; + if( pBtree ){ + BtShared *pBt = pCur->pBt; + sqlite3BtreeEnter(pBtree); + assert( pBt->pCursor!=0 ); + if( pBt->pCursor==pCur ){ + pBt->pCursor = pCur->pNext; + }else{ + BtCursor *pPrev = pBt->pCursor; + do{ + if( pPrev->pNext==pCur ){ + pPrev->pNext = pCur->pNext; + break; + } + pPrev = pPrev->pNext; + }while( ALWAYS(pPrev) ); + } + btreeReleaseAllCursorPages(pCur); + unlockBtreeIfUnused(pBt); + sqlite3_free(pCur->aOverflow); + sqlite3_free(pCur->pKey); + if( (pBt->openFlags & BTREE_SINGLE) && pBt->pCursor==0 ){ + /* Since the BtShared is not sharable, there is no need to + ** worry about the missing sqlite3BtreeLeave() call here. */ + assert( pBtree->sharable==0 ); + sqlite3BtreeClose(pBtree); + }else{ + sqlite3BtreeLeave(pBtree); + } + pCur->pBtree = 0; + } + return SQLITE_OK; +} + +/* +** Make sure the BtCursor* given in the argument has a valid +** BtCursor.info structure. If it is not already valid, call +** btreeParseCell() to fill it in. +** +** BtCursor.info is a cache of the information in the current cell. +** Using this cache reduces the number of calls to btreeParseCell(). +*/ +#ifndef NDEBUG + static int cellInfoEqual(CellInfo *a, CellInfo *b){ + if( a->nKey!=b->nKey ) return 0; + if( a->pPayload!=b->pPayload ) return 0; + if( a->nPayload!=b->nPayload ) return 0; + if( a->nLocal!=b->nLocal ) return 0; + if( a->nSize!=b->nSize ) return 0; + return 1; + } + static void assertCellInfo(BtCursor *pCur){ + CellInfo info; + memset(&info, 0, sizeof(info)); + btreeParseCell(pCur->pPage, pCur->ix, &info); + assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) ); + } +#else + #define assertCellInfo(x) +#endif +static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + pCur->curFlags |= BTCF_ValidNKey; + btreeParseCell(pCur->pPage,pCur->ix,&pCur->info); + }else{ + assertCellInfo(pCur); + } +} + +#ifndef NDEBUG /* The next routine used only within assert() statements */ +/* +** Return true if the given BtCursor is valid. A valid cursor is one +** that is currently pointing to a row in a (non-empty) table. +** This is a verification routine is used only within assert() statements. +*/ +int sqlite3BtreeCursorIsValid(BtCursor *pCur){ + return pCur && pCur->eState==CURSOR_VALID; +} +#endif /* NDEBUG */ +int sqlite3BtreeCursorIsValidNN(BtCursor *pCur){ + assert( pCur!=0 ); + return pCur->eState==CURSOR_VALID; +} + +/* +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. +*/ +i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); + getCellInfo(pCur); + return pCur->info.nKey; +} + +/* +** Pin or unpin a cursor. +*/ +void sqlite3BtreeCursorPin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)==0 ); + pCur->curFlags |= BTCF_Pinned; +} +void sqlite3BtreeCursorUnpin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)!=0 ); + pCur->curFlags &= ~BTCF_Pinned; +} + +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +/* +** Return the offset into the database file for the start of the +** payload to which the cursor is pointing. +*/ +i64 sqlite3BtreeOffset(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + + (i64)(pCur->info.pPayload - pCur->pPage->aData); +} +#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ + +/* +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. +** +** The caller must guarantee that the cursor is pointing to a non-NULL +** valid entry. In other words, the calling procedure must guarantee +** that the cursor has Cursor.eState==CURSOR_VALID. +*/ +u32 sqlite3BtreePayloadSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + return pCur->info.nPayload; +} + +/* +** Return an upper bound on the size of any record for the table +** that the cursor is pointing into. +** +** This is an optimization. Everything will still work if this +** routine always returns 2147483647 (which is the largest record +** that SQLite can handle) or more. But returning a smaller value might +** prevent large memory allocations when trying to interpret a +** corrupt datrabase. +** +** The current implementation merely returns the size of the underlying +** database file. +*/ +sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + return pCur->pBt->pageSize * (sqlite3_int64)pCur->pBt->nPage; +} + +/* +** Given the page number of an overflow page in the database (parameter +** ovfl), this function finds the page number of the next page in the +** linked list of overflow pages. If possible, it uses the auto-vacuum +** pointer-map data instead of reading the content of page ovfl to do so. +** +** If an error occurs an SQLite error code is returned. Otherwise: +** +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. +** +** If ppPage is not NULL, and a reference to the MemPage object corresponding +** to page number pOvfl was obtained, then *ppPage is set to point to that +** reference. It is the responsibility of the caller to call releasePage() +** on *ppPage to free the reference. In no reference was obtained (because +** the pointer-map was used to obtain the value for *pPgnoNext), then +** *ppPage is set to zero. +*/ +static int getOverflowPage( + BtShared *pBt, /* The database file */ + Pgno ovfl, /* Current overflow page number */ + MemPage **ppPage, /* OUT: MemPage handle (may be NULL) */ + Pgno *pPgnoNext /* OUT: Next overflow page number */ +){ + Pgno next = 0; + MemPage *pPage = 0; + int rc = SQLITE_OK; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert(pPgnoNext); + +#ifndef SQLITE_OMIT_AUTOVACUUM + /* Try to find the next page in the overflow list using the + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page + ** number ovfl to determine the next page number. + */ + if( pBt->autoVacuum ){ + Pgno pgno; + Pgno iGuess = ovfl+1; + u8 eType; + + while( PTRMAP_ISPAGE(pBt, iGuess) || iGuess==PENDING_BYTE_PAGE(pBt) ){ + iGuess++; + } + + if( iGuess<=btreePagecount(pBt) ){ + rc = ptrmapGet(pBt, iGuess, &eType, &pgno); + if( rc==SQLITE_OK && eType==PTRMAP_OVERFLOW2 && pgno==ovfl ){ + next = iGuess; + rc = SQLITE_DONE; + } + } + } +#endif + + assert( next==0 || rc==SQLITE_DONE ); + if( rc==SQLITE_OK ){ + rc = btreeGetPage(pBt, ovfl, &pPage, (ppPage==0) ? PAGER_GET_READONLY : 0); + assert( rc==SQLITE_OK || pPage==0 ); + if( rc==SQLITE_OK ){ + next = get4byte(pPage->aData); + } + } + + *pPgnoNext = next; + if( ppPage ){ + *ppPage = pPage; + }else{ + releasePage(pPage); + } + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Copy data from a buffer to a page, or from a page to a buffer. +** +** pPayload is a pointer to data stored on database page pDbPage. +** If argument eOp is false, then nByte bytes of data are copied +** from pPayload to the buffer pointed at by pBuf. If eOp is true, +** then sqlite3PagerWrite() is called on pDbPage and nByte bytes +** of data are copied from the buffer pBuf to pPayload. +** +** SQLITE_OK is returned on success, otherwise an error code. +*/ +static int copyPayload( + void *pPayload, /* Pointer to page data */ + void *pBuf, /* Pointer to buffer */ + int nByte, /* Number of bytes to copy */ + int eOp, /* 0 -> copy from page, 1 -> copy to page */ + DbPage *pDbPage /* Page containing pPayload */ +){ + if( eOp ){ + /* Copy data from buffer to page (a write operation) */ + int rc = sqlite3PagerWrite(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + memcpy(pPayload, pBuf, nByte); + }else{ + /* Copy data from page to buffer (a read operation) */ + memcpy(pBuf, pPayload, nByte); + } + return SQLITE_OK; +} + +/* +** This function is used to read or overwrite payload information +** for the entry that the pCur cursor is pointing to. The eOp +** argument is interpreted as follows: +** +** 0: The operation is a read. Populate the overflow cache. +** 1: The operation is a write. Populate the overflow cache. +** +** A total of "amt" bytes are read or written beginning at "offset". +** Data is read to or from the buffer pBuf. +** +** The content being read or written might appear on the main page +** or be scattered out on multiple overflow pages. +** +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset +** more efficient. +** +** Once an overflow page-list cache has been allocated, it must be +** invalidated if some other cursor writes to the same table, or if +** the cursor is moved to a different row. Additionally, in auto-vacuum +** mode, the following events may invalidate an overflow page-list cache. +** +** * An incremental vacuum, +** * A commit in auto_vacuum="full" mode, +** * Creating a table (may require moving an overflow page). +*/ +static int accessPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 offset, /* Begin reading this far into payload */ + u32 amt, /* Read this many bytes */ + unsigned char *pBuf, /* Write the bytes into this buffer */ + int eOp /* zero to read. non-zero to write. */ +){ + unsigned char *aPayload; + int rc = SQLITE_OK; + int iIdx = 0; + MemPage *pPage = pCur->pPage; /* Btree page of current entry */ + BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ +#endif + + assert( pPage ); + assert( eOp==0 || eOp==1 ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->ixnCell ); + assert( cursorHoldsMutex(pCur) ); + + getCellInfo(pCur); + aPayload = pCur->info.pPayload; + assert( offset+amt <= pCur->info.nPayload ); + + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ + return SQLITE_CORRUPT_PAGE(pPage); + } + + /* Check if data must be read/written to/from the btree page itself. */ + if( offsetinfo.nLocal ){ + int a = amt; + if( a+offset>pCur->info.nLocal ){ + a = pCur->info.nLocal - offset; + } + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); + offset = 0; + pBuf += a; + amt -= a; + }else{ + offset -= pCur->info.nLocal; + } + + + if( rc==SQLITE_OK && amt>0 ){ + const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ + Pgno nextPage; + + nextPage = get4byte(&aPayload[pCur->info.nLocal]); + + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. + ** + ** The aOverflow[] array is sized at one entry for each overflow page + ** in the overflow chain. The page number of the first overflow page is + ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array + ** means "not yet known" (the cache is lazily populated). + */ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; + if( pCur->aOverflow==0 + || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + ){ + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) + ); + if( aNew==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + pCur->aOverflow = aNew; + } + } + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + }else{ + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); + } + } + + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ + /* If required, populate the overflow page-list cache. */ + if( nextPage > pBt->nPage ) return SQLITE_CORRUPT_BKPT; + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; + + if( offset>=ovflSize ){ + /* The only reason to read this page is to obtain the page + ** number for the next page in the overflow chain. The page + ** data is not required. So first try to lookup the overflow + ** page-list cache, if any, then fall back to the getOverflowPage() + ** function. + */ + assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); + if( pCur->aOverflow[iIdx+1] ){ + nextPage = pCur->aOverflow[iIdx+1]; + }else{ + rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + } + offset -= ovflSize; + }else{ + /* Need to read this page properly. It contains some of the + ** range of data that is being read (eOp==0) or written (eOp!=0). + */ + int a = amt; + if( a + offset > ovflSize ){ + a = ovflSize - offset; + } + +#ifdef SQLITE_DIRECT_OVERFLOW_READ + /* If all the following are true: + ** + ** 1) this is a read operation, and + ** 2) data is required from the start of this overflow page, and + ** 3) there are no dirty pages in the page-cache + ** 4) the database is file-backed, and + ** 5) the page is not in the WAL file + ** 6) at least 4 bytes have already been read into the output buffer + ** + ** then data can be read directly from the database file into the + ** output buffer, bypassing the page-cache altogether. This speeds + ** up loading large records that span many overflow pages. + */ + if( eOp==0 /* (1) */ + && offset==0 /* (2) */ + && sqlite3PagerDirectReadOk(pBt->pPager, nextPage) /* (3,4,5) */ + && &pBuf[-4]>=pBufStart /* (6) */ + ){ + sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); + u8 aSave[4]; + u8 *aWrite = &pBuf[-4]; + assert( aWrite>=pBufStart ); /* due to (6) */ + memcpy(aSave, aWrite, 4); + rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); + if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; + nextPage = get4byte(aWrite); + memcpy(aWrite, aSave, 4); + }else +#endif + + { + DbPage *pDbPage; + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_GET_READONLY : 0) + ); + if( rc==SQLITE_OK ){ + aPayload = sqlite3PagerGetData(pDbPage); + nextPage = get4byte(aPayload); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + sqlite3PagerUnref(pDbPage); + offset = 0; + } + } + amt -= a; + if( amt==0 ) return rc; + pBuf += a; + } + if( rc ) break; + iIdx++; + } + } + + if( rc==SQLITE_OK && amt>0 ){ + /* Overflow chain ends prematurely */ + return SQLITE_CORRUPT_PAGE(pPage); + } + return rc; +} + +/* +** Read part of the payload for the row at which that cursor pCur is currently +** pointing. "amt" bytes will be transferred into pBuf[]. The transfer +** begins at "offset". +** +** pCur can be pointing to either a table or an index b-tree. +** If pointing to a table btree, then the content section is read. If +** pCur is pointing to an index b-tree then the key section is read. +** +** For sqlite3BtreePayload(), the caller must ensure that pCur is pointing +** to a valid row in the table. For sqlite3BtreePayloadChecked(), the +** cursor might be invalid or might need to be restored before being read. +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. +*/ +int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->pPage ); + assert( pCur->ixpPage->nCell ); + return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); +} + +/* +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. +*/ +#ifndef SQLITE_OMIT_INCRBLOB +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ + int rc; + if ( pCur->eState==CURSOR_INVALID ){ + return SQLITE_ABORT; + } + assert( cursorOwnsBtShared(pCur) ); + rc = btreeRestoreCursorPosition(pCur); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); +} +int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); + } +} +#endif /* SQLITE_OMIT_INCRBLOB */ + +/* +** Return a pointer to payload information from the entry that the +** pCur cursor is pointing to. The pointer is to the beginning of +** the key if index btrees (pPage->intKey==0) and is the data for +** table btrees (pPage->intKey==1). The number of bytes of available +** key/data is written into *pAmt. If *pAmt==0, then the value +** returned will not be a valid pointer. +** +** This routine is an optimization. It is common for the entire key +** and data to fit on the local page and for there to be no overflow +** pages. When that is so, this routine can be used to access the +** key and data without making a copy. If the key and/or data spills +** onto overflow pages, then accessPayload() must be used to reassemble +** the key/data and copy it into a preallocated buffer. +** +** The pointer returned by this routine looks directly into the cached +** page of the database. The data might change or move the next time +** any btree routine is called. +*/ +static const void *fetchPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 *pAmt /* Write the number of available bytes here */ +){ + int amt; + assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); + assert( pCur->eState==CURSOR_VALID ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->ixpPage->nCell ); + assert( pCur->info.nSize>0 ); + assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); + amt = pCur->info.nLocal; + if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){ + /* There is too little space on the page for the expected amount + ** of local content. Database must be corrupt. */ + assert( CORRUPT_DB ); + amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload)); + } + *pAmt = (u32)amt; + return (void*)pCur->info.pPayload; +} + + +/* +** For the entry that cursor pCur is point to, return as +** many bytes of the key or data as are available on the local +** b-tree page. Write the number of available bytes into *pAmt. +** +** The pointer returned is ephemeral. The key/data may move +** or be destroyed on the next call to any Btree routine, +** including calls from other threads against the same cache. +** Hence, a mutex on the BtShared should be held prior to calling +** this routine. +** +** These routines is used to get quick access to key and data +** in the common case where no overflow pages are used. +*/ +const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ + return fetchPayload(pCur, pAmt); +} + + +/* +** Move the cursor down to a new child page. The newPgno argument is the +** page number of the child page to move to. +** +** This function returns SQLITE_CORRUPT if the page-header flags field of +** the new child page does not match the flags field of the parent (i.e. +** if an intkey page appears to be the parent of a non-intkey page, or +** vice-versa). +*/ +static int moveToChild(BtCursor *pCur, u32 newPgno){ + BtShared *pBt = pCur->pBt; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPageiPage>=0 ); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + pCur->aiIdx[pCur->iPage] = pCur->ix; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); +} + +#ifdef SQLITE_DEBUG +/* +** Page pParent is an internal (non-leaf) tree page. This function +** asserts that page number iChild is the left-child if the iIdx'th +** cell in page pParent. Or, if iIdx is equal to the total number of +** cells in pParent, that page number iChild is the right-child of +** the page. +*/ +static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + if( CORRUPT_DB ) return; /* The conditions tested below might not be true + ** in a corrupt database */ + assert( iIdx<=pParent->nCell ); + if( iIdx==pParent->nCell ){ + assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); + }else{ + assert( get4byte(findCell(pParent, iIdx))==iChild ); + } +} +#else +# define assertParentIndex(x,y,z) +#endif + +/* +** Move the cursor up to the parent page. +** +** pCur->idx is set to the cell index that contains the pointer +** to the page we are coming from. If we are coming from the +** right-most child page then pCur->idx is set to one more than +** the largest cell index. +*/ +static void moveToParent(BtCursor *pCur){ + MemPage *pLeaf; + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>0 ); + assert( pCur->pPage ); + assertParentIndex( + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->pPage->pgno + ); + testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + pCur->ix = pCur->aiIdx[pCur->iPage-1]; + pLeaf = pCur->pPage; + pCur->pPage = pCur->apPage[--pCur->iPage]; + releasePageNotNull(pLeaf); +} + +/* +** Move the cursor to point to the root page of its b-tree structure. +** +** If the table has a virtual root page, then the cursor is moved to point +** to the virtual root page instead of the actual root page. A table has a +** virtual root page when the actual root page contains no cells and a +** single child page. This can only happen with the table rooted at page 1. +** +** If the b-tree structure is empty, the cursor state is set to +** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise, +** the cursor is set to point to the first cell located on the root +** (or virtual root) page and the cursor state is set to CURSOR_VALID. +** +** If this function returns successfully, it may be assumed that the +** page-header flags indicate that the [virtual] root-page is the expected +** kind of b-tree page (i.e. if when opening the cursor the caller did not +** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, +** indicating a table b-tree, or if the caller did specify a KeyInfo +** structure the flags byte is set to 0x02 or 0x0A, indicating an index +** b-tree). +*/ +static int moveToRoot(BtCursor *pCur){ + MemPage *pRoot; + int rc = SQLITE_OK; + + assert( cursorOwnsBtShared(pCur) ); + assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + assert( pCur->eState < CURSOR_REQUIRESEEK || pCur->iPage<0 ); + assert( pCur->pgnoRoot>0 || pCur->iPage<0 ); + + if( pCur->iPage>=0 ){ + if( pCur->iPage ){ + releasePageNotNull(pCur->pPage); + while( --pCur->iPage ){ + releasePageNotNull(pCur->apPage[pCur->iPage]); + } + pCur->pPage = pCur->apPage[0]; + goto skip_init; + } + }else if( pCur->pgnoRoot==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_EMPTY; + }else{ + assert( pCur->iPage==(-1) ); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + sqlite3BtreeClearCursor(pCur); + } + rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, + 0, pCur->curPagerFlags); + if( rc!=SQLITE_OK ){ + pCur->eState = CURSOR_INVALID; + return rc; + } + pCur->iPage = 0; + pCur->curIntKey = pCur->pPage->intKey; + } + pRoot = pCur->pPage; + assert( pRoot->pgno==pCur->pgnoRoot ); + + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. + ** + ** Earlier versions of SQLite assumed that this test could not fail + ** if the root page was already loaded when this function was called (i.e. + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ + return SQLITE_CORRUPT_PAGE(pCur->pPage); + } + +skip_init: + pCur->ix = 0; + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); + + pRoot = pCur->pPage; + if( pRoot->nCell>0 ){ + pCur->eState = CURSOR_VALID; + }else if( !pRoot->leaf ){ + Pgno subpage; + if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; + subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); + pCur->eState = CURSOR_VALID; + rc = moveToChild(pCur, subpage); + }else{ + pCur->eState = CURSOR_INVALID; + rc = SQLITE_EMPTY; + } + return rc; +} + +/* +** Move the cursor down to the left-most leaf entry beneath the +** entry to which it is currently pointing. +** +** The left-most leaf is the one with the smallest key - the first +** in ascending order. +*/ +static int moveToLeftmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->pPage)->leaf ){ + assert( pCur->ixnCell ); + pgno = get4byte(findCell(pPage, pCur->ix)); + rc = moveToChild(pCur, pgno); + } + return rc; +} + +/* +** Move the cursor down to the right-most leaf entry beneath the +** page to which it is currently pointing. Notice the difference +** between moveToLeftmost() and moveToRightmost(). moveToLeftmost() +** finds the left-most entry beneath the *entry* whereas moveToRightmost() +** finds the right-most entry beneath the *page*. +** +** The right-most entry is the one with the largest key - the last +** key in ascending order. +*/ +static int moveToRightmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage = 0; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( !(pPage = pCur->pPage)->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + pCur->ix = pPage->nCell; + rc = moveToChild(pCur, pgno); + if( rc ) return rc; + } + pCur->ix = pPage->nCell-1; + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); + return SQLITE_OK; +} + +/* Move the cursor to the first entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->pPage->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} + +/* Move the cursor to the last entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor already points to the last entry, this is a no-op. */ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ +#ifdef SQLITE_DEBUG + /* This block serves to assert() that the cursor really does point + ** to the last entry in the b-tree. */ + int ii; + for(ii=0; iiiPage; ii++){ + assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); + } + assert( pCur->ix==pCur->pPage->nCell-1 ); + assert( pCur->pPage->leaf ); +#endif + *pRes = 0; + return SQLITE_OK; + } + + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} + +/* Move the cursor so that it points to an entry near the key +** specified by pIdxKey or intKey. Return a success code. +** +** For INTKEY tables, the intKey parameter is used. pIdxKey +** must be NULL. For index tables, pIdxKey is used and intKey +** is ignored. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than intKey/pIdxKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches intKey/pIdxKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than intKey/pIdxKey. +** +** For index tables, the pIdxKey->eqSeen field is set to 1 if there +** exists an entry in the table that exactly matches pIdxKey. +*/ +int sqlite3BtreeMovetoUnpacked( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + i64 intKey, /* The table key */ + int biasRight, /* If true, bias the search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; + RecordCompare xRecordCompare; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) ); + + /* If the cursor is already positioned at the point we are trying + ** to move to, then just return without doing any work */ + if( pIdxKey==0 + && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 + ){ + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + *pRes = -1; + return SQLITE_OK; + } + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this case, only a little more slowely */ + if( pCur->info.nKey+1==intKey ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, 0); + if( rc==SQLITE_OK ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } + }else if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + }else{ + return rc; + } + } + } + } + +#ifdef SQLITE_DEBUG + pCur->pBtree->nSeek++; /* Performance measurement during testing */ +#endif + + if( pIdxKey ){ + xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->errCode = 0; + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); + }else{ + xRecordCompare = 0; /* All keys are integers */ + } + + rc = moveToRoot(pCur); + if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } + return rc; + } + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey || pIdxKey ); + for(;;){ + int lwr, upr, idx, c; + Pgno chldPg; + MemPage *pPage = pCur->pPage; + u8 *pCell; /* Pointer to current cell in pPage */ + + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey==(pIdxKey==0) ); + lwr = 0; + upr = pPage->nCell-1; + assert( biasRight==0 || biasRight==1 ); + idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ + pCur->ix = (u16)idx; + if( xRecordCompare==0 ){ + for(;;){ + i64 nCellKey; + pCell = findCellPastPtr(pPage, idx); + if( pPage->intKeyLeaf ){ + while( 0x80 <= *(pCell++) ){ + if( pCell>=pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + } + getVarint(pCell, (u64*)&nCellKey); + if( nCellKeyupr ){ c = -1; break; } + }else if( nCellKey>intKey ){ + upr = idx-1; + if( lwr>upr ){ c = +1; break; } + }else{ + assert( nCellKey==intKey ); + pCur->ix = (u16)idx; + if( !pPage->leaf ){ + lwr = idx; + goto moveto_next_layer; + }else{ + pCur->curFlags |= BTCF_ValidNKey; + pCur->info.nKey = nCellKey; + pCur->info.nSize = 0; + *pRes = 0; + return SQLITE_OK; + } + } + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2; */ + } + }else{ + for(;;){ + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); + + /* The maximum supported page-size is 65536 bytes. This means that + ** the maximum number of record bytes stored on an index B-Tree + ** page is less than 16384 bytes and may be stored as a 2-byte + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first + ** 2 bytes of the cell. + */ + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* The record flows over onto one or more overflow pages. In + ** this case the whole cell needs to be parsed, a buffer allocated + ** and accessPayload() used to retrieve the record into the + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ + pPage->xParseCell(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ + rc = SQLITE_CORRUPT_PAGE(pPage); + goto moveto_finish; + } + pCellKey = sqlite3Malloc( nCell+nOverrun ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto moveto_finish; + } + pCur->ix = (u16)idx; + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ + pCur->curFlags &= ~BTCF_ValidOvfl; + if( rc ){ + sqlite3_free(pCellKey); + goto moveto_finish; + } + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + sqlite3_free(pCellKey); + } + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) + ); + if( c<0 ){ + lwr = idx+1; + }else if( c>0 ){ + upr = idx-1; + }else{ + assert( c==0 ); + *pRes = 0; + rc = SQLITE_OK; + pCur->ix = (u16)idx; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT_BKPT; + goto moveto_finish; + } + if( lwr>upr ) break; + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ + } + } + assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); + assert( pPage->isInit ); + if( pPage->leaf ){ + assert( pCur->ixpPage->nCell ); + pCur->ix = (u16)idx; + *pRes = c; + rc = SQLITE_OK; + goto moveto_finish; + } +moveto_next_layer: + if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); + }else{ + chldPg = get4byte(findCell(pPage, lwr)); + } + pCur->ix = (u16)lwr; + rc = moveToChild(pCur, chldPg); + if( rc ) break; + } +moveto_finish: + pCur->info.nSize = 0; + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + return rc; +} + + +/* +** Return TRUE if the cursor is not pointing at an entry of the table. +** +** TRUE will be returned after a call to sqlite3BtreeNext() moves +** past the last entry in the table or sqlite3BtreePrev() moves past +** the first entry. TRUE is also returned if the table is empty. +*/ +int sqlite3BtreeEof(BtCursor *pCur){ + /* TODO: What if the cursor is in CURSOR_REQUIRESEEK but all table entries + ** have been deleted? This API will need to change to return an error code + ** as well as the boolean result value. + */ + return (CURSOR_VALID!=pCur->eState); +} + +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->pPage->leaf==0) ) return -1; + + n = pCur->pPage->nCell; + for(i=0; iiPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + +/* +** Advance the cursor to the next entry in the database. +** Return value: +** +** SQLITE_OK success +** SQLITE_DONE cursor is already pointing at the last element +** otherwise some kind of error occurred +** +** The main entry point is sqlite3BtreeNext(). That routine is optimized +** for the common case of merely incrementing the cell counter BtCursor.aiIdx +** to the next cell on the current page. The (slower) btreeNext() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** +** If bit 0x01 of the F argument in sqlite3BtreeNext(C,F) is 1, then the +** cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument +** is a hint to the implement. SQLite btree implementation does not use +** this hint, but COMDB2 does. +*/ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ + int rc; + int idx; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + if( pCur->eState!=CURSOR_VALID ){ + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + if( CURSOR_INVALID==pCur->eState ){ + return SQLITE_DONE; + } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + if( pCur->skipNext>0 ) return SQLITE_OK; + } + } + + pPage = pCur->pPage; + idx = ++pCur->ix; + if( !pPage->isInit || sqlite3FaultSim(412) ){ + /* The only known way for this to happen is for there to be a + ** recursive SQL function that does a DELETE operation as part of a + ** SELECT which deletes content out from under an active cursor + ** in a corrupt database file where the table being DELETE-ed from + ** has pages in common with the table being queried. See TH3 + ** module cov1/btree78.test testcase 220 (2018-06-08) for an + ** example. */ + return SQLITE_CORRUPT_BKPT; + } + + /* If the database file is corrupt, it is possible for the value of idx + ** to be invalid here. This can only occur if a second cursor modifies + ** the page while cursor pCur is holding a reference to it. Which can + ** only happen if the database is corrupt in such a way as to link the + ** page into more than one b-tree structure. + ** + ** Update 2019-12-23: appears to long longer be possible after the + ** addition of anotherValidCursor() condition on balance_deeper(). */ + harmless( idx>pPage->nCell ); + + if( idx>=pPage->nCell ){ + if( !pPage->leaf ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + if( rc ) return rc; + return moveToLeftmost(pCur); + } + do{ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_DONE; + } + moveToParent(pCur); + pPage = pCur->pPage; + }while( pCur->ix>=pPage->nCell ); + if( pPage->intKey ){ + return sqlite3BtreeNext(pCur, 0); + }else{ + return SQLITE_OK; + } + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} +int sqlite3BtreeNext(BtCursor *pCur, int flags){ + MemPage *pPage; + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ + assert( cursorOwnsBtShared(pCur) ); + assert( flags==0 || flags==1 ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); + pPage = pCur->pPage; + if( (++pCur->ix)>=pPage->nCell ){ + pCur->ix--; + return btreeNext(pCur); + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} + +/* +** Step the cursor to the back to the previous entry in the database. +** Return values: +** +** SQLITE_OK success +** SQLITE_DONE the cursor is already on the first element of the table +** otherwise some kind of error occurred +** +** The main entry point is sqlite3BtreePrevious(). That routine is optimized +** for the common case of merely decrementing the cell counter BtCursor.aiIdx +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. +** +** If bit 0x01 of the F argument to sqlite3BtreePrevious(C,F) is 1, then +** the cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument is a +** hint to the implement. The native SQLite btree implementation does not +** use this hint, but COMDB2 does. +*/ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ + int rc; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); + assert( pCur->info.nSize==0 ); + if( pCur->eState!=CURSOR_VALID ){ + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + if( CURSOR_INVALID==pCur->eState ){ + return SQLITE_DONE; + } + if( CURSOR_SKIPNEXT==pCur->eState ){ + pCur->eState = CURSOR_VALID; + if( pCur->skipNext<0 ) return SQLITE_OK; + } + } + + pPage = pCur->pPage; + assert( pPage->isInit ); + if( !pPage->leaf ){ + int idx = pCur->ix; + rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); + if( rc ) return rc; + rc = moveToRightmost(pCur); + }else{ + while( pCur->ix==0 ){ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_DONE; + } + moveToParent(pCur); + } + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 ); + + pCur->ix--; + pPage = pCur->pPage; + if( pPage->intKey && !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, 0); + }else{ + rc = SQLITE_OK; + } + } + return rc; +} +int sqlite3BtreePrevious(BtCursor *pCur, int flags){ + assert( cursorOwnsBtShared(pCur) ); + assert( flags==0 || flags==1 ); + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); + pCur->info.nSize = 0; + if( pCur->eState!=CURSOR_VALID + || pCur->ix==0 + || pCur->pPage->leaf==0 + ){ + return btreePrevious(pCur); + } + pCur->ix--; + return SQLITE_OK; +} + +/* +** Allocate a new page from the database file. +** +** The new page is marked as dirty. (In other words, sqlite3PagerWrite() +** has already been called on the new page.) The new page has also +** been referenced and the calling routine is responsible for calling +** sqlite3PagerUnref() on the new page when it is done. +** +** SQLITE_OK is returned on success. Any other return value indicates +** an error. *ppPage is set to NULL in the event of an error. +** +** If the "nearby" parameter is not 0, then an effort is made to +** locate a page close to the page number "nearby". This can be used in an +** attempt to keep related pages close to each other in the database file, +** which in turn can make database access faster. +** +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. +*/ +static int allocateBtreePage( + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ +){ + MemPage *pPage1; + int rc; + u32 n; /* Number of pages on the freelist */ + u32 k; /* Number of leaves on the trunk of the freelist */ + MemPage *pTrunk = 0; + MemPage *pPrevTrunk = 0; + Pgno mxPage; /* Total size of the database file */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); + pPage1 = pBt->pPage1; + mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ + n = get4byte(&pPage1->aData[36]); + testcase( n==mxPage-1 ); + if( n>=mxPage ){ + return SQLITE_CORRUPT_BKPT; + } + if( n>0 ){ + /* There are pages on the freelist. Reuse one of those pages. */ + Pgno iTrunk; + u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + u32 nSearch = 0; /* Count of the number of search attempts */ + + /* If eMode==BTALLOC_EXACT and a query of the pointer-map + ** shows that the page 'nearby' is somewhere on the free-list, then + ** the entire-list will be searched for that page. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + } + }else if( eMode==BTALLOC_LE ){ + searchList = 1; + } +#endif + + /* Decrement the free-list count by 1. Set iTrunk to the index of the + ** first free-list trunk page. iPrevTrunk is initially 1. + */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + put4byte(&pPage1->aData[36], n-1); + + /* The code within this loop is run only once if the 'searchList' variable + ** is not true. Otherwise, it runs once for each trunk-page on the + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) + */ + do { + pPrevTrunk = pTrunk; + if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ + iTrunk = get4byte(&pPrevTrunk->aData[0]); + }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ + iTrunk = get4byte(&pPage1->aData[32]); + } + testcase( iTrunk==mxPage ); + if( iTrunk>mxPage || nSearch++ > n ){ + rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1); + }else{ + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); + } + if( rc ){ + pTrunk = 0; + goto end_allocate_page; + } + assert( pTrunk!=0 ); + assert( pTrunk->aData!=0 ); + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); + if( k==0 && !searchList ){ + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly + ** allocated page */ + assert( pPrevTrunk==0 ); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + *pPgno = iTrunk; + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + *ppPage = pTrunk; + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + }else if( k>(u32)(pBt->usableSize/4 - 2) ){ + /* Value of k is out of range. Database corruption */ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; +#ifndef SQLITE_OMIT_AUTOVACUUM + }else if( searchList + && (nearby==iTrunk || (iTrunkpDbPage); + if( rc ){ + goto end_allocate_page; + } + if( k==0 ){ + if( !pPrevTrunk ){ + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); + } + }else{ + /* The trunk page is required by the caller but it contains + ** pointers to free-list leaves. The first leaf becomes a trunk + ** page in this case. + */ + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + if( iNewTrunk>mxPage ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; + } + testcase( iNewTrunk==mxPage ); + rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + rc = sqlite3PagerWrite(pNewTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pNewTrunk); + goto end_allocate_page; + } + memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); + put4byte(&pNewTrunk->aData[4], k-1); + memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); + releasePage(pNewTrunk); + if( !pPrevTrunk ){ + assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); + put4byte(&pPage1->aData[32], iNewTrunk); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + put4byte(&pPrevTrunk->aData[0], iNewTrunk); + } + } + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); +#endif + }else if( k>0 ){ + /* Extract a leaf from the trunk */ + u32 closest; + Pgno iPage; + unsigned char *aData = pTrunk->aData; + if( nearby>0 ){ + u32 i; + closest = 0; + if( eMode==BTALLOC_LE ){ + for(i=0; imxPage ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; + } + testcase( iPage==mxPage ); + if( !searchList + || (iPage==nearby || (iPagepgno, n-1)); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ) goto end_allocate_page; + if( closestpDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + *ppPage = 0; + } + } + searchList = 0; + } + } + releasePage(pPrevTrunk); + pPrevTrunk = 0; + }while( searchList ); + }else{ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; + + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; + +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){ + /* If *pPgno refers to a pointer-map page, allocate two new pages + ** at the end of the file instead of one. The first allocated page + ** becomes a new pointer-map page, the second is used by the caller. + */ + MemPage *pPg = 0; + TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg->pDbPage); + releasePage(pPg); + } + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; } + } +#endif + put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); + *pPgno = pBt->nPage; + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); + if( rc ) return rc; + rc = sqlite3PagerWrite((*ppPage)->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + *ppPage = 0; + } + TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + } + + assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); + +end_allocate_page: + releasePage(pTrunk); + releasePage(pPrevTrunk); + assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); + assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); + return rc; +} + +/* +** This function is used to add page iPage to the database file free-list. +** It is assumed that the page is not already a part of the free-list. +** +** The value passed as the second argument to this function is optional. +** If the caller happens to have a pointer to the MemPage object +** corresponding to page iPage handy, it may pass it as the second value. +** Otherwise, it may pass NULL. +** +** If a pointer to a MemPage object is passed as the second argument, +** its reference count is not altered by this function. +*/ +static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ + MemPage *pTrunk = 0; /* Free-list trunk page */ + Pgno iTrunk = 0; /* Page number of free-list trunk page */ + MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ + MemPage *pPage; /* Page being freed. May be NULL. */ + int rc; /* Return Code */ + u32 nFree; /* Initial number of pages on free-list */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( CORRUPT_DB || iPage>1 ); + assert( !pMemPage || pMemPage->pgno==iPage ); + + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } + if( pMemPage ){ + pPage = pMemPage; + sqlite3PagerRef(pPage->pDbPage); + }else{ + pPage = btreePageLookup(pBt, iPage); + } + + /* Increment the free page count on pPage1 */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) goto freepage_out; + nFree = get4byte(&pPage1->aData[36]); + put4byte(&pPage1->aData[36], nFree+1); + + if( pBt->btsFlags & BTS_SECURE_DELETE ){ + /* If the secure_delete option is enabled, then + ** always fully overwrite deleted information with zeros. + */ + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) + ){ + goto freepage_out; + } + memset(pPage->aData, 0, pPage->pBt->pageSize); + } + + /* If the database supports auto-vacuum, write an entry in the pointer-map + ** to indicate that the page is free. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); + if( rc ) goto freepage_out; + } + + /* Now manipulate the actual database free-list structure. There are two + ** possibilities. If the free-list is currently empty, or if the first + ** trunk page in the free-list is full, then this page will become a + ** new free-list trunk page. Otherwise, it will become a leaf of the + ** first trunk page in the current free-list. This block tests if it + ** is possible to add the page as a new free-list leaf. + */ + if( nFree!=0 ){ + u32 nLeaf; /* Initial number of leaf cells on trunk page */ + + iTrunk = get4byte(&pPage1->aData[32]); + if( iTrunk>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + + nLeaf = get4byte(&pTrunk->aData[4]); + assert( pBt->usableSize>32 ); + if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } + if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ + /* In this case there is room on the trunk page to insert the page + ** being freed as a new leaf. + ** + ** Note that the trunk page is not really full until it contains + ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have + ** coded. But due to a coding error in versions of SQLite prior to + ** 3.6.0, databases with freelist trunk pages holding more than + ** usableSize/4 - 8 entries will be reported as corrupt. In order + ** to maintain backwards compatibility with older versions of SQLite, + ** we will continue to restrict the number of entries to usableSize/4 - 8 + ** for now. At some point in the future (once everyone has upgraded + ** to 3.6.0 or later) we should consider fixing the conditional above + ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. + */ + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pTrunk->aData[4], nLeaf+1); + put4byte(&pTrunk->aData[8+nLeaf*4], iPage); + if( pPage && (pBt->btsFlags & BTS_SECURE_DELETE)==0 ){ + sqlite3PagerDontWrite(pPage->pDbPage); + } + rc = btreeSetHasContent(pBt, iPage); + } + TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + goto freepage_out; + } + } + + /* If control flows to this point, then it was not possible to add the + ** the page being freed as a leaf page of the first trunk in the free-list. + ** Possibly because the free-list is empty, or possibly because the + ** first trunk in the free-list is full. Either way, the page being freed + ** will become the new first trunk page in the free-list. + */ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + goto freepage_out; + } + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + put4byte(pPage->aData, iTrunk); + put4byte(&pPage->aData[4], 0); + put4byte(&pPage1->aData[32], iPage); + TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + +freepage_out: + if( pPage ){ + pPage->isInit = 0; + } + releasePage(pPage); + releasePage(pTrunk); + return rc; +} +static void freePage(MemPage *pPage, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + *pRC = freePage2(pPage->pBt, pPage, pPage->pgno); + } +} + +/* +** Free any overflow pages associated with the given Cell. Store +** size information about the cell in pInfo. +*/ +static int clearCell( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + CellInfo *pInfo /* Size information about the cell */ +){ + BtShared *pBt; + Pgno ovflPgno; + int rc; + int nOvfl; + u32 ovflPageSize; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->xParseCell(pPage, pCell, pInfo); + if( pInfo->nLocal==pInfo->nPayload ){ + return SQLITE_OK; /* No overflow pages. Return without doing anything */ + } + testcase( pCell + pInfo->nSize == pPage->aDataEnd ); + testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd ); + if( pCell + pInfo->nSize > pPage->aDataEnd ){ + /* Cell extends past end of page */ + return SQLITE_CORRUPT_PAGE(pPage); + } + ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; + assert( pBt->usableSize > 4 ); + ovflPageSize = pBt->usableSize - 4; + nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; + assert( nOvfl>0 || + (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)btreePagecount(pBt) ){ + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the + ** file the database must be corrupt. */ + return SQLITE_CORRUPT_BKPT; + } + if( nOvfl ){ + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); + if( rc ) return rc; + } + + if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) + && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 + ){ + /* There is no reason any cursor should have an outstanding reference + ** to an overflow page belonging to a cell that is being deleted/updated. + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as + ** freePage2() may zero the page contents if secure-delete mode is + ** enabled. If this 'overflow' page happens to be a page that the + ** caller is iterating through or using in some other way, this + ** can be problematic. + */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = freePage2(pBt, pOvfl, ovflPgno); + } + + if( pOvfl ){ + sqlite3PagerUnref(pOvfl->pDbPage); + } + if( rc ) return rc; + ovflPgno = iNext; + } + return SQLITE_OK; +} + +/* +** Create the byte sequence used to represent a cell on page pPage +** and write that byte sequence into pCell[]. Overflow pages are +** allocated and filled in as necessary. The calling procedure +** is responsible for making sure sufficient space has been allocated +** for pCell[]. +** +** Note that pCell does not necessary need to point to the pPage->aData +** area. pCell might point to some temporary storage. The cell will +** be constructed in this temporary area then copied into pPage->aData +** later. +*/ +static int fillInCell( + MemPage *pPage, /* The page that contains the cell */ + unsigned char *pCell, /* Complete text of the cell */ + const BtreePayload *pX, /* Payload with which to construct the cell */ + int *pnSize /* Write cell size here */ +){ + int nPayload; + const u8 *pSrc; + int nSrc, n, rc, mn; + int spaceLeft; + MemPage *pToRelease; + unsigned char *pPrior; + unsigned char *pPayload; + BtShared *pBt; + Pgno pgnoOvfl; + int nHeader; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + + /* pPage is not necessarily writeable since pCell might be auxiliary + ** buffer space that is separate from the pPage buffer area */ + assert( pCellaData || pCell>=&pPage->aData[pPage->pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + /* Fill in the header. */ + nHeader = pPage->childPtrSize; + if( pPage->intKey ){ + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + }else{ + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); + } + + /* Fill in the payload */ + pPayload = &pCell[nHeader]; + if( nPayload<=pPage->maxLocal ){ + /* This is the common case where everything fits on the btree page + ** and no overflow pages are required. */ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + assert( nSrc<=nPayload ); + testcase( nSrcminLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + pToRelease = 0; + pgnoOvfl = 0; + pBt = pPage->pBt; + + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#ifdef SQLITE_DEBUG + { + CellInfo info; + pPage->xParseCell(pPage, pCell, &info); + assert( nHeader==(int)(info.pPayload - pCell) ); + assert( info.nKey==pX->nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ + while( 1 ){ + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>=n ){ + memcpy(pPayload, pSrc, n); + }else if( nSrc>0 ){ + n = nSrc; + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + if( nPayload<=0 ) break; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; + if( spaceLeft==0 ){ + MemPage *pOvfl = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ + if( pBt->autoVacuum ){ + do{ + pgnoOvfl++; + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + ); + } +#endif + rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent + ** overflow page is being allocated, add an entry to the pointer-map + ** for that page now. + ** + ** If this is the first overflow page, then write a partial entry + ** to the pointer-map. If we write nothing to this pointer-map slot, + ** then the optimistic overflow chain processing in clearCell() + ** may misinterpret the uninitialized values and delete the + ** wrong pages from the database. + */ + if( pBt->autoVacuum && rc==SQLITE_OK ){ + u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); + ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); + if( rc ){ + releasePage(pOvfl); + } + } +#endif + if( rc ){ + releasePage(pToRelease); + return rc; + } + + /* If pToRelease is not zero than pPrior points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPrior is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + put4byte(pPrior, pgnoOvfl); + releasePage(pToRelease); + pToRelease = pOvfl; + pPrior = pOvfl->aData; + put4byte(pPrior, 0); + pPayload = &pOvfl->aData[4]; + spaceLeft = pBt->usableSize - 4; + } + } + releasePage(pToRelease); + return SQLITE_OK; +} + +/* +** Remove the i-th cell from pPage. This routine effects pPage only. +** The cell content is not freed or deallocated. It is assumed that +** the cell content has been copied someplace else. This routine just +** removes the reference to the cell from pPage. +** +** "sz" must be the number of bytes in the cell. +*/ +static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ + u32 pc; /* Offset to cell content of cell being deleted */ + u8 *data; /* pPage->aData */ + u8 *ptr; /* Used to move bytes around within data[] */ + int rc; /* The return code */ + int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ + + if( *pRC ) return; + assert( idx>=0 && idxnCell ); + assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); + data = pPage->aData; + ptr = &pPage->aCellIdx[2*idx]; + pc = get2byte(ptr); + hdr = pPage->hdrOffset; + testcase( pc==get2byte(&data[hdr+5]) ); + testcase( pc+sz==pPage->pBt->usableSize ); + if( pc+sz > pPage->pBt->usableSize ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + rc = freeSpace(pPage, pc, sz); + if( rc ){ + *pRC = rc; + return; + } + pPage->nCell--; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } +} + +/* +** Insert a new cell on pPage at cell index "i". pCell points to the +** content of the cell. +** +** If the cell content will fit on the page, then put it there. If it +** will not fit, then make a copy of the cell content into pTemp if +** pTemp is not null. Regardless of pTemp, allocate a new entry +** in pPage->apOvfl[] and make it point to the cell content (either +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that +** pPage->nOverflow is incremented. +** +** *pRC must be SQLITE_OK when this routine is called. +*/ +static void insertCell( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz, /* Bytes of content in pCell */ + u8 *pTemp, /* Temp storage space for pCell, if needed */ + Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ + int *pRC /* Read and write return code from here */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( *pRC==SQLITE_OK ); + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + if( pTemp ){ + memcpy(pTemp, pCell, sz); + pCell = pTemp; + } + if( iChild ){ + put4byte(pCell, iChild); + } + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ *pRC = rc; return; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + if( iChild ){ + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + }else{ + memcpy(&data[idx], pCell, sz); + } + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + } +#endif + } +} + +/* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +** +** (Later:) The description above makes it seem as if these values are +** tunable - as if you could change them and recompile and it would all work. +** But that is unlikely. NB has been 3 since the inception of SQLite and +** we have never tested any other value. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB 3 /* (NN*2+1): Total pages involved in the balance */ + +/* +** A CellArray object contains a cache of pointers and sizes for a +** consecutive sequence of cells that might be held on multiple pages. +** +** The cells in this array are the divider cell or cells from the pParent +** page plus up to three child pages. There are a total of nCell cells. +** +** pRef is a pointer to one of the pages that contributes cells. This is +** used to access information such as MemPage.intKey and MemPage.pBt->pageSize +** which should be common to all pages that contribute cells to this array. +** +** apCell[] and szCell[] hold, respectively, pointers to the start of each +** cell and the size of each cell. Some of the apCell[] pointers might refer +** to overflow cells. In other words, some apCel[] pointers might not point +** to content area of the pages. +** +** A szCell[] of zero means the size of that cell has not yet been computed. +** +** The cells come from as many as four different pages: +** +** ----------- +** | Parent | +** ----------- +** / | \ +** / | \ +** --------- --------- --------- +** |Child-1| |Child-2| |Child-3| +** --------- --------- --------- +** +** The order of cells is in the array is for an index btree is: +** +** 1. All cells from Child-1 in order +** 2. The first divider cell from Parent +** 3. All cells from Child-2 in order +** 4. The second divider cell from Parent +** 5. All cells from Child-3 in order +** +** For a table-btree (with rowids) the items 2 and 4 are empty because +** content exists only in leaves and there are no divider cells. +** +** For an index btree, the apEnd[] array holds pointer to the end of page +** for Child-1, the Parent, Child-2, the Parent (again), and Child-3, +** respectively. The ixNx[] array holds the number of cells contained in +** each of these 5 stages, and all stages to the left. Hence: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 plus 1 for first divider. +** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. +** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[4] = Total number of cells. +** +** For a table-btree, the concept is similar, except only apEnd[0]..apEnd[2] +** are used and they point to the leaf pages only, and the ixNx value are: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous +*/ +typedef struct CellArray CellArray; +struct CellArray { + int nCell; /* Number of cells in apCell[] */ + MemPage *pRef; /* Reference page */ + u8 **apCell; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */ + int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */ +}; + +/* +** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been +** computed. +*/ +static void populateCellCache(CellArray *p, int idx, int N){ + assert( idx>=0 && idx+N<=p->nCell ); + while( N>0 ){ + assert( p->apCell[idx]!=0 ); + if( p->szCell[idx]==0 ){ + p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + }else{ + assert( CORRUPT_DB || + p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + } + idx++; + N--; + } +} + +/* +** Return the size of the Nth element of the cell array +*/ +static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + assert( p->szCell[N]==0 ); + p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); + return p->szCell[N]; +} +static u16 cachedCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + if( p->szCell[N] ) return p->szCell[N]; + return computeCellSize(p, N); +} + +/* +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. +*/ +static int rebuildPage( + CellArray *pCArray, /* Content to be added to page pPg */ + int iFirst, /* First cell in pCArray to use */ + int nCell, /* Final number of cells on page */ + MemPage *pPg /* The page to be reconstructed */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i = iFirst; /* Which cell to copy from pCArray*/ + u32 j; /* Start of cell content area */ + int iEnd = i+nCell; /* Loop terminator */ + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + + assert( i(u32)usableSize) ){ j = 0; } + memcpy(&pTmp[j], &aData[j], usableSize - j); + + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kapEnd[k]; + + pData = pEnd; + while( 1/*exit by break*/ ){ + u8 *pCell = pCArray->apCell[i]; + u16 sz = pCArray->szCell[i]; + assert( sz>0 ); + if( SQLITE_WITHIN(pCell,aData,pEnd) ){ + if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT; + pCell = &pTmp[pCell - aData]; + }else if( (uptr)(pCell+sz)>(uptr)pSrcEnd + && (uptr)(pCell)<(uptr)pSrcEnd + ){ + return SQLITE_CORRUPT_BKPT; + } + + pData -= sz; + put2byte(pCellptr, (pData - aData)); + pCellptr += 2; + if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; + memcpy(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( sz!=pPg->xCellSize(pPg,pCell) ) + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pSrcEnd = pCArray->apEnd[k]; + } + } + + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; + return SQLITE_OK; +} + +/* +** The pCArray objects contains pointers to b-tree cells and the cell sizes. +** This function attempts to add the cells stored in the array to page pPg. +** If it cannot (because the page needs to be defragmented before the cells +** will fit), non-zero is returned. Otherwise, if the cells are added +** successfully, zero is returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ +static int pageInsertArray( + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content-area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int iFirst, /* Index of first cell to add */ + int nCell, /* Number of cells to add to pPg */ + CellArray *pCArray /* Array of cells */ +){ + int i = iFirst; /* Loop counter - cell index to insert */ + u8 *aData = pPg->aData; /* Complete page */ + u8 *pData = *ppData; /* Content area. A subset of aData[] */ + int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */ + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pEnd; /* Maximum extent of cell data */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ + if( iEnd<=iFirst ) return 0; + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kapEnd[k]; + while( 1 /*Exit by break*/ ){ + int sz, rc; + u8 *pSlot; + assert( pCArray->szCell[i]!=0 ); + sz = pCArray->szCell[i]; + if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ + if( (pData - pBegin)apCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd + && (uptr)(pCArray->apCell[i])<(uptr)pEnd + ){ + assert( CORRUPT_DB ); + (void)SQLITE_CORRUPT_BKPT; + return 1; + } + memmove(pSlot, pCArray->apCell[i], sz); + put2byte(pCellptr, (pSlot - aData)); + pCellptr += 2; + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pEnd = pCArray->apEnd[k]; + } + } + *ppData = pData; + return 0; +} + +/* +** The pCArray object contains pointers to b-tree cells and their sizes. +** +** This function adds the space associated with each cell in the array +** that is currently stored within the body of pPg to the pPg free-list. +** The cell-pointers and other fields of the page are not updated. +** +** This function returns the total number of cells added to the free-list. +*/ +static int pageFreeArray( + MemPage *pPg, /* Page to edit */ + int iFirst, /* First cell to delete */ + int nCell, /* Cells to delete */ + CellArray *pCArray /* Array of cells */ +){ + u8 * const aData = pPg->aData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; + int nRet = 0; + int i; + int iEnd = iFirst + nCell; + u8 *pFree = 0; + int szFree = 0; + + for(i=iFirst; iapCell[i]; + if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ + int sz; + /* No need to use cachedCellSize() here. The sizes of all cells that + ** are to be freed have already been computing while deciding which + ** cells need freeing */ + sz = pCArray->szCell[i]; assert( sz>0 ); + if( pFree!=(pCell + sz) ){ + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + pFree = pCell; + szFree = sz; + if( pFree+sz>pEnd ) return 0; + }else{ + pFree = pCell; + szFree += sz; + } + nRet++; + } + } + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + return nRet; +} + +/* +** pCArray contains pointers to and sizes of all cells in the page being +** balanced. The current page, pPg, has pPg->nCell cells starting with +** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** +** The pPg->nFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ +static int editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + CellArray *pCArray /* Array of cells and sizes */ +){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + assert( nCell>=0 ); + if( iOldnCell) ) return SQLITE_CORRUPT_BKPT; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + int nTail = pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + assert( nCell>=nTail ); + nCell -= nTail; + } + + pData = &aData[get2byteNotZero(&aData[hdr+5])]; + if( pData=0 ); + pCellptr = pPg->aCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew, nAdd, pCArray + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } + nCell++; + cachedCellSize(pCArray, iCell+iNew); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iCell+iNew, 1, pCArray + ) ) goto editpage_fail; + } + } + + /* Append cells to the end of the page */ + assert( nCell>=0 ); + pCellptr = &pPg->aCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew+nCell, nNew-nCell, pCArray + ) ) goto editpage_fail; + + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iapCell[i+iNew]; + int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); + if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], + pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); + } +#endif + + return SQLITE_OK; + editpage_fail: + /* Unable to edit this page. Rebuild it from scratch instead. */ + populateCellCache(pCArray, iNew, nNew); + return rebuildPage(pCArray, iNew, nNew, pPg); +} + + +#ifndef SQLITE_OMIT_QUICKBALANCE +/* +** This version of balance() handles the common special case where +** a new entry is being inserted on the extreme right-end of the +** tree, in other words, when the new entry will become the largest +** entry in the tree. +** +** Instead of trying to balance the 3 right-most leaf pages, just add +** a new page to the right-hand side and put the one new entry in +** that page. This leaves the right side of the tree somewhat +** unbalanced. But odds are that we will be inserting new entries +** at the end soon afterwards so the nearly empty page will quickly +** fill up. On average. +** +** pPage is the leaf page which is the right-most page in the tree. +** pParent is its parent. pPage must have a single overflow entry +** which is also the right-most entry on the page. +** +** The pSpace buffer is used to store a temporary copy of the divider +** cell that will be inserted into pParent. Such a cell consists of a 4 +** byte page number followed by a variable length integer. In other +** words, at most 13 bytes. Hence the pSpace buffer must be at +** least 13 bytes in size. +*/ +static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ + BtShared *const pBt = pPage->pBt; /* B-Tree Database */ + MemPage *pNew; /* Newly allocated page */ + int rc; /* Return Code */ + Pgno pgnoNew; /* Page number of pNew */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( pPage->nOverflow==1 ); + + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); + + /* Allocate a new page. This page will become the right-sibling of + ** pPage. Make the parent page writable, so that the new divider cell + ** may be inserted. If both these operations are successful, proceed. + */ + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + + if( rc==SQLITE_OK ){ + + u8 *pOut = &pSpace[4]; + u8 *pCell = pPage->apOvfl[0]; + u16 szCell = pPage->xCellSize(pPage, pCell); + u8 *pStop; + CellArray b; + + assert( sqlite3PagerIswriteable(pNew->pDbPage) ); + assert( CORRUPT_DB || pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); + b.nCell = 1; + b.pRef = pPage; + b.apCell = &pCell; + b.szCell = &szCell; + b.apEnd[0] = pPage->aDataEnd; + b.ixNx[0] = 2; + rc = rebuildPage(&b, 0, 1, pNew); + if( NEVER(rc) ){ + releasePage(pNew); + return rc; + } + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; + + /* If this is an auto-vacuum database, update the pointer map + ** with entries for the new page, and any pointer from the + ** cell on the page to an overflow page. If either of these + ** operations fails, the return code is set, but the contents + ** of the parent page are still manipulated by thh code below. + ** That is Ok, at this point the parent page is guaranteed to + ** be marked as dirty. Returning an error code will cause a + ** rollback, undoing any changes made to the parent page. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); + if( szCell>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); + } + } + + /* Create a divider cell to insert into pParent. The divider cell + ** consists of a 4-byte page number (the page number of pPage) and + ** a variable length key value (which must be the same value as the + ** largest key on pPage). + ** + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the + ** record-length (a variable length integer at most 32-bits in size) + ** and the key value (a variable length integer, may have any value). + ** The first of the while(...) loops below skips over the record-length + ** field. The second while(...) loop copies the key value from the + ** cell on pPage into the pSpace buffer. + */ + pCell = findCell(pPage, pPage->nCell-1); + pStop = &pCell[9]; + while( (*(pCell++)&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + } + + /* Set the right-child pointer of pParent to point to the new page. */ + put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); + + /* Release the reference to the new page. */ + releasePage(pNew); + } + + return rc; +} +#endif /* SQLITE_OMIT_QUICKBALANCE */ + +#if 0 +/* +** This function does not contribute anything to the operation of SQLite. +** it is sometimes activated temporarily while debugging code responsible +** for setting pointer-map entries. +*/ +static int ptrmapCheckPages(MemPage **apPage, int nPage){ + int i, j; + for(i=0; ipBt; + assert( pPage->isInit ); + + for(j=0; jnCell; j++){ + CellInfo info; + u8 *z; + + z = findCell(pPage, j); + pPage->xParseCell(pPage, z, &info); + if( info.nLocalpgno && e==PTRMAP_OVERFLOW1 ); + } + if( !pPage->leaf ){ + Pgno child = get4byte(z); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + if( !pPage->leaf ){ + Pgno child = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + return 1; +} +#endif + +/* +** This function is used to copy the contents of the b-tree node stored +** on page pFrom to page pTo. If page pFrom was not a leaf page, then +** the pointer-map entries for each child page are updated so that the +** parent page stored in the pointer map is page pTo. If pFrom contained +** any cells with overflow page pointers, then the corresponding pointer +** map entries are also updated so that the parent page is page pTo. +** +** If pFrom is currently carrying any overflow cells (entries in the +** MemPage.apOvfl[] array), they are not copied to pTo. +** +** Before returning, page pTo is reinitialized using btreeInitPage(). +** +** The performance of this function is not critical. It is only used by +** the balance_shallower() and balance_deeper() procedures, neither of +** which are called often under normal circumstances. +*/ +static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + BtShared * const pBt = pFrom->pBt; + u8 * const aFrom = pFrom->aData; + u8 * const aTo = pTo->aData; + int const iFromHdr = pFrom->hdrOffset; + int const iToHdr = ((pTo->pgno==1) ? 100 : 0); + int rc; + int iData; + + + assert( pFrom->isInit ); + assert( pFrom->nFree>=iToHdr ); + assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); + + /* Copy the b-tree node content from page pFrom to page pTo. */ + iData = get2byte(&aFrom[iFromHdr+5]); + memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); + memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); + + /* Reinitialize page pTo so that the contents of the MemPage structure + ** match the new data. The initialization of pTo can actually fail under + ** fairly obscure circumstances, even though it is a copy of initialized + ** page pFrom. + */ + pTo->isInit = 0; + rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + + /* If this is an auto-vacuum database, update the pointer-map entries + ** for any b-tree or overflow pages that pTo now contains the pointers to. + */ + if( ISAUTOVACUUM ){ + *pRC = setChildPtrmaps(pTo); + } + } +} + +/* +** This routine redistributes cells on the iParentIdx'th child of pParent +** (hereafter "the page") and up to 2 siblings so that all pages have about the +** same amount of free space. Usually a single sibling on either side of the +** page are used in the balancing, though both siblings might come from one +** side if the page is the first or last child of its parent. If the page +** has fewer than 2 siblings (something which can only happen if the page +** is a root page or a child of a root page) then all available siblings +** participate in the balancing. +** +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. +** +** Note that when this routine is called, some of the cells on the page +** might not actually be stored in MemPage.aData[]. This can happen +** if the page is overfull. This routine ensures that all cells allocated +** to the page and its siblings fit into MemPage.aData[] before returning. +** +** In the course of balancing the page and its siblings, cells may be +** inserted into or removed from the parent page (pParent). Doing so +** may cause the parent page to become overfull or underfull. If this +** happens, it is the responsibility of the caller to invoke the correct +** balancing routine to fix this problem (see the balance() routine). +** +** If this routine fails for any reason, it might leave the database +** in a corrupted state. So if this routine fails, the database should +** be rolled back. +** +** The third argument to this function, aOvflSpace, is a pointer to a +** buffer big enough to hold one page. If while inserting cells into the parent +** page (pParent) the parent page becomes overfull, this buffer is +** used to store the parent's overflow cells. Because this function inserts +** a maximum of four divider cells into the parent page, and the maximum +** size of a cell stored within an internal node is always less than 1/4 +** of the page-size, the aOvflSpace[] buffer is guaranteed to be large +** enough for all overflow cells. +** +** If aOvflSpace is set to a null pointer, this function returns +** SQLITE_NOMEM. +*/ +static int balance_nonroot( + MemPage *pParent, /* Parent page of siblings being balanced */ + int iParentIdx, /* Index of "the page" in pParent */ + u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ + int isRoot, /* True if pParent is a root-page */ + int bBulk /* True if this call is part of a bulk load */ +){ + BtShared *pBt; /* The whole database */ + int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ + int nNew = 0; /* Number of pages in apNew[] */ + int nOld; /* Number of pages in apOld[] */ + int i, j, k; /* Loop counters */ + int nxDiv; /* Next divider slot in pParent->aCell[] */ + int rc = SQLITE_OK; /* The return code */ + u16 leafCorrection; /* 4 if pPage is a leaf. 0 if not */ + int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ + int usableSpace; /* Bytes in pPage beyond the header */ + int pageFlags; /* Value of pPage->aData[0] */ + int iSpace1 = 0; /* First unused byte of aSpace1[] */ + int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ + int szScratch; /* Size of scratch memory requested */ + MemPage *apOld[NB]; /* pPage and up to two siblings */ + MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ + u8 *pRight; /* Location in parent of right-sibling pointer */ + u8 *apDiv[NB-1]; /* Divider cells in pParent */ + int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in b.apCell[] */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ + u8 *aSpace1; /* Space for copies of dividers cells */ + Pgno pgno; /* Temp var to store a page number in */ + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + CellArray b; /* Parsed information on cells being balanced */ + + memset(abDone, 0, sizeof(abDone)); + b.nCell = 0; + b.apCell = 0; + pBt = pParent->pBt; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + + /* At this point pParent may have at most one overflow cell. And if + ** this overflow cell is present, it must be the cell with + ** index iParentIdx. This scenario comes about when this function + ** is called (indirectly) from sqlite3BtreeDelete(). + */ + assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); + assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); + + if( !aOvflSpace ){ + return SQLITE_NOMEM_BKPT; + } + assert( pParent->nFree>=0 ); + + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, + ** if there are fewer than NN siblings on the other side. If pParent + ** has NB or fewer children then all children of pParent are taken. + ** + ** This loop also drops the divider cells from the parent page. This + ** way, the remainder of the function does not have to deal with any + ** overflow cells in the parent page, since if any existed they will + ** have already been removed. + */ + i = pParent->nOverflow + pParent->nCell; + if( i<2 ){ + nxDiv = 0; + }else{ + assert( bBulk==0 || bBulk==1 ); + if( iParentIdx==0 ){ + nxDiv = 0; + }else if( iParentIdx==i ){ + nxDiv = i-2+bBulk; + }else{ + nxDiv = iParentIdx-1; + } + i = 2-bBulk; + } + nOld = i+1; + if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ + pRight = &pParent->aData[pParent->hdrOffset+8]; + }else{ + pRight = findCell(pParent, i+nxDiv-pParent->nOverflow); + } + pgno = get4byte(pRight); + while( 1 ){ + if( rc==SQLITE_OK ){ + rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + } + if( rc ){ + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + } + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } + if( (i--)==0 ) break; + + if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ + apDiv[i] = pParent->apOvfl[0]; + pgno = get4byte(apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + pParent->nOverflow = 0; + }else{ + apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); + pgno = get4byte(apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + + /* Drop the cell from the parent page. apDiv[i] still points to + ** the cell within the parent, even though it has been dropped. + ** This is safe because dropping a cell only overwrites the first + ** four bytes of it, and this function does not need the first + ** four bytes of the divider cell. So the pointer is safe to use + ** later on. + ** + ** But not if we are in secure-delete mode. In secure-delete mode, + ** the dropCell() routine will overwrite the entire cell with zeroes. + ** In this case, temporarily copy the cell into the aOvflSpace[] + ** buffer. It will be copied out again as soon as the aSpace[] buffer + ** is allocated. */ + if( pBt->btsFlags & BTS_FAST_SECURE ){ + int iOff; + + /* If the following if() condition is not true, the db is corrupted. + ** The call to dropCell() below will detect this. */ + iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + if( (iOff+szNew[i])<=(int)pBt->usableSize ){ + memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); + apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; + } + } + dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); + } + } + + /* Make nMaxCells a multiple of 4 in order to preserve 8-byte + ** alignment */ + nMaxCells = nOld*(MX_CELL(pBt) + ArraySize(pParent->apOvfl)); + nMaxCells = (nMaxCells + 3)&~3; + + /* + ** Allocate space for memory structures + */ + szScratch = + nMaxCells*sizeof(u8*) /* b.apCell */ + + nMaxCells*sizeof(u16) /* b.szCell */ + + pBt->pageSize; /* aSpace1 */ + + assert( szScratch<=7*(int)pBt->pageSize ); + b.apCell = sqlite3StackAllocRaw(0, szScratch ); + if( b.apCell==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto balance_cleanup; + } + b.szCell = (u16*)&b.apCell[nMaxCells]; + aSpace1 = (u8*)&b.szCell[nMaxCells]; + assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); + + /* + ** Load pointers to all cells on sibling pages and the divider cells + ** into the local b.apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. + ** + ** If the siblings are on leaf pages, then the child pointers of the + ** divider cells are stripped from the cells before they are copied + ** into aSpace1[]. In this way, all cells in b.apCell[] are without + ** child pointers. If siblings are not leaves, then all cell in + ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] + ** are alike. + ** + ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. + ** leafData: 1 if pPage holds key+data and pParent holds only keys. + */ + b.pRef = apOld[0]; + leafCorrection = b.pRef->leaf*4; + leafData = b.pRef->intKeyLeaf; + for(i=0; inCell; + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) + + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + + /* Load b.apCell[] with pointers to all cells in pOld. If pOld + ** contains overflow cells, include them in the b.apCell[] array + ** in the correct spot. + ** + ** Note that when there are multiple overflow cells, it is always the + ** case that they are sequential and adjacent. This invariant arises + ** because multiple overflows can only occurs when inserting divider + ** cells into a parent on a prior balance, and divider cells are always + ** adjacent and are inserted in order. There is an assert() tagged + ** with "NOTE 1" in the overflow cell insertion loop to prove this + ** invariant. + ** + ** This must be done in advance. Once the balance starts, the cell + ** offset section of the btree page will be overwritten and we will no + ** long be able to find the cells if a pointer to each cell is not saved + ** first. + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); + if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + limit = pOld->aiOvfl[0]; + for(j=0; jnOverflow; k++){ + assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ + b.apCell[b.nCell] = pOld->apOvfl[k]; + b.nCell++; + } + } + piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + while( piCellnCell+pOld->nOverflow) ); + + cntOld[i] = b.nCell; + if( imaxLocal+23 ); + assert( iSpace1 <= (int)pBt->pageSize ); + memcpy(pTemp, apDiv[i], sz); + b.apCell[b.nCell] = pTemp+leafCorrection; + assert( leafCorrection==0 || leafCorrection==4 ); + b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; + if( !pOld->leaf ){ + assert( leafCorrection==0 ); + assert( pOld->hdrOffset==0 ); + /* The right pointer of the child page pOld becomes the left + ** pointer of the divider cell */ + memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); + }else{ + assert( leafCorrection==4 ); + while( b.szCell[b.nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); + assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); + aSpace1[iSpace1++] = 0x00; + b.szCell[b.nCell]++; + } + } + b.nCell++; + } + } + + /* + ** Figure out the number of pages needed to hold all b.nCell cells. + ** Store this number in "k". Also compute szNew[] which is the total + ** size of all cells on the i-th page and cntNew[] which is the index + ** in b.apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal b.nCell. + ** + ** Values computed by this block: + ** + ** k: The total number of sibling pages + ** szNew[i]: Spaced used on the i-th sibling page. + ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to + ** the right of the i-th sibling page. + ** usableSpace: Number of bytes of space available on each sibling. + ** + */ + usableSpace = pBt->usableSize - 12 + leafCorrection; + for(i=k=0; iaDataEnd; + b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } + if( !leafData ){ + k++; + b.apEnd[k] = pParent->aDataEnd; + b.ixNx[k] = cntOld[i]+1; + } + assert( p->nFree>=0 ); + szNew[i] = usableSpace - p->nFree; + for(j=0; jnOverflow; j++){ + szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); + } + cntNew[i] = cntOld[i]; + } + k = nOld; + for(i=0; iusableSpace ){ + if( i+1>=k ){ + k = i+2; + if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + szNew[k-1] = 0; + cntNew[k-1] = b.nCell; + } + sz = 2 + cachedCellSize(&b, cntNew[i]-1); + szNew[i] -= sz; + if( !leafData ){ + if( cntNew[i]usableSpace ) break; + szNew[i] += sz; + cntNew[i]++; + if( !leafData ){ + if( cntNew[i]=b.nCell ){ + k = i+1; + }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } + + /* + ** The packing computed by the previous block is biased toward the siblings + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. + ** + ** This adjustment is more than an optimization. The packing above might + ** be so out of balance as to be illegal. For example, the right-most + ** sibling might be completely empty. This adjustment is not optional. + */ + for(i=k-1; i>0; i--){ + int szRight = szNew[i]; /* Size of sibling on the right */ + int szLeft = szNew[i-1]; /* Size of sibling on the left */ + int r; /* Index of right-most cell in left sibling */ + int d; /* Index of first cell to the left of right sibling */ + + r = cntNew[i-1] - 1; + d = r + 1 - leafData; + (void)cachedCellSize(&b, d); + do{ + assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + break; + } + szRight += b.szCell[d] + 2; + szLeft -= b.szCell[r] + 2; + cntNew[i-1] = r; + r--; + d--; + }while( r>=0 ); + szNew[i] = szRight; + szNew[i-1] = szLeft; + if( cntNew[i-1] <= (i>1 ? cntNew[i-2] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } + + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. + */ + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 + )); + + /* + ** Allocate k new pages. Reuse old pages where possible. + */ + pageFlags = apOld[0]->aData[0]; + for(i=0; ipDbPage); + nNew++; + if( sqlite3PagerPageRefcount(pNew->pDbPage)!=1+(i==(iParentIdx-nxDiv)) ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc ) goto balance_cleanup; + }else{ + assert( i>0 ); + rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); + if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); + apNew[i] = pNew; + nNew++; + cntOld[i] = b.nCell; + + /* Set the pointer-map entry for the new sibling page. */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } + } + } + + /* + ** Reassign page numbers so that the new pages are in ascending order. + ** This helps to keep entries in the disk file in order so that a scan + ** of the table is closer to a linear scan through the file. That in turn + ** helps the operating system to deliver pages from the disk more rapidly. + ** + ** An O(n^2) insertion sort algorithm is used, but since n is never more + ** than (NB+2) (a small constant), that should not be a problem. + ** + ** When NB==3, this one optimization makes the database about 25% faster + ** for large insertions and deletions. + */ + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; ji ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; + } + } + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], + nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, + nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, + nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); + + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( nNew>=1 && nNew<=ArraySize(apNew) ); + assert( apNew[nNew-1]!=0 ); + put4byte(pRight, apNew[nNew-1]->pgno); + + /* If the sibling pages are not leaves, ensure that the right-child pointer + ** of the right-most new sibling page is set to the value that was + ** originally in the same field of the right-most old sibling page. */ + if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ + MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } + + /* Make any required updates to pointer map entries associated with + ** cells stored on sibling pages following the balance operation. Pointer + ** map entries associated with divider cells are set by the insertCell() + ** routine. The associated pointer map entries are: + ** + ** a) if the cell contains a reference to an overflow chain, the + ** entry associated with the first page in the overflow chain, and + ** + ** b) if the sibling pages are not leaves, the child page associated + ** with the cell. + ** + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been + ** populated, not here. + */ + if( ISAUTOVACUUM ){ + MemPage *pOld; + MemPage *pNew = pOld = apNew[0]; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int iNew = 0; + int iOld = 0; + + for(i=0; i=0 && iOldnCell + pOld->nOverflow + !leafData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow cell), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) + ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( cachedCellSize(&b,i)>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pOld, pCell, &rc); + } + if( rc ) goto balance_cleanup; + } + } + } + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + pNew->xParseCell(pNew, b.apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" + ** and WITHOUT ROWID tables with exactly one column which is the + ** primary key. + */ + if( b.szCell[j]==4 ){ + assert(leafCorrection==4); + sz = pParent->xCellSize(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. + ** + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. + ** + ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. + */ + for(i=1-nNew; i=0 && iPg=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ + ){ + int iNew; + int iOld; + int nNewCell; + + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnFree = usableSpace-szNew[iPg]; + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); + } + } + + /* All pages have been processed exactly once */ + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + + assert( nOld>0 ); + assert( nNew>0 ); + + if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ + /* The root page of the b-tree now contains no cells. The only sibling + ** page is the right-child of the parent. Copy the contents of the + ** child page into the parent, decreasing the overall height of the + ** b-tree structure by one. This is described as the "balance-shallower" + ** sub-algorithm in some documentation. + ** + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages + ** for which the pointer is stored within the content being copied. + ** + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ + assert( nNew==1 || CORRUPT_DB ); + rc = defragmentPage(apNew[0], -1); + testcase( rc!=SQLITE_OK ); + assert( apNew[0]->nFree == + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) + || rc!=SQLITE_OK + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); + } + } + + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, b.nCell)); + + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ + /* The ptrmapCheckPages() contains assert() statements that verify that + ** all pointer map pages are set correctly. This is helpful while + ** debugging. This is usually disabled because a corrupt database may + ** cause an assert() statement to fail. */ + ptrmapCheckPages(apNew, nNew); + ptrmapCheckPages(&pParent, 1); + } +#endif + + /* + ** Cleanup before returning. + */ +balance_cleanup: + sqlite3StackFree(0, b.apCell); + for(i=0; ipBt; /* The BTree */ + + assert( pRoot->nOverflow>0 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + + /* Make pRoot, the root page of the b-tree, writable. Allocate a new + ** page that will become the new right-child of pPage. Copy the contents + ** of the node stored on pRoot into the new child page. + */ + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc==SQLITE_OK ){ + rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); + copyNodeContent(pRoot, pChild, &rc); + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); + } + } + if( rc ){ + *ppChild = 0; + releasePage(pChild); + return rc; + } + assert( sqlite3PagerIswriteable(pChild->pDbPage) ); + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); + + TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + + /* Copy the overflow cells from pRoot to pChild */ + memcpy(pChild->aiOvfl, pRoot->aiOvfl, + pRoot->nOverflow*sizeof(pRoot->aiOvfl[0])); + memcpy(pChild->apOvfl, pRoot->apOvfl, + pRoot->nOverflow*sizeof(pRoot->apOvfl[0])); + pChild->nOverflow = pRoot->nOverflow; + + /* Zero the contents of pRoot. Then install pChild as the right-child. */ + zeroPage(pRoot, pChild->aData[0] & ~PTF_LEAF); + put4byte(&pRoot->aData[pRoot->hdrOffset+8], pgnoChild); + + *ppChild = pChild; + return SQLITE_OK; +} + +/* +** Return SQLITE_CORRUPT if any cursor other than pCur is currently valid +** on the same B-tree as pCur. +** +** This can if a database is corrupt with two or more SQL tables +** pointing to the same b-tree. If an insert occurs on one SQL table +** and causes a BEFORE TRIGGER to do a secondary insert on the other SQL +** table linked to the same b-tree. If the secondary insert causes a +** rebalance, that can change content out from under the cursor on the +** first SQL table, violating invariants on the first insert. +*/ +static int anotherValidCursor(BtCursor *pCur){ + BtCursor *pOther; + for(pOther=pCur->pBt->pCursor; pOther; pOther=pOther->pNext){ + if( pOther!=pCur + && pOther->eState==CURSOR_VALID + && pOther->pPage==pCur->pPage + ){ + return SQLITE_CORRUPT_BKPT; + } + } + return SQLITE_OK; +} + +/* +** The page that pCur currently points to has just been modified in +** some way. This function figures out if this modification means the +** tree needs to be balanced, and if so calls the appropriate balancing +** routine. Balancing routines are: +** +** balance_quick() +** balance_deeper() +** balance_nonroot() +*/ +static int balance(BtCursor *pCur){ + int rc = SQLITE_OK; + const int nMin = pCur->pBt->usableSize * 2 / 3; + u8 aBalanceQuickSpace[13]; + u8 *pFree = 0; + + VVA_ONLY( int balance_quick_called = 0 ); + VVA_ONLY( int balance_deeper_called = 0 ); + + do { + int iPage; + MemPage *pPage = pCur->pPage; + + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; + if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else if( (iPage = pCur->iPage)==0 ){ + if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ + /* The root page of the b-tree is overfull. In this case call the + ** balance_deeper() function to create a new child for the root-page + ** and copy the current contents of the root-page to it. The + ** next iteration of the do-loop will balance the child page. + */ + assert( balance_deeper_called==0 ); + VVA_ONLY( balance_deeper_called++ ); + rc = balance_deeper(pPage, &pCur->apPage[1]); + if( rc==SQLITE_OK ){ + pCur->iPage = 1; + pCur->ix = 0; + pCur->aiIdx[0] = 0; + pCur->apPage[0] = pPage; + pCur->pPage = pCur->apPage[1]; + assert( pCur->pPage->nOverflow ); + } + }else{ + break; + } + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; + + rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } + if( rc==SQLITE_OK ){ +#ifndef SQLITE_OMIT_QUICKBALANCE + if( pPage->intKeyLeaf + && pPage->nOverflow==1 + && pPage->aiOvfl[0]==pPage->nCell + && pParent->pgno!=1 + && pParent->nCell==iIdx + ){ + /* Call balance_quick() to create a new sibling of pPage on which + ** to store the overflow cell. balance_quick() inserts a new cell + ** into pParent, which may cause pParent overflow. If this + ** happens, the next iteration of the do-loop will balance pParent + ** use either balance_nonroot() or balance_deeper(). Until this + ** happens, the overflow cell is stored in the aBalanceQuickSpace[] + ** buffer. + ** + ** The purpose of the following assert() is to check that only a + ** single call to balance_quick() is made for each call to this + ** function. If this were not verified, a subtle bug involving reuse + ** of the aBalanceQuickSpace[] might sneak in. + */ + assert( balance_quick_called==0 ); + VVA_ONLY( balance_quick_called++ ); + rc = balance_quick(pParent, pPage, aBalanceQuickSpace); + }else +#endif + { + /* In this case, call balance_nonroot() to redistribute cells + ** between pPage and up to 2 of its sibling pages. This involves + ** modifying the contents of pParent, which may cause pParent to + ** become overfull or underfull. The next iteration of the do-loop + ** will balance the parent page to correct this. + ** + ** If the parent page becomes overfull, the overflow cell or cells + ** are stored in the pSpace buffer allocated immediately below. + ** A subsequent iteration of the do-loop will deal with this by + ** calling balance_nonroot() (balance_deeper() may be called first, + ** but it doesn't deal with overflow cells - just moves them to a + ** different page). Once this subsequent call to balance_nonroot() + ** has completed, it is safe to release the pSpace buffer used by + ** the previous call, as the overflow cell data will have been + ** copied either into the body of a database page or into the new + ** pSpace buffer passed to the latter call to balance_nonroot(). + */ + u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, + pCur->hints&BTREE_BULKLOAD); + if( pFree ){ + /* If pFree is not NULL, it points to the pSpace buffer used + ** by a previous call to balance_nonroot(). Its contents are + ** now stored either on real database pages or within the + ** new pSpace buffer, so it may be safely freed here. */ + sqlite3PageFree(pFree); + } + + /* The pSpace buffer will be freed after the next call to + ** balance_nonroot(), or just before this function returns, whichever + ** comes first. */ + pFree = pSpace; + } + } + + pPage->nOverflow = 0; + + /* The next iteration of the do-loop balances the parent page. */ + releasePage(pPage); + pCur->iPage--; + assert( pCur->iPage>=0 ); + pCur->pPage = pCur->apPage[pCur->iPage]; + } + }while( rc==SQLITE_OK ); + + if( pFree ){ + sqlite3PageFree(pFree); + } + return rc; +} + +/* Overwrite content from pX into pDest. Only do the write if the +** content is different from what is already there. +*/ +static int btreeOverwriteContent( + MemPage *pPage, /* MemPage on which writing will occur */ + u8 *pDest, /* Pointer to the place to start writing */ + const BtreePayload *pX, /* Source of data to write */ + int iOffset, /* Offset of first byte to write */ + int iAmt /* Number of bytes to be written */ +){ + int nData = pX->nData - iOffset; + if( nData<=0 ){ + /* Overwritting with zeros */ + int i; + for(i=0; ipDbPage); + if( rc ) return rc; + memset(pDest + i, 0, iAmt - i); + } + }else{ + if( nDatapData) + iOffset, iAmt)!=0 ){ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + /* In a corrupt database, it is possible for the source and destination + ** buffers to overlap. This is harmless since the database is already + ** corrupt but it does cause valgrind and ASAN warnings. So use + ** memmove(). */ + memmove(pDest, ((u8*)pX->pData) + iOffset, iAmt); + } + } + return SQLITE_OK; +} + +/* +** Overwrite the cell that cursor pCur is pointing to with fresh content +** contained in pX. +*/ +static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ + int iOffset; /* Next byte of pX->pData to write */ + int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ + int rc; /* Return code */ + MemPage *pPage = pCur->pPage; /* Page being written */ + BtShared *pBt; /* Btree */ + Pgno ovflPgno; /* Next overflow page to write */ + u32 ovflPageSize; /* Size to write on overflow page */ + + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ + return SQLITE_CORRUPT_BKPT; + } + /* Overwrite the local portion first */ + rc = btreeOverwriteContent(pPage, pCur->info.pPayload, pX, + 0, pCur->info.nLocal); + if( rc ) return rc; + if( pCur->info.nLocal==nTotal ) return SQLITE_OK; + + /* Now overwrite the overflow pages */ + iOffset = pCur->info.nLocal; + assert( nTotal>=0 ); + assert( iOffset>=0 ); + ovflPgno = get4byte(pCur->info.pPayload + iOffset); + pBt = pPage->pBt; + ovflPageSize = pBt->usableSize - 4; + do{ + rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); + if( rc ) return rc; + if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + if( iOffset+ovflPageSize<(u32)nTotal ){ + ovflPgno = get4byte(pPage->aData); + }else{ + ovflPageSize = nTotal - iOffset; + } + rc = btreeOverwriteContent(pPage, pPage->aData+4, pX, + iOffset, ovflPageSize); + } + sqlite3PagerUnref(pPage->pDbPage); + if( rc ) return rc; + iOffset += ovflPageSize; + }while( iOffset0 then pCur points to a cell +** that is larger than (pKey,nKey). +** +** If seekResult==0, that means pCur is pointing at some unknown location. +** In that case, this routine must seek the cursor to the correct insertion +** point for (pKey,nKey) before doing the insertion. For index btrees, +** if pX->nMem is non-zero, then pX->aMem contains pointers to the unpacked +** key values and pX->aMem can be used instead of pX->pKey to avoid having +** to decode the key. +*/ +int sqlite3BtreeInsert( + BtCursor *pCur, /* Insert data into the table of this cursor */ + const BtreePayload *pX, /* Content of the row to be inserted */ + int flags, /* True if this is likely an append */ + int seekResult /* Result of prior MovetoUnpacked() call */ +){ + int rc; + int loc = seekResult; /* -1: before desired location +1: after */ + int szNew = 0; + int idx; + MemPage *pPage; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + unsigned char *oldCell; + unsigned char *newCell = 0; + + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags ); + assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 ); + + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE + && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); + + /* Save the positions of any other cursors open on this table. + ** + ** In some cases, the call to btreeMoveto() below is a no-op. For + ** example, when inserting data into a table with auto-generated integer + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the + ** data into the intkey B-Tree. In this case btreeMoveto() recognizes + ** that the cursor is already where it needs to be and returns without + ** doing any work. To avoid thwarting these optimizations, it is important + ** not to clear the cursor here. + */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } + + if( pCur->pKeyInfo==0 ){ + assert( pX->pKey==0 ); + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced */ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ +#ifdef SQLITE_DEBUG + if( flags & BTREE_SAVEPOSITION ){ + assert( pCur->curFlags & BTCF_ValidNKey ); + assert( pX->nKey==pCur->info.nKey ); + assert( loc==0 ); + } +#endif + + /* On the other hand, BTREE_SAVEPOSITION==0 does not imply + ** that the cursor is not pointing to a row to be overwritten. + ** So do a complete check. + */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ + /* The cursor is pointing to the entry that is to be + ** overwritten */ + assert( pX->nData>=0 && pX->nZero>=0 ); + if( pCur->info.nSize!=0 + && pCur->info.nPayload==(u32)pX->nData+pX->nZero + ){ + /* New entry is the same size as the old. Do an overwrite */ + return btreeOverwriteCell(pCur, pX); + } + assert( loc==0 ); + }else if( loc==0 ){ + /* The cursor is *not* pointing to the cell to be overwritten, nor + ** to an adjacent cell. Move the cursor so that it is pointing either + ** to the cell to be overwritten or an adjacent cell. + */ + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); + if( rc ) return rc; + } + }else{ + /* This is an index or a WITHOUT ROWID table */ + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ + assert( (flags & BTREE_SAVEPOSITION)==0 || loc==0 ); + + /* If the cursor is not already pointing either to the cell to be + ** overwritten, or if a new cell is being inserted, if the cursor is + ** not pointing to an immediately adjacent cell, then move the cursor + ** so that it does. + */ + if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ + if( pX->nMem ){ + UnpackedRecord r; + r.pKeyInfo = pCur->pKeyInfo; + r.aMem = pX->aMem; + r.nField = pX->nMem; + r.default_rc = 0; + r.errCode = 0; + r.r1 = 0; + r.r2 = 0; + r.eqSeen = 0; + rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, flags!=0, &loc); + }else{ + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, flags!=0, &loc); + } + if( rc ) return rc; + } + + /* If the cursor is currently pointing to an entry to be overwritten + ** and the new content is the same as as the old, then use the + ** overwrite optimization. + */ + if( loc==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==pX->nKey ){ + BtreePayload x2; + x2.pData = pX->pKey; + x2.nData = pX->nKey; + x2.nZero = 0; + return btreeOverwriteCell(pCur, &x2); + } + } + + } + assert( pCur->eState==CURSOR_VALID + || (pCur->eState==CURSOR_INVALID && loc) + || CORRUPT_DB ); + + pPage = pCur->pPage; + assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); + assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + if( pCur->eState>CURSOR_INVALID ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = btreeComputeFreeSpace(pPage); + } + if( rc ) return rc; + } + + TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, + loc==0 ? "overwrite" : "new entry")); + assert( pPage->isInit ); + newCell = pBt->pTmpSpace; + assert( newCell!=0 ); + if( flags & BTREE_PREFORMAT ){ + rc = SQLITE_OK; + szNew = pBt->nPreformatSize; + if( szNew<4 ) szNew = 4; + if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + CellInfo info; + pPage->xParseCell(pPage, newCell, &info); + if( info.nPayload!=info.nLocal ){ + Pgno ovfl = get4byte(&newCell[szNew-4]); + ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + } + } + }else{ + rc = fillInCell(pPage, newCell, pX, &szNew); + } + if( rc ) goto end_insert; + assert( szNew==pPage->xCellSize(pPage, newCell) ); + assert( szNew <= MX_CELL_SIZE(pBt) ); + idx = pCur->ix; + if( loc==0 ){ + CellInfo info; + assert( idxnCell ); + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ + goto end_insert; + } + oldCell = findCell(pPage, idx); + if( !pPage->leaf ){ + memcpy(newCell, oldCell, 4); + } + rc = clearCell(pPage, oldCell, &info); + testcase( pCur->curFlags & BTCF_ValidOvfl ); + invalidateOverflowCache(pCur); + if( info.nSize==szNew && info.nLocal==info.nPayload + && (!ISAUTOVACUUM || szNewminLocal) + ){ + /* Overwrite the old cell with the new if they are the same size. + ** We could also try to do this if the old cell is smaller, then add + ** the leftover space to the free list. But experiments show that + ** doing that is no faster then skipping this optimization and just + ** calling dropCell() and insertCell(). + ** + ** This optimization cannot be used on an autovacuum database if the + ** new entry uses overflow pages, as the insertCell() call below is + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ + return SQLITE_CORRUPT_BKPT; + } + if( oldCell+szNew > pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; + } + dropCell(pPage, idx, info.nSize, &rc); + if( rc ) goto end_insert; + }else if( loc<0 && pPage->nCell>0 ){ + assert( pPage->leaf ); + idx = ++pCur->ix; + pCur->curFlags &= ~BTCF_ValidNKey; + }else{ + assert( pPage->leaf ); + } + insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); + assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); + + /* If no error has occurred and pPage has an overflow cell, call balance() + ** to redistribute the cells within the tree. Since balance() may move + ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey + ** variables. + ** + ** Previous versions of SQLite called moveToRoot() to move the cursor + ** back to the root page as balance() used to invalidate the contents + ** of BtCursor.apPage[] and BtCursor.aiIdx[]. Instead of doing that, + ** set the cursor state to "invalid". This makes common insert operations + ** slightly faster. + ** + ** There is a subtle but important optimization here too. When inserting + ** multiple records into an intkey b-tree using a single cursor (as can + ** happen while processing an "INSERT INTO ... SELECT" statement), it + ** is advantageous to leave the cursor pointing to the last entry in + ** the b-tree if possible. If the cursor is left pointing to the last + ** entry in the table, and the next row inserted has an integer key + ** larger than the largest existing key, it is possible to insert the + ** row without seeking the cursor. This can be a big performance boost. + */ + pCur->info.nSize = 0; + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); + pCur->curFlags &= ~(BTCF_ValidNKey); + rc = balance(pCur); + + /* Must make sure nOverflow is reset to zero even if the balance() + ** fails. Internal data structure corruption will result otherwise. + ** Also, set the cursor state to invalid. This stops saveCursorPosition() + ** from trying to save the current position of the cursor. */ + pCur->pPage->nOverflow = 0; + pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + if( pCur->pKeyInfo ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } + } + assert( pCur->iPage<0 || pCur->pPage->nOverflow==0 ); + +end_insert: + return rc; +} + +/* +** This function is used as part of copying the current row from cursor +** pSrc into cursor pDest. If the cursors are open on intkey tables, then +** parameter iKey is used as the rowid value when the record is copied +** into pDest. Otherwise, the record is copied verbatim. +** +** This function does not actually write the new value to cursor pDest. +** Instead, it creates and populates any required overflow pages and +** writes the data for the new cell into the BtShared.pTmpSpace buffer +** for the destination database. The size of the cell, in bytes, is left +** in BtShared.nPreformatSize. The caller completes the insertion by +** calling sqlite3BtreeInsert() with the BTREE_PREFORMAT flag specified. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ + int rc = SQLITE_OK; + BtShared *pBt = pDest->pBt; + u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ + const u8 *aIn; /* Pointer to next input buffer */ + u32 nIn; /* Size of input buffer aIn[] */ + u32 nRem; /* Bytes of data still to copy */ + + getCellInfo(pSrc); + aOut += putVarint32(aOut, pSrc->info.nPayload); + if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey); + nIn = pSrc->info.nLocal; + aIn = pSrc->info.pPayload; + if( aIn+nIn>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + nRem = pSrc->info.nPayload; + if( nIn==nRem && nInpPage->maxLocal ){ + memcpy(aOut, aIn, nIn); + pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + }else{ + Pager *pSrcPager = pSrc->pBt->pPager; + u8 *pPgnoOut = 0; + Pgno ovflIn = 0; + DbPage *pPageIn = 0; + MemPage *pPageOut = 0; + u32 nOut; /* Size of output buffer aOut[] */ + + nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); + pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + if( nOutinfo.nPayload ){ + pPgnoOut = &aOut[nOut]; + pBt->nPreformatSize += 4; + } + + if( nRem>nIn ){ + if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + ovflIn = get4byte(&pSrc->info.pPayload[nIn]); + } + + do { + nRem -= nOut; + do{ + assert( nOut>0 ); + if( nIn>0 ){ + int nCopy = MIN(nOut, nIn); + memcpy(aOut, aIn, nCopy); + nOut -= nCopy; + nIn -= nCopy; + aOut += nCopy; + aIn += nCopy; + } + if( nOut>0 ){ + sqlite3PagerUnref(pPageIn); + pPageIn = 0; + rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + aIn = (const u8*)sqlite3PagerGetData(pPageIn); + ovflIn = get4byte(aIn); + aIn += 4; + nIn = pSrc->pBt->usableSize - 4; + } + } + }while( rc==SQLITE_OK && nOut>0 ); + + if( rc==SQLITE_OK && nRem>0 ){ + Pgno pgnoNew; + MemPage *pNew = 0; + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + put4byte(pPgnoOut, pgnoNew); + if( ISAUTOVACUUM && pPageOut ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); + } + releasePage(pPageOut); + pPageOut = pNew; + if( pPageOut ){ + pPgnoOut = pPageOut->aData; + put4byte(pPgnoOut, 0); + aOut = &pPgnoOut[4]; + nOut = MIN(pBt->usableSize - 4, nRem); + } + } + }while( nRem>0 && rc==SQLITE_OK ); + + releasePage(pPageOut); + sqlite3PagerUnref(pPageIn); + } + + return rc; +} + +/* +** Delete the entry that the cursor is pointing to. +** +** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then +** the cursor is left pointing at an arbitrary location after the delete. +** But if that bit is set, then the cursor is left in a state such that +** the next call to BtreeNext() or BtreePrev() moves it to the same row +** as it would have been on if the call to BtreeDelete() had been omitted. +** +** The BTREE_AUXDELETE bit of flags indicates that is one of several deletes +** associated with a single table entry and its indexes. Only one of those +** deletes is considered the "primary" delete. The primary delete occurs +** on a cursor that is not a BTREE_FORDELETE cursor. All but one delete +** operation on non-FORDELETE cursors is tagged with the AUXDELETE flag. +** The BTREE_AUXDELETE bit is a hint that is not used by this implementation, +** but which might be used by alternative storage engines. +*/ +int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + CellInfo info; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ + u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ + + assert( cursorOwnsBtShared(pCur) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( pCur->curFlags & BTCF_WriteFlag ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + assert( !hasReadConflicts(p, pCur->pgnoRoot) ); + assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState==CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + if( rc ) return rc; + } + assert( pCur->eState==CURSOR_VALID ); + + iCellDepth = pCur->iPage; + iCellIdx = pCur->ix; + pPage = pCur->pPage; + pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ) return SQLITE_CORRUPT; + + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + || pPage->nCell==1 /* See dbfuzz001.test for a test case */ + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* If the page containing the entry to delete is not a leaf page, move + ** the cursor to the largest entry in the tree that is smaller than + ** the entry being deleted. This cell will replace the cell being deleted + ** from the internal node. The 'previous' entry is used for this instead + ** of the 'next' entry, as the previous entry is always a part of the + ** sub-tree headed by the child page of the cell being deleted. This makes + ** balancing the tree following the delete operation easier. */ + if( !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, 0); + assert( rc!=SQLITE_DONE ); + if( rc ) return rc; + } + + /* Save the positions of any other cursors open on this table before + ** making any modifications. */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + rc = clearCell(pPage, pCell, &info); + dropCell(pPage, iCellIdx, info.nSize, &rc); + if( rc ) return rc; + + /* If the cell deleted was not located on a leaf page, then the cursor + ** is currently pointing to the largest entry in the sub-tree headed + ** by the child-page of the cell that was just deleted from an internal + ** node. The cell from the leaf node needs to be moved to the internal + ** node to replace the deleted cell. */ + if( !pPage->leaf ){ + MemPage *pLeaf = pCur->pPage; + int nCell; + Pgno n; + unsigned char *pTmp; + + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } + if( iCellDepthiPage-1 ){ + n = pCur->apPage[iCellDepth+1]->pgno; + }else{ + n = pCur->pPage->pgno; + } + pCell = findCell(pLeaf, pLeaf->nCell-1); + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + nCell = pLeaf->xCellSize(pLeaf, pCell); + assert( MX_CELL_SIZE(pBt) >= nCell ); + pTmp = pBt->pTmpSpace; + assert( pTmp!=0 ); + rc = sqlite3PagerWrite(pLeaf->pDbPage); + if( rc==SQLITE_OK ){ + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + } + dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); + if( rc ) return rc; + } + + /* Balance the tree. If the entry deleted was located on a leaf page, + ** then the cursor still points to that page. In this case the first + ** call to balance() repairs the tree, and the if(...) condition is + ** never true. + ** + ** Otherwise, if the entry deleted was on an internal node page, then + ** pCur is pointing to the leaf page from which a cell was removed to + ** replace the cell deleted from the internal node. This is slightly + ** tricky as the leaf node may be underfull, and the internal node may + ** be either under or overfull. In this case run the balancing algorithm + ** on the leaf node first. If the balance proceeds far enough up the + ** tree that we can be sure that any problem in the internal node has + ** been corrected, so be it. Otherwise, after balancing the leaf node, + ** walk the cursor up the tree to the internal node and balance it as + ** well. */ + rc = balance(pCur); + if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + releasePageNotNull(pCur->pPage); + pCur->iPage--; + while( pCur->iPage>iCellDepth ){ + releasePage(pCur->apPage[pCur->iPage--]); + } + pCur->pPage = pCur->apPage[pCur->iPage]; + rc = balance(pCur); + } + + if( rc==SQLITE_OK ){ + if( bSkipnext ){ + assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); + assert( pPage==pCur->pPage || CORRUPT_DB ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->ix = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + btreeReleaseAllCursorPages(pCur); + pCur->eState = CURSOR_REQUIRESEEK; + } + if( rc==SQLITE_EMPTY ) rc = SQLITE_OK; + } + } + return rc; +} + +/* +** Create a new BTree table. Write into *piTable the page +** number for the root page of the new table. +** +** The type of type is determined by the flags parameter. Only the +** following values of flags are currently in use. Other values for +** flags might not work: +** +** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys +** BTREE_ZERODATA Used for SQL indices +*/ +static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ + BtShared *pBt = p->pBt; + MemPage *pRoot; + Pgno pgnoRoot; + int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + +#ifdef SQLITE_OMIT_AUTOVACUUM + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ){ + return rc; + } +#else + if( pBt->autoVacuum ){ + Pgno pgnoMove; /* Move a page here to make room for the root-page */ + MemPage *pPageMove; /* The page to move to. */ + + /* Creating a new table may probably require moving an existing database + ** to make room for the new tables root page. In case this page turns + ** out to be an overflow page, delete all overflow page-map caches + ** held by open cursors. + */ + invalidateAllOverflowCache(pBt); + + /* Read the value of meta[3] from the database to determine where the + ** root page of the new table should go. meta[3] is the largest root-page + ** created so far, so the new root-page is (meta[3]+1). + */ + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + if( pgnoRoot>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + pgnoRoot++; + + /* The new root-page may not be allocated on a pointer-map page, or the + ** PENDING_BYTE page. + */ + while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || + pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ + pgnoRoot++; + } + assert( pgnoRoot>=3 ); + + /* Allocate a page. The page that currently resides at pgnoRoot will + ** be moved to the allocated page (unless the allocated page happens + ** to reside at pgnoRoot). + */ + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + + if( pgnoMove!=pgnoRoot ){ + /* pgnoRoot is the page that will be used for the root-page of + ** the new table (assuming an error did not occur). But we were + ** allocated pgnoMove. If required (i.e. if it was not allocated + ** by extending the file), the current page at position pgnoMove + ** is already journaled. + */ + u8 eType = 0; + Pgno iPtrPage = 0; + + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Move the page currently at pgnoRoot to pgnoMove. */ + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); + if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + assert( eType!=PTRMAP_ROOTPAGE ); + assert( eType!=PTRMAP_FREEPAGE ); + rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); + releasePage(pRoot); + + /* Obtain the page at pgnoRoot */ + if( rc!=SQLITE_OK ){ + return rc; + } + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + }else{ + pRoot = pPageMove; + } + + /* Update the pointer-map and meta-data with the new root-page number. */ + ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); + if( rc ){ + releasePage(pRoot); + return rc; + } + + /* When the new root page was allocated, page 1 was made writable in + ** order either to increase the database filesize, or to decrement the + ** freelist count. Hence, the sqlite3BtreeUpdateMeta() call cannot fail. + */ + assert( sqlite3PagerIswriteable(pBt->pPage1->pDbPage) ); + rc = sqlite3BtreeUpdateMeta(p, 4, pgnoRoot); + if( NEVER(rc) ){ + releasePage(pRoot); + return rc; + } + + }else{ + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ) return rc; + } +#endif + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); + sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); + *piTable = pgnoRoot; + return SQLITE_OK; +} +int sqlite3BtreeCreateTable(Btree *p, Pgno *piTable, int flags){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCreateTable(p, piTable, flags); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Erase the given database page and all its children. Return +** the page to the freelist. +*/ +static int clearDatabasePage( + BtShared *pBt, /* The BTree that contains the table */ + Pgno pgno, /* Page number to clear */ + int freePageFlag, /* Deallocate page if true */ + int *pnChange /* Add number of Cells freed to this counter */ +){ + MemPage *pPage; + int rc; + unsigned char *pCell; + int i; + int hdr; + CellInfo info; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + if( rc ) return rc; + if( pPage->bBusy ){ + rc = SQLITE_CORRUPT_BKPT; + goto cleardatabasepage_out; + } + pPage->bBusy = 1; + hdr = pPage->hdrOffset; + for(i=0; inCell; i++){ + pCell = findCell(pPage, i); + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + } + rc = clearCell(pPage, pCell, &info); + if( rc ) goto cleardatabasepage_out; + } + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + }else if( pnChange ){ + assert( pPage->intKey || CORRUPT_DB ); + testcase( !pPage->intKey ); + *pnChange += pPage->nCell; + } + if( freePageFlag ){ + freePage(pPage, &rc); + }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); + } + +cleardatabasepage_out: + pPage->bBusy = 0; + releasePage(pPage); + return rc; +} + +/* +** Delete all information from a single table in the database. iTable is +** the page number of the root of the table. After this routine returns, +** the root page is empty, but still exists. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** read cursors on the table. Open write cursors are moved to the +** root of the table. +** +** If pnChange is not NULL, then table iTable must be an intkey table. The +** integer value pointed to by pnChange is incremented by the number of +** entries in the table. +*/ +int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + + rc = saveAllCursors(pBt, (Pgno)iTable, 0); + + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + +/* +** Erase all information in a table and add the root of the table to +** the freelist. Except, the root of the principle table (the one on +** page 1) is never added to the freelist. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** cursors on the table. +** +** If AUTOVACUUM is enabled and the page at iTable is not the last +** root page in the database file, then the last root page +** in the database file is moved into the slot formerly occupied by +** iTable and that last slot formerly occupied by the last root page +** is added to the freelist instead of iTable. In this say, all +** root pages are kept at the beginning of the database file, which +** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** page number that used to be the last root page in the file before +** the move. If no page gets moved, *piMoved is set to 0. +** The last root page is recorded in meta[3] and the value of +** meta[3] is updated by this procedure. +*/ +static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ + int rc; + MemPage *pPage = 0; + BtShared *pBt = p->pBt; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->inTrans==TRANS_WRITE ); + assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + if( rc ) return rc; + rc = sqlite3BtreeClearTable(p, iTable, 0); + if( rc ){ + releasePage(pPage); + return rc; + } + + *piMoved = 0; + +#ifdef SQLITE_OMIT_AUTOVACUUM + freePage(pPage, &rc); + releasePage(pPage); +#else + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; + } + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. + */ + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); + } +#endif + return rc; +} +int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeDropTable(p, iTable, piMoved); + sqlite3BtreeLeave(p); + return rc; +} + + +/* +** This function may only be called if the b-tree connection already +** has a read or write transaction open on the database. +** +** Read the meta-information out of a database file. Meta[0] +** is the number of free pages currently in the database. Meta[1] +** through meta[15] are available for use by higher layers. Meta[0] +** is read-only, the others are read/write. +** +** The schema layer numbers meta values differently. At the schema +** layer (and the SetCookie and ReadCookie opcodes) the number of +** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. +*/ +void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ + BtShared *pBt = p->pBt; + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE ); + assert( SQLITE_OK==querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK) ); + assert( pBt->pPage1 ); + assert( idx>=0 && idx<=15 ); + + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iBDataVersion; + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } + + /* If auto-vacuum is disabled in this build and this is an auto-vacuum + ** database, mark the database as read-only. */ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } +#endif + + sqlite3BtreeLeave(p); +} + +/* +** Write meta-information back into the database. Meta[0] is +** read-only and may not be written. +*/ +int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ + BtShared *pBt = p->pBt; + unsigned char *pP1; + int rc; + assert( idx>=1 && idx<=15 ); + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1!=0 ); + pP1 = pBt->pPage1->aData; + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pP1[36 + idx*4], iMeta); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_INCR_VACUUM ){ + assert( pBt->autoVacuum || iMeta==0 ); + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; + } +#endif + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** The first argument, pCur, is a cursor opened on some b-tree. Count the +** number of entries in the b-tree and write the result to *pnEntry. +** +** SQLITE_OK is returned if the operation is successfully executed. +** Otherwise, if an error is encountered (i.e. an IO error or database +** corruption) an SQLite error code is returned. +*/ +int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ + i64 nEntry = 0; /* Value to return in *pnEntry */ + int rc; /* Return code */ + + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ + *pnEntry = 0; + return SQLITE_OK; + } + + /* Unless an error occurs, the following loop runs one iteration for each + ** page in the B-Tree structure (not including overflow pages). + */ + while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ + int iIdx; /* Index of child node in parent */ + MemPage *pPage; /* Current page of the b-tree */ + + /* If this is a leaf page or the tree is not an int-key tree, then + ** this page contains countable entries. Increment the entry counter + ** accordingly. + */ + pPage = pCur->pPage; + if( pPage->leaf || !pPage->intKey ){ + nEntry += pPage->nCell; + } + + /* pPage is a leaf node. This loop navigates the cursor so that it + ** points to the first interior cell that it points to the parent of + ** the next page in the tree that has not yet been visited. The + ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell + ** of the page, or to the number of cells in the page if the next page + ** to visit is the right-child of its parent. + ** + ** If all pages in the tree have been visited, return SQLITE_OK to the + ** caller. + */ + if( pPage->leaf ){ + do { + if( pCur->iPage==0 ){ + /* All pages of the b-tree have been visited. Return successfully. */ + *pnEntry = nEntry; + return moveToRoot(pCur); + } + moveToParent(pCur); + }while ( pCur->ix>=pCur->pPage->nCell ); + + pCur->ix++; + pPage = pCur->pPage; + } + + /* Descend to the child node of the cell that the cursor currently + ** points at. This is the right-child if (iIdx==pPage->nCell). + */ + iIdx = pCur->ix; + if( iIdx==pPage->nCell ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + }else{ + rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx))); + } + } + + /* An error has occurred. Return an error code. */ + return rc; +} + +/* +** Return the pager associated with a BTree. This routine is used for +** testing and debugging only. +*/ +Pager *sqlite3BtreePager(Btree *p){ + return p->pBt->pPager; +} + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Append a message to the error message string. +*/ +static void checkAppendMsg( + IntegrityCk *pCheck, + const char *zFormat, + ... +){ + va_list ap; + if( !pCheck->mxErr ) return; + pCheck->mxErr--; + pCheck->nErr++; + va_start(ap, zFormat); + if( pCheck->errMsg.nChar ){ + sqlite3_str_append(&pCheck->errMsg, "\n", 1); + } + if( pCheck->zPfx ){ + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + } + sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); + va_end(ap); + if( pCheck->errMsg.accError==SQLITE_NOMEM ){ + pCheck->bOomFault = 1; + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + +/* +** Return non-zero if the bit in the IntegrityCk.aPgRef[] array that +** corresponds to page iPg is already set. +*/ +static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); +} + +/* +** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. +*/ +static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); +} + + +/* +** Add 1 to the reference count for page iPage. If this is the second +** reference to the page, add an error message to pCheck->zErrMsg. +** Return 1 if there are 2 or more references to the page and 0 if +** if this is the first reference to the page. +** +** Also check that the page number is in bounds. +*/ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ + if( iPage>pCheck->nPage || iPage==0 ){ + checkAppendMsg(pCheck, "invalid page number %d", iPage); + return 1; + } + if( getPageReferenced(pCheck, iPage) ){ + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + return 1; + } + if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; + setPageReferenced(pCheck, iPage); + return 0; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Check that the entry in the pointer-map for page iChild maps to +** page iParent, pointer type ptrType. If not, append an error message +** to pCheck. +*/ +static void checkPtrmap( + IntegrityCk *pCheck, /* Integrity check context */ + Pgno iChild, /* Child page number */ + u8 eType, /* Expected pointer map type */ + Pgno iParent /* Expected pointer map parent page number */ +){ + int rc; + u8 ePtrmapType; + Pgno iPtrmapParent; + + rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->bOomFault = 1; + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + return; + } + + if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ + checkAppendMsg(pCheck, + "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + iChild, eType, iParent, ePtrmapType, iPtrmapParent); + } +} +#endif + +/* +** Check the integrity of the freelist or of an overflow page list. +** Verify that the number of pages on the list is N. +*/ +static void checkList( + IntegrityCk *pCheck, /* Integrity checking context */ + int isFreeList, /* True for a freelist. False for overflow page list */ + Pgno iPage, /* Page number for first page in the list */ + u32 N /* Expected number of pages in the list */ +){ + int i; + u32 expected = N; + int nErrAtStart = pCheck->nErr; + while( iPage!=0 && pCheck->mxErr ){ + DbPage *pOvflPage; + unsigned char *pOvflData; + if( checkRef(pCheck, iPage) ) break; + N--; + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ + checkAppendMsg(pCheck, "failed to get page %d", iPage); + break; + } + pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); + if( isFreeList ){ + u32 n = (u32)get4byte(&pOvflData[4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); + } +#endif + if( n>pCheck->pBt->usableSize/4-2 ){ + checkAppendMsg(pCheck, + "freelist leaf count too big on page %d", iPage); + N--; + }else{ + for(i=0; i<(int)n; i++){ + Pgno iFreePage = get4byte(&pOvflData[8+i*4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); + } +#endif + checkRef(pCheck, iFreePage); + } + N -= n; + } + } +#ifndef SQLITE_OMIT_AUTOVACUUM + else{ + /* If this database supports auto-vacuum and iPage is not the last + ** page in this overflow list, check that the pointer-map entry for + ** the following page matches iPage. + */ + if( pCheck->pBt->autoVacuum && N>0 ){ + i = get4byte(pOvflData); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); + } + } +#endif + iPage = get4byte(pOvflData); + sqlite3PagerUnref(pOvflPage); + } + if( N && nErrAtStart==pCheck->nErr ){ + checkAppendMsg(pCheck, + "%s is %d but should be %d", + isFreeList ? "size" : "overflow list length", + expected-N, expected); + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* +** An implementation of a min-heap. +** +** aHeap[0] is the number of elements on the heap. aHeap[1] is the +** root element. The daughter nodes of aHeap[N] are aHeap[N*2] +** and aHeap[N*2+1]. +** +** The heap property is this: Every node is less than or equal to both +** of its daughter nodes. A consequence of the heap property is that the +** root node aHeap[1] is always the minimum value currently in the heap. +** +** The btreeHeapInsert() routine inserts an unsigned 32-bit number onto +** the heap, preserving the heap property. The btreeHeapPull() routine +** removes the root element from the heap (the minimum value in the heap) +** and then moves other nodes around as necessary to preserve the heap +** property. +** +** This heap is used for cell overlap and coverage testing. Each u32 +** entry represents the span of a cell or freeblock on a btree page. +** The upper 16 bits are the index of the first byte of a range and the +** lower 16 bits are the index of the last byte of that range. +*/ +static void btreeHeapInsert(u32 *aHeap, u32 x){ + u32 j, i = ++aHeap[0]; + aHeap[i] = x; + while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ + x = aHeap[j]; + aHeap[j] = aHeap[i]; + aHeap[i] = x; + i = j; + } +} +static int btreeHeapPull(u32 *aHeap, u32 *pOut){ + u32 j, i, x; + if( (x = aHeap[0])==0 ) return 0; + *pOut = aHeap[1]; + aHeap[1] = aHeap[x]; + aHeap[x] = 0xffffffff; + aHeap[0]--; + i = 1; + while( (j = i*2)<=aHeap[0] ){ + if( aHeap[j]>aHeap[j+1] ) j++; + if( aHeap[i]zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; + u8 savedIsInit = 0; + + /* Check that the page exists + */ + pBt = pCheck->pBt; + usableSize = pBt->usableSize; + if( iPage==0 ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %u: "; + pCheck->v1 = iPage; + if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ + checkAppendMsg(pCheck, + "unable to get the page. error code=%d", rc); + goto end_of_check; + } + + /* Clear MemPage.isInit to make sure the corruption detection code in + ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; + pPage->isInit = 0; + if( (rc = btreeInitPage(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ + checkAppendMsg(pCheck, + "btreeInitPage() returns error code %d", rc); + goto end_of_check; + } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } + data = pPage->aData; + hdr = pPage->hdrOffset; + + /* Set up for cell analysis */ + pCheck->zPfx = "On tree page %u cell %d: "; + contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + assert( pPage->nCell==nCell ); + + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + assert( pPage->aCellIdx==&data[cellStart] ); + pCellIdx = &data[cellStart + 2*(nCell-1)]; + + if( !pPage->leaf ){ + /* Analyze the right-child page of internal pages */ + pgno = get4byte(&data[hdr+8]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + pCheck->zPfx = "On page %u at right child: "; + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + }else{ + /* For leaf pages, the coverage check will occur in the same loop + ** as the other cell checks, so initialize the heap. */ + heap = pCheck->heap; + heap[0] = 0; + } + + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ + CellInfo info; + + /* Check cell size */ + pCheck->v2 = i; + assert( pCellIdx==&data[cellStart + i*2] ); + pc = get2byteAligned(pCellIdx); + pCellIdx -= 2; + if( pcusableSize-4 ){ + checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + pc, contentOffset, usableSize-4); + doCoverageCheck = 0; + continue; + } + pCell = &data[pc]; + pPage->xParseCell(pPage, pCell, &info); + if( pc+info.nSize>usableSize ){ + checkAppendMsg(pCheck, "Extends off end of page"); + doCoverageCheck = 0; + continue; + } + + /* Check for integer primary key out of range */ + if( pPage->intKey ){ + if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ + checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); + } + maxKey = info.nKey; + keyCanBeEqual = 0; /* Only the first key on the page may ==maxKey */ + } + + /* Check the content overflow list */ + if( info.nPayload>info.nLocal ){ + u32 nPage; /* Number of pages on the overflow chain */ + Pgno pgnoOvfl; /* First page of the overflow chain */ + assert( pc + info.nSize - 4 <= usableSize ); + nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); + pgnoOvfl = get4byte(&pCell[info.nSize - 4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); + } +#endif + checkList(pCheck, 0, pgnoOvfl, nPage); + } + + if( !pPage->leaf ){ + /* Check sanity of left child page for internal pages */ + pgno = get4byte(pCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + if( d2!=depth ){ + checkAppendMsg(pCheck, "Child page depth differs"); + depth = d2; + } + }else{ + /* Populate the coverage-checking heap for leaf pages */ + btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); + } + } + *piMinKey = maxKey; + + /* Check for complete coverage of the page + */ + pCheck->zPfx = 0; + if( doCoverageCheck && pCheck->mxErr>0 ){ + /* For leaf pages, the min-heap has already been initialized and the + ** cells have already been inserted. But for internal pages, that has + ** not yet been done, so do it now */ + if( !pPage->leaf ){ + heap = pCheck->heap; + heap[0] = 0; + for(i=nCell-1; i>=0; i--){ + u32 size; + pc = get2byteAligned(&data[cellStart+i*2]); + size = pPage->xCellSize(pPage, &data[pc]); + btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); + } + } + /* Add the freeblocks to the min-heap + ** + ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. + */ + i = get2byte(&data[hdr+1]); + while( i>0 ){ + int size, j; + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ + size = get2byte(&data[i+2]); + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ + j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ + i = j; + } + /* Analyze the min-heap looking for overlap between cells and/or + ** freeblocks, and counting the number of untracked bytes in nFrag. + ** + ** Each min-heap entry is of the form: (start_address<<16)|end_address. + ** There is an implied first entry the covers the page header, the cell + ** pointer index, and the gap between the cell pointer index and the start + ** of cell content. + ** + ** The loop below pulls entries from the min-heap in order and compares + ** the start_address against the previous end_address. If there is an + ** overlap, that means bytes are used multiple times. If there is a gap, + ** that gap is added to the fragmentation count. + */ + nFrag = 0; + prev = contentOffset - 1; /* Implied first min-heap entry */ + while( btreeHeapPull(heap,&x) ){ + if( (prev&0xffff)>=(x>>16) ){ + checkAppendMsg(pCheck, + "Multiple uses for byte %u of page %u", x>>16, iPage); + break; + }else{ + nFrag += (x>>16) - (prev&0xffff) - 1; + prev = x; + } + } + nFrag += usableSize - (prev&0xffff) - 1; + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ + if( heap[0]==0 && nFrag!=data[hdr+7] ){ + checkAppendMsg(pCheck, + "Fragmentation of %d bytes reported as %d on page %u", + nFrag, data[hdr+7], iPage); + } + } + +end_of_check: + if( !doCoverageCheck ) pPage->isInit = savedIsInit; + releasePage(pPage); + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; + return depth+1; +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** This routine does a complete check of the given BTree file. aRoot[] is +** an array of pages numbers were each page number is the root page of +** a table. nRoot is the number of entries in aRoot. +** +** A read-only or read-write transaction must be opened before calling +** this function. +** +** Write the number of error seen in *pnErr. Except for some memory +** allocation errors, an error message held in memory obtained from +** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is +** returned. If a memory allocation error occurs, NULL is returned. +** +** If the first entry in aRoot[] is 0, that indicates that the list of +** root pages is incomplete. This is a "partial integrity-check". This +** happens when performing an integrity check on a single table. The +** zero is skipped, of course. But in addition, the freelist checks +** and the checks to make sure every page is referenced are also skipped, +** since obviously it is not possible to know which pages are covered by +** the unverified btrees. Except, if aRoot[1] is 1, then the freelist +** checks are still performed. +*/ +char *sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr /* Write number of errors seen to this variable */ +){ + Pgno i; + IntegrityCk sCheck; + BtShared *pBt = p->pBt; + u64 savedDbFlags = pBt->db->flags; + char zErr[100]; + int bPartial = 0; /* True if not checking all btrees */ + int bCkFreelist = 1; /* True to scan the freelist */ + VVA_ONLY( int nRef ); + assert( nRoot>0 ); + + /* aRoot[0]==0 means this is a partial check */ + if( aRoot[0]==0 ){ + assert( nRoot>1 ); + bPartial = 1; + if( aRoot[1]!=1 ) bCkFreelist = 0; + } + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); + sCheck.db = db; + sCheck.pBt = pBt; + sCheck.pPager = pBt->pPager; + sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.mxErr = mxErr; + sCheck.nErr = 0; + sCheck.bOomFault = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; + sCheck.aPgRef = 0; + sCheck.heap = 0; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); + sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; + if( sCheck.nPage==0 ){ + goto integrity_ck_cleanup; + } + + sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + if( !sCheck.aPgRef ){ + sCheck.bOomFault = 1; + goto integrity_ck_cleanup; + } + sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + if( sCheck.heap==0 ){ + sCheck.bOomFault = 1; + goto integrity_ck_cleanup; + } + + i = PENDING_BYTE_PAGE(pBt); + if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + + /* Check the integrity of the freelist + */ + if( bCkFreelist ){ + sCheck.zPfx = "Main freelist: "; + checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; + } + + /* Check all the tables. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( !bPartial ){ + if( pBt->autoVacuum ){ + Pgno mx = 0; + Pgno mxInHdr; + for(i=0; (int)ipPage1->aData[52]); + if( mx!=mxInHdr ){ + checkAppendMsg(&sCheck, + "max rootpage (%d) disagrees with header (%d)", + mx, mxInHdr + ); + } + }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){ + checkAppendMsg(&sCheck, + "incremental_vacuum enabled with a max rootpage of zero" + ); + } + } +#endif + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; + for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } +#endif + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + pBt->db->flags = savedDbFlags; + + /* Make sure every page in the file is referenced + */ + if( !bPartial ){ + for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( getPageReferenced(&sCheck, i)==0 ){ + checkAppendMsg(&sCheck, "Page %d is never used", i); + } +#else + /* If the database supports auto-vacuum, make sure no tables contain + ** references to pointer-map pages. + */ + if( getPageReferenced(&sCheck, i)==0 && + (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Page %d is never used", i); + } + if( getPageReferenced(&sCheck, i)!=0 && + (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + } +#endif + } + } + + /* Clean up and report errors. + */ +integrity_ck_cleanup: + sqlite3PageFree(sCheck.heap); + sqlite3_free(sCheck.aPgRef); + if( sCheck.bOomFault ){ + sqlite3_str_reset(&sCheck.errMsg); + sCheck.nErr++; + } + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ) sqlite3_str_reset(&sCheck.errMsg); + /* Make sure this analysis did not leave any unref() pages. */ + assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); + sqlite3BtreeLeave(p); + return sqlite3StrAccumFinish(&sCheck.errMsg); +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* +** Return the full pathname of the underlying database file. Return +** an empty string if the database is in-memory or a TEMP database. +** +** The pager filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +const char *sqlite3BtreeGetFilename(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerFilename(p->pBt->pPager, 1); +} + +/* +** Return the pathname of the journal file for this database. The return +** value of this routine is the same regardless of whether the journal file +** has been created or not. +** +** The pager journal filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +const char *sqlite3BtreeGetJournalname(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerJournalname(p->pBt->pPager); +} + +/* +** Return one of SQLITE_TXN_NONE, SQLITE_TXN_READ, or SQLITE_TXN_WRITE +** to describe the current transaction state of Btree p. +*/ +int sqlite3BtreeTxnState(Btree *p){ + assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); + return p ? p->inTrans : 0; +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +/* +** Return true if there is currently a backup running on Btree p. +*/ +int sqlite3BtreeIsInBackup(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->nBackup!=0; +} + +/* +** This function returns a pointer to a blob of memory associated with +** a single shared-btree. The memory is used by client code for its own +** purposes (for example, to store a high-level schema associated with +** the shared-btree). The btree layer manages reference counting issues. +** +** The first time this is called on a shared-btree, nBytes bytes of memory +** are allocated, zeroed, and returned to the caller. For each subsequent +** call the nBytes parameter is ignored and a pointer to the same blob +** of memory returned. +** +** If the nBytes parameter is 0 and the blob of memory has not yet been +** allocated, a null pointer is returned. If the blob has already been +** allocated, it is returned as normal. +** +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the +** blob of allocated memory. The xFree function should not call sqlite3_free() +** on the memory, the btree layer does that. +*/ +void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( !pBt->pSchema && nBytes ){ + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); + pBt->xFreeSchema = xFree; + } + sqlite3BtreeLeave(p); + return pBt->pSchema; +} + +/* +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the +** sqlite_schema table. Otherwise SQLITE_OK. +*/ +int sqlite3BtreeSchemaLocked(Btree *p){ + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); + assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); + sqlite3BtreeLeave(p); + return rc; +} + + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Obtain a lock on the table whose root page is iTab. The +** lock is a write lock if isWritelock is true or a read lock +** if it is false. +*/ +int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ + int rc = SQLITE_OK; + assert( p->inTrans!=TRANS_NONE ); + if( p->sharable ){ + u8 lockType = READ_LOCK + isWriteLock; + assert( READ_LOCK+1==WRITE_LOCK ); + assert( isWriteLock==0 || isWriteLock==1 ); + + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, iTab, lockType); + if( rc==SQLITE_OK ){ + rc = setSharedCacheTableLock(p, iTab, lockType); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. +** This function modifies the data stored as part of that entry. +** +** Only the data content may only be modified, it is not possible to +** change the length of the data stored. If this function is called with +** parameters that attempt to write past the end of the existing data, +** no modifications are made and SQLITE_CORRUPT is returned. +*/ +int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ + int rc; + assert( cursorOwnsBtShared(pCsr) ); + assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); + assert( pCsr->curFlags & BTCF_Incrblob ); + + rc = restoreCursorPosition(pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pCsr->eState!=CURSOR_REQUIRESEEK ); + if( pCsr->eState!=CURSOR_VALID ){ + return SQLITE_ABORT; + } + + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + + /* Check some assumptions: + ** (a) the cursor is open for writing, + ** (b) there is a read/write transaction open, + ** (c) the connection holds a write-lock on the table (if required), + ** (d) there are no conflicting read-locks, and + ** (e) the cursor points at a valid row of an intKey table. + */ + if( (pCsr->curFlags & BTCF_WriteFlag)==0 ){ + return SQLITE_READONLY; + } + assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 + && pCsr->pBt->inTransaction==TRANS_WRITE ); + assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); + assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); + assert( pCsr->pPage->intKey ); + + return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); +} + +/* +** Mark this cursor as an incremental blob cursor. +*/ +void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ + pCur->curFlags |= BTCF_Incrblob; + pCur->pBtree->hasIncrblobCur = 1; +} +#endif + +/* +** Set both the "read version" (single byte at byte offset 18) and +** "write version" (single byte at byte offset 19) fields in the database +** header to iVersion. +*/ +int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ + BtShared *pBt = pBtree->pBt; + int rc; /* Return code */ + + assert( iVersion==1 || iVersion==2 ); + + /* If setting the version fields to 1, do not automatically open the + ** WAL connection, even if the version fields are currently set to 2. + */ + pBt->btsFlags &= ~BTS_NO_WAL; + if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; + + rc = sqlite3BtreeBeginTrans(pBtree, 0, 0); + if( rc==SQLITE_OK ){ + u8 *aData = pBt->pPage1->aData; + if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ + rc = sqlite3BtreeBeginTrans(pBtree, 2, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; + } + } + } + } + + pBt->btsFlags &= ~BTS_NO_WAL; + return rc; +} + +/* +** Return true if the cursor has a hint specified. This routine is +** only used from within assert() statements +*/ +int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ + return (pCsr->hints & mask)!=0; +} + +/* +** Return true if the given Btree is read-only. +*/ +int sqlite3BtreeIsReadonly(Btree *p){ + return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; +} + +/* +** Return the size of the header added to each page by this module. +*/ +int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) +/* +** Return true if the Btree passed as the only argument is sharable. +*/ +int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} +#endif diff --git a/third_party/sqlite3/btree.inc b/third_party/sqlite3/btree.inc new file mode 100644 index 000000000..9c77efd09 --- /dev/null +++ b/third_party/sqlite3/btree.inc @@ -0,0 +1,394 @@ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H +/* clang-format off */ + +/* TODO: This definition is just included so other modules compile. It +** needs to be revisited. +*/ +#define SQLITE_N_BTREE_META 16 + +/* +** If defined as non-zero, auto-vacuum is enabled by default. Otherwise +** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +*/ +#ifndef SQLITE_DEFAULT_AUTOVACUUM + #define SQLITE_DEFAULT_AUTOVACUUM 0 +#endif + +#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ +#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ +#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ + +/* +** Forward declarations of structure +*/ +typedef struct Btree Btree; +typedef struct BtCursor BtCursor; +typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; + + +int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ + const char *zFilename, /* Name of database file to open */ + sqlite3 *db, /* Associated database connection */ + Btree **ppBtree, /* Return open Btree* here */ + int flags, /* Flags */ + int vfsFlags /* Flags passed through to VFS open */ +); + +/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the +** following values. +** +** NOTE: These values must match the corresponding PAGER_ values in +** pager.h. +*/ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ + +int sqlite3BtreeClose(Btree*); +int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetSpillSize(Btree*,int); +#if SQLITE_MAX_MMAP_SIZE>0 + int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif +int sqlite3BtreeSetPagerFlags(Btree*,unsigned); +int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); +int sqlite3BtreeGetPageSize(Btree*); +Pgno sqlite3BtreeMaxPageCount(Btree*,Pgno); +Pgno sqlite3BtreeLastPage(Btree*); +int sqlite3BtreeSecureDelete(Btree*,int); +int sqlite3BtreeGetRequestedReserve(Btree*); +int sqlite3BtreeGetReserveNoMutex(Btree *p); +int sqlite3BtreeSetAutoVacuum(Btree *, int); +int sqlite3BtreeGetAutoVacuum(Btree *); +int sqlite3BtreeBeginTrans(Btree*,int,int*); +int sqlite3BtreeCommitPhaseOne(Btree*, const char*); +int sqlite3BtreeCommitPhaseTwo(Btree*, int); +int sqlite3BtreeCommit(Btree*); +int sqlite3BtreeRollback(Btree*,int,int); +int sqlite3BtreeBeginStmt(Btree*,int); +int sqlite3BtreeCreateTable(Btree*, Pgno*, int flags); +int sqlite3BtreeTxnState(Btree*); +int sqlite3BtreeIsInBackup(Btree*); + +void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); +int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE +int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif + +/* Savepoints are named, nestable SQL transactions mostly implemented */ +/* in vdbe.c and pager.c See https://sqlite.org/lang_savepoint.html */ +int sqlite3BtreeSavepoint(Btree *, int, int); + +/* "Checkpoint" only refers to WAL. See https://sqlite.org/wal.html#ckpt */ +#ifndef SQLITE_OMIT_WAL + int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +const char *sqlite3BtreeGetFilename(Btree *); +const char *sqlite3BtreeGetJournalname(Btree *); +int sqlite3BtreeCopyFile(Btree *, Btree *); + +int sqlite3BtreeIncrVacuum(Btree *); + +/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) +*/ +#define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ + +int sqlite3BtreeDropTable(Btree*, int, int*); +int sqlite3BtreeClearTable(Btree*, int, int*); +int sqlite3BtreeClearTableOfCursor(BtCursor*); +int sqlite3BtreeTripAllCursors(Btree*, int, int); + +void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); +int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); + +int sqlite3BtreeNewDb(Btree *p); + +/* +** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta +** should be one of the following values. The integer values are assigned +** to constants so that the offset of the corresponding field in an +** SQLite database header may be found using the following formula: +** +** offset = 36 + (idx * 4) +** +** For example, the free-page-count field is located at byte offset 36 of +** the database file header. The incr-vacuum-flag field is located at +** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". +*/ +#define BTREE_FREE_PAGE_COUNT 0 +#define BTREE_SCHEMA_VERSION 1 +#define BTREE_FILE_FORMAT 2 +#define BTREE_DEFAULT_CACHE_SIZE 3 +#define BTREE_LARGEST_ROOT_PAGE 4 +#define BTREE_TEXT_ENCODING 5 +#define BTREE_USER_VERSION 6 +#define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ + +/* +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extentions that use +** the SQLite parser and code generator but substitute their own storage +** engine. +*/ +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** +** The BTREE_BULKLOAD flag is set on index cursors when the index is going +** to be filled with content that is already in sorted order. +** +** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or +** OP_SeekLE opcodes for a range search, but where the range of entries +** selected will all have the same key. In other words, the cursor will +** be used only for equality key searches. +** +*/ +#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ +#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ + +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just +** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and then delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +** +** The BTREE_FORDELETE flag is an optimization hint. It is not used by +** by this, the native b-tree engine of SQLite, but it is available to +** alternative storage engines that might be substituted in place of this +** b-tree system. For alternative storage engines in which a delete of +** the main table row automatically deletes corresponding index rows, +** the FORDELETE flag hint allows those alternative storage engines to +** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK +** and DELETE operations as no-ops, and any READ operation against a +** FORDELETE cursor may return a null row: 0x01 0x00. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ + +int sqlite3BtreeCursor( + Btree*, /* BTree containing table to open */ + Pgno iTable, /* Index of root page */ + int wrFlag, /* 1 for writing. 0 for read-only */ + struct KeyInfo*, /* First argument to compare function */ + BtCursor *pCursor /* Space to write cursor structure */ +); +BtCursor *sqlite3BtreeFakeValidCursor(void); +int sqlite3BtreeCursorSize(void); +void sqlite3BtreeCursorZero(BtCursor*); +void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif + +int sqlite3BtreeCloseCursor(BtCursor*); +int sqlite3BtreeMovetoUnpacked( + BtCursor*, + UnpackedRecord *pUnKey, + i64 intKey, + int bias, + int *pRes +); +int sqlite3BtreeCursorHasMoved(BtCursor*); +int sqlite3BtreeCursorRestore(BtCursor*, int*); +int sqlite3BtreeDelete(BtCursor*, u8 flags); + +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ +#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ +#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ +#define BTREE_PREFORMAT 0x80 /* Inserted data is a preformated cell */ + +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to the +** key and the pData,nData,nZero fields are uninitialized. The aMem,nMem +** fields give an array of Mem objects that are a decomposition of the key. +** The nMem field might be zero, indicating that no decomposition is available. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** The aMem,nMem fields are uninitialized for table btrees. +** +** Field usage summary: +** +** Table BTrees Index Btrees +** +** pKey always NULL encoded key +** nKey the ROWID length of pKey +** pData data not used +** aMem not used decomposed key value +** nMem not used entries in aMem +** nData length of pData not used +** nZero extra zeros after pData not used +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. */ + sqlite3_value *aMem; /* First of nMem value in the unpacked pKey */ + u16 nMem; /* Number of aMem[] value. Might be zero */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int flags, int seekResult); +int sqlite3BtreeFirst(BtCursor*, int *pRes); +int sqlite3BtreeLast(BtCursor*, int *pRes); +int sqlite3BtreeNext(BtCursor*, int flags); +int sqlite3BtreeEof(BtCursor*); +int sqlite3BtreePrevious(BtCursor*, int flags); +i64 sqlite3BtreeIntegerKey(BtCursor*); +void sqlite3BtreeCursorPin(BtCursor*); +void sqlite3BtreeCursorUnpin(BtCursor*); +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +i64 sqlite3BtreeOffset(BtCursor*); +#endif +int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); +const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +u32 sqlite3BtreePayloadSize(BtCursor*); +sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); + +char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,Pgno*aRoot,int nRoot,int,int*); +struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); + +#ifndef SQLITE_OMIT_INCRBLOB +int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); +int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); +void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif +void sqlite3BtreeClearCursor(BtCursor *); +int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); +int sqlite3BtreeIsReadonly(Btree *pBt); +int sqlite3HeaderSizeBtree(void); + +#ifdef SQLITE_DEBUG +sqlite3_uint64 sqlite3BtreeSeekCount(Btree*); +#else +# define sqlite3BtreeSeekCount(X) 0 +#endif + +#ifndef NDEBUG +int sqlite3BtreeCursorIsValid(BtCursor*); +#endif +int sqlite3BtreeCursorIsValidNN(BtCursor*); + +int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); + +#ifdef SQLITE_TEST +int sqlite3BtreeCursorInfo(BtCursor*, int*, int); +void sqlite3BtreeCursorList(Btree*); +#endif + +#ifndef SQLITE_OMIT_WAL + /* int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); */ +#endif + +int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); + +/* +** If we are not using shared cache, then there is no need to +** use mutexes to access the BtShared structures. So make the +** Enter and Leave procedures no-ops. +*/ +#ifndef SQLITE_OMIT_SHARED_CACHE + void sqlite3BtreeEnter(Btree*); + void sqlite3BtreeEnterAll(sqlite3*); + int sqlite3BtreeSharable(Btree*); + void sqlite3BtreeEnterCursor(BtCursor*); + int sqlite3BtreeConnectionCount(Btree*); +#else +# define sqlite3BtreeEnter(X) +# define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE + void sqlite3BtreeLeave(Btree*); + void sqlite3BtreeLeaveCursor(BtCursor*); + void sqlite3BtreeLeaveAll(sqlite3*); +#ifndef NDEBUG + /* These routines are used inside assert() statements only. */ + int sqlite3BtreeHoldsMutex(Btree*); + int sqlite3BtreeHoldsAllMutexes(sqlite3*); + int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); +#endif +#else + +# define sqlite3BtreeLeave(X) +# define sqlite3BtreeLeaveCursor(X) +# define sqlite3BtreeLeaveAll(X) + +# define sqlite3BtreeHoldsMutex(X) 1 +# define sqlite3BtreeHoldsAllMutexes(X) 1 +# define sqlite3SchemaMutexHeld(X,Y,Z) 1 +#endif + + +#endif /* SQLITE_BTREE_H */ diff --git a/third_party/sqlite3/btreeInt.inc b/third_party/sqlite3/btreeInt.inc new file mode 100644 index 000000000..ad36ead69 --- /dev/null +++ b/third_party/sqlite3/btreeInt.inc @@ -0,0 +1,731 @@ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an external (disk-based) database using BTrees. +** For a detailed discussion of BTrees, refer to +** +** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: +** "Sorting And Searching", pages 473-480. Addison-Wesley +** Publishing Company, Reading, Massachusetts. +** +** The basic idea is that each page of the file contains N database +** entries and N+1 pointers to subpages. +** +** ---------------------------------------------------------------- +** | Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) | +** ---------------------------------------------------------------- +** +** All of the keys on the page that Ptr(0) points to have values less +** than Key(0). All of the keys on page Ptr(1) and its subpages have +** values greater than Key(0) and less than Key(1). All of the keys +** on Ptr(N) and its subpages have values greater than Key(N-1). And +** so forth. +** +** Finding a particular key requires reading O(log(M)) pages from the +** disk where M is the number of entries in the tree. +** +** In this implementation, a single file can hold one or more separate +** BTrees. Each BTree is identified by the index of its root page. The +** key and data for any entry are combined to form the "payload". A +** fixed amount of payload can be carried directly on the database +** page. If the payload is larger than the preset amount then surplus +** bytes are stored on overflow pages. The payload for an entry +** and the preceding pointer are combined to form a "Cell". Each +** page has a small header which contains the Ptr(N) pointer and other +** information such as the size of key and data. +** +** FORMAT DETAILS +** +** The file is divided into pages. The first page is called page 1, +** the second is page 2, and so forth. A page number of zero indicates +** "no such page". The page size can be any power of 2 between 512 and 65536. +** Each page can be either a btree page, a freelist page, an overflow +** page, or a pointer-map page. +** +** The first page is always a btree page. The first 100 bytes of the first +** page contain a special header (the "file header") that describes the file. +** The format of the file header is as follows: +** +** OFFSET SIZE DESCRIPTION +** 0 16 Header string: "SQLite format 3\000" +** 16 2 Page size in bytes. (1 means 65536) +** 18 1 File format write version +** 19 1 File format read version +** 20 1 Bytes of unused space at the end of each page +** 21 1 Max embedded payload fraction (must be 64) +** 22 1 Min embedded payload fraction (must be 32) +** 23 1 Min leaf payload fraction (must be 32) +** 24 4 File change counter +** 28 4 Reserved for future use +** 32 4 First freelist page +** 36 4 Number of freelist pages in the file +** 40 60 15 4-byte meta values passed to higher layers +** +** 40 4 Schema cookie +** 44 4 File format of schema layer +** 48 4 Size of page cache +** 52 4 Largest root-page (auto/incr_vacuum) +** 56 4 1=UTF-8 2=UTF16le 3=UTF16be +** 60 4 User version +** 64 4 Incremental vacuum mode +** 68 4 Application-ID +** 72 20 unused +** 92 4 The version-valid-for number +** 96 4 SQLITE_VERSION_NUMBER +** +** All of the integer values are big-endian (most significant byte first). +** +** The file change counter is incremented when the database is changed +** This counter allows other processes to know when the file has changed +** and thus when they need to flush their cache. +** +** The max embedded payload fraction is the amount of the total usable +** space in a page that can be consumed by a single cell for standard +** B-tree (non-LEAFDATA) tables. A value of 255 means 100%. The default +** is to limit the maximum cell size so that at least 4 cells will fit +** on one page. Thus the default max embedded payload fraction is 64. +** +** If the payload for a cell is larger than the max payload, then extra +** payload is spilled to overflow pages. Once an overflow page is allocated, +** as many bytes as possible are moved into the overflow pages without letting +** the cell size drop below the min embedded payload fraction. +** +** The min leaf payload fraction is like the min embedded payload fraction +** except that it applies to leaf nodes in a LEAFDATA tree. The maximum +** payload fraction for a LEAFDATA tree is always 100% (or 255) and it +** not specified in the header. +** +** Each btree pages is divided into three sections: The header, the +** cell pointer array, and the cell content area. Page 1 also has a 100-byte +** file header that occurs before the page header. +** +** |----------------| +** | file header | 100 bytes. Page 1 only. +** |----------------| +** | page header | 8 bytes for leaves. 12 bytes for interior nodes +** |----------------| +** | cell pointer | | 2 bytes per cell. Sorted order. +** | array | | Grows downward +** | | v +** |----------------| +** | unallocated | +** | space | +** |----------------| ^ Grows upwards +** | cell content | | Arbitrary order interspersed with freeblocks. +** | area | | and free space fragments. +** |----------------| +** +** The page headers looks like this: +** +** OFFSET SIZE DESCRIPTION +** 0 1 Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf +** 1 2 byte offset to the first freeblock +** 3 2 number of cells on this page +** 5 2 first byte of the cell content area +** 7 1 number of fragmented free bytes +** 8 4 Right child (the Ptr(N) value). Omitted on leaves. +** +** The flags define the format of this btree page. The leaf flag means that +** this page has no children. The zerodata flag means that this page carries +** only keys and no data. The intkey flag means that the key is an integer +** which is stored in the key size entry of the cell header rather than in +** the payload area. +** +** The cell pointer array begins on the first byte after the page header. +** The cell pointer array contains zero or more 2-byte numbers which are +** offsets from the beginning of the page to the cell content in the cell +** content area. The cell pointers occur in sorted order. The system strives +** to keep free space after the last cell pointer so that new cells can +** be easily added without having to defragment the page. +** +** Cell content is stored at the very end of the page and grows toward the +** beginning of the page. +** +** Unused space within the cell content area is collected into a linked list of +** freeblocks. Each freeblock is at least 4 bytes in size. The byte offset +** to the first freeblock is given in the header. Freeblocks occur in +** increasing order. Because a freeblock must be at least 4 bytes in size, +** any group of 3 or fewer unused bytes in the cell content area cannot +** exist on the freeblock chain. A group of 3 or fewer free bytes is called +** a fragment. The total number of bytes in all fragments is recorded. +** in the page header at offset 7. +** +** SIZE DESCRIPTION +** 2 Byte offset of the next freeblock +** 2 Bytes in this freeblock +** +** Cells are of variable length. Cells are stored in the cell content area at +** the end of the page. Pointers to the cells are in the cell pointer array +** that immediately follows the page header. Cells is not necessarily +** contiguous or in order, but cell pointers are contiguous and in order. +** +** Cell content makes use of variable length integers. A variable +** length integer is 1 to 9 bytes where the lower 7 bits of each +** byte are used. The integer consists of all bytes that have bit 8 set and +** the first byte with bit 8 clear. The most significant byte of the integer +** appears first. A variable-length integer may not be more than 9 bytes long. +** As a special case, all 8 bytes of the 9th byte are used as data. This +** allows a 64-bit integer to be encoded in 9 bytes. +** +** 0x00 becomes 0x00000000 +** 0x7f becomes 0x0000007f +** 0x81 0x00 becomes 0x00000080 +** 0x82 0x00 becomes 0x00000100 +** 0x80 0x7f becomes 0x0000007f +** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 +** +** Variable length integers are used for rowids and to hold the number of +** bytes of key and data in a btree cell. +** +** The content of a cell looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of the left child. Omitted if leaf flag is set. +** var Number of bytes of data. Omitted if the zerodata flag is set. +** var Number of bytes of key. Or the key itself if intkey flag is set. +** * Payload +** 4 First page of the overflow chain. Omitted if no overflow +** +** Overflow pages form a linked list. Each page except the last is completely +** filled with data (pagesize - 4 bytes). The last page can have as little +** as 1 byte of data. +** +** SIZE DESCRIPTION +** 4 Page number of next overflow page +** * Data +** +** Freelist pages come in two subtypes: trunk pages and leaf pages. The +** file header points to the first in a linked list of trunk page. Each trunk +** page points to multiple leaf pages. The content of a leaf page is +** unspecified. A trunk page looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of next trunk page +** 4 Number of leaf pointers on this page +** * zero or more pages numbers of leaves +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +/* The following value is the maximum cell size assuming a maximum page +** size give above. +*/ +#define MX_CELL_SIZE(pBt) ((int)(pBt->pageSize-8)) + +/* The maximum number of cells on a single page of the database. This +** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself +** plus 2 bytes for the index to the cell in the page header). Such +** small cells will be rare, but they are possible. +*/ +#define MX_CELL(pBt) ((pBt->pageSize-8)/6) + +/* Forward declarations */ +typedef struct MemPage MemPage; +typedef struct BtLock BtLock; +typedef struct CellInfo CellInfo; + +/* +** This is a magic string that appears at the beginning of every +** SQLite database in order to identify the file as a real database. +** +** You can change this value at compile-time by specifying a +** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The +** header must be exactly 16 bytes including the zero-terminator so +** the string itself should be 15 characters long. If you change +** the header, then your custom library will not be able to read +** databases generated by the standard tools and the standard tools +** will not be able to read databases created by your custom library. +*/ +#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ +# define SQLITE_FILE_HEADER "SQLite format 3" +#endif + +/* +** Page type flags. An ORed combination of these flags appear as the +** first byte of on-disk image of every BTree page. +*/ +#define PTF_INTKEY 0x01 +#define PTF_ZERODATA 0x02 +#define PTF_LEAFDATA 0x04 +#define PTF_LEAF 0x08 + +/* +** An instance of this object stores information about each a single database +** page that has been loaded into memory. The information in this object +** is derived from the raw on-disk page content. +** +** As each database page is loaded into memory, the pager allocats an +** instance of this object and zeros the first 8 bytes. (This is the +** "extra" information associated with each page of the pager.) +** +** Access to all fields of this structure is controlled by the mutex +** stored in MemPage.pBt->mutex. +*/ +struct MemPage { + u8 isInit; /* True if previously initialized. MUST BE FIRST! */ + u8 bBusy; /* Prevent endless loops on corrupt database files */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + Pgno pgno; /* Page number for this page */ + /* Only the first 8 bytes (above) are zeroed by pager.c when a new page + ** is allocated. All fields that follow must be initialized before use */ + u8 leaf; /* True if a leaf page */ + u8 hdrOffset; /* 100 for page 1. 0 otherwise */ + u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ + u8 max1bytePayload; /* min(maxLocal,127) */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ + u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ + u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ + u16 cellOffset; /* Index in aData of first cell pointer */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ + u16 nCell; /* Number of cells on this page, local and ovfl */ + u16 maskPage; /* Mask for page offset */ + u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th + ** non-overflow cell */ + u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ + BtShared *pBt; /* Pointer to BtShared that this page is part of */ + u8 *aData; /* Pointer to disk image of the page data */ + u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aCellIdx; /* The cell index area */ + u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ + DbPage *pDbPage; /* Pager page handle */ + u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ + void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ +}; + +/* +** A linked list of the following structures is stored at BtShared.pLock. +** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor +** is opened on the table with root page BtShared.iTable. Locks are removed +** from this list when a transaction is committed or rolled back, or when +** a btree handle is closed. +*/ +struct BtLock { + Btree *pBtree; /* Btree handle holding this lock */ + Pgno iTable; /* Root page of table */ + u8 eLock; /* READ_LOCK or WRITE_LOCK */ + BtLock *pNext; /* Next in BtShared.pLock list */ +}; + +/* Candidate values for BtLock.eLock */ +#define READ_LOCK 1 +#define WRITE_LOCK 2 + +/* A Btree handle +** +** A database connection contains a pointer to an instance of +** this object for every database file that it has open. This structure +** is opaque to the database connection. The database connection cannot +** see the internals of this structure and only deals with pointers to +** this structure. +** +** For some database files, the same underlying database cache might be +** shared between multiple connections. In that case, each connection +** has it own instance of this object. But each instance of this object +** points to the same BtShared object. The database cache and the +** schema associated with the database file are all contained within +** the BtShared object. +** +** All fields in this structure are accessed under sqlite3.mutex. +** The pBt pointer itself may not be changed while there exists cursors +** in the referenced BtShared that point back to this Btree since those +** cursors have to go through this Btree to find their BtShared and +** they often do so without holding sqlite3.mutex. +*/ +struct Btree { + sqlite3 *db; /* The database connection holding this btree */ + BtShared *pBt; /* Sharable content of this btree */ + u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + u8 sharable; /* True if we can share pBt with another db */ + u8 locked; /* True if db currently has pBt locked */ + u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ + int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + int nBackup; /* Number of backup operations reading this btree */ + u32 iBDataVersion; /* Combines with pBt->pPager->iDataVersion */ + Btree *pNext; /* List of other sharable Btrees from the same db */ + Btree *pPrev; /* Back pointer of the same list */ +#ifdef SQLITE_DEBUG + u64 nSeek; /* Calls to sqlite3BtreeMovetoUnpacked() */ +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE + BtLock lock; /* Object used to lock page 1 */ +#endif +}; + +/* +** Btree.inTrans may take one of the following values. +** +** If the shared-data extension is enabled, there may be multiple users +** of the Btree structure. At most one of these may open a write transaction, +** but any number may have active read transactions. +** +** These values must match SQLITE_TXN_NONE, SQLITE_TXN_READ, and +** SQLITE_TXN_WRITE +*/ +#define TRANS_NONE 0 +#define TRANS_READ 1 +#define TRANS_WRITE 2 + +#if TRANS_NONE!=SQLITE_TXN_NONE +# error wrong numeric code for no-transaction +#endif +#if TRANS_READ!=SQLITE_TXN_READ +# error wrong numeric code for read-transaction +#endif +#if TRANS_WRITE!=SQLITE_TXN_WRITE +# error wrong numeric code for write-transaction +#endif + + +/* +** An instance of this object represents a single database file. +** +** A single database file can be in use at the same time by two +** or more database connections. When two or more connections are +** sharing the same database file, each connection has it own +** private Btree object for the file and each of those Btrees points +** to this one BtShared object. BtShared.nRef is the number of +** connections currently sharing this database file. +** +** Fields in this structure are accessed under the BtShared.mutex +** mutex, except for nRef and pNext which are accessed under the +** global SQLITE_MUTEX_STATIC_MAIN mutex. The pPager field +** may not be modified once it is initially set as long as nRef>0. +** The pSchema field may be set once under BtShared.mutex and +** thereafter is unchanged as long as nRef>0. +** +** isPending: +** +** If a BtShared client fails to obtain a write-lock on a database +** table (because there exists one or more read-locks on the table), +** the shared-cache enters 'pending-lock' state and isPending is +** set to true. +** +** The shared-cache leaves the 'pending lock' state when either of +** the following occur: +** +** 1) The current writer (BtShared.pWriter) concludes its transaction, OR +** 2) The number of locks held by other connections drops to zero. +** +** while in the 'pending-lock' state, no connection may start a new +** transaction. +** +** This feature is included to help prevent writer-starvation. +*/ +struct BtShared { + Pager *pPager; /* The page cache */ + sqlite3 *db; /* Database connection currently using this Btree */ + BtCursor *pCursor; /* A list of all open cursors */ + MemPage *pPage1; /* First page of the database */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ +#ifndef SQLITE_OMIT_AUTOVACUUM + u8 autoVacuum; /* True if auto-vacuum is enabled */ + u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ +#endif + u8 inTransaction; /* Transaction state */ + u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u8 nReserveWanted; /* Desired number of extra bytes per page */ + u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ + u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ + u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ + int nTransaction; /* Number of open transactions (read + write) */ + u32 nPage; /* Number of pages in the database */ + void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ + void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ + sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */ + Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nRef; /* Number of references to this structure */ + BtShared *pNext; /* Next on a list of sharable BtShared structs */ + BtLock *pLock; /* List of locks held on this shared-btree struct */ + Btree *pWriter; /* Btree with currently open write transaction */ +#endif + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ + int nPreformatSize; /* Size of last cell written by TransferRow() */ +}; + +/* +** Allowed values for BtShared.btsFlags +*/ +#define BTS_READ_ONLY 0x0001 /* Underlying file is readonly */ +#define BTS_PAGESIZE_FIXED 0x0002 /* Page size can no longer be changed */ +#define BTS_SECURE_DELETE 0x0004 /* PRAGMA secure_delete is enabled */ +#define BTS_OVERWRITE 0x0008 /* Overwrite deleted content with zeros */ +#define BTS_FAST_SECURE 0x000c /* Combination of the previous two */ +#define BTS_INITIALLY_EMPTY 0x0010 /* Database was empty at trans start */ +#define BTS_NO_WAL 0x0020 /* Do not open write-ahead-log files */ +#define BTS_EXCLUSIVE 0x0040 /* pWriter has an exclusive lock */ +#define BTS_PENDING 0x0080 /* Waiting for read-locks to clear */ + +/* +** An instance of the following structure is used to hold information +** about a cell. The parseCellPtr() function fills in this structure +** based on information extract from the raw disk page. +*/ +struct CellInfo { + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ + u16 nSize; /* Size of the cell content on the main b-tree page */ +}; + +/* +** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than +** this will be declared corrupt. This value is calculated based on a +** maximum database size of 2^31 pages a minimum fanout of 2 for a +** root-node and 3 for all other internal nodes. +** +** If a tree that appears to be taller than this is encountered, it is +** assumed that the database is corrupt. +*/ +#define BTCURSOR_MAX_DEPTH 20 + +/* +** A cursor is a pointer to a particular entry within a particular +** b-tree within a database file. +** +** The entry is identified by its MemPage and the index in +** MemPage.aCell[] of the entry. +** +** A single database file can be shared by two more database connections, +** but cursors cannot be shared. Each cursor is associated with a +** particular database connection identified BtCursor.pBtree.db. +** +** Fields in this structure are accessed under the BtShared.mutex +** found at self->pBt->mutex. +** +** skipNext meaning: +** The meaning of skipNext depends on the value of eState: +** +** eState Meaning of skipNext +** VALID skipNext is meaningless and is ignored +** INVALID skipNext is meaningless and is ignored +** SKIPNEXT sqlite3BtreeNext() is a no-op if skipNext>0 and +** sqlite3BtreePrevious() is no-op if skipNext<0. +** REQUIRESEEK restoreCursorPosition() restores the cursor to +** eState=SKIPNEXT if skipNext!=0 +** FAULT skipNext holds the cursor fault error code. +*/ +struct BtCursor { + u8 eState; /* One of the CURSOR_XXX constants (see below) */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ + u8 hints; /* As configured by CursorSetHints() */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ + Btree *pBtree; /* The Btree to which this cursor belongs */ + Pgno *aOverflow; /* Cache of overflow page locations */ + void *pKey; /* Saved key that was cursor last known position */ + /* All fields above are zeroed when the cursor is allocated. See + ** sqlite3BtreeCursorZero(). Fields that follow must be manually + ** initialized. */ +#define BTCURSOR_FIRST_UNINIT pBt /* Name of first uninitialized field */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext; /* Forms a linked list of all cursors */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + Pgno pgnoRoot; /* The root page of this tree */ + i8 iPage; /* Index of current page in apPage */ + u8 curIntKey; /* Value of apPage[0]->intKey */ + u16 ix; /* Current index for apPage[iPage] */ + u16 aiIdx[BTCURSOR_MAX_DEPTH-1]; /* Current index in apPage[i] */ + struct KeyInfo *pKeyInfo; /* Arg passed to comparison function */ + MemPage *pPage; /* Current page */ + MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */ +}; + +/* +** Legal values for BtCursor.curFlags +*/ +#define BTCF_WriteFlag 0x01 /* True if a write cursor */ +#define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ +#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ +#define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ + +/* +** Potential values for BtCursor.eState. +** +** CURSOR_INVALID: +** Cursor does not point to a valid entry. This can happen (for example) +** because the table is empty or because BtreeCursorFirst() has not been +** called. +** +** CURSOR_VALID: +** Cursor points to a valid entry. getPayload() etc. may be called. +** +** CURSOR_SKIPNEXT: +** Cursor is valid except that the Cursor.skipNext field is non-zero +** indicating that the next sqlite3BtreeNext() or sqlite3BtreePrevious() +** operation should be a no-op. +** +** CURSOR_REQUIRESEEK: +** The table that this cursor was opened on still exists, but has been +** modified since the cursor was last used. The cursor position is saved +** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in +** this state, restoreCursorPosition() can be called to attempt to +** seek the cursor to the saved position. +** +** CURSOR_FAULT: +** An unrecoverable error (an I/O error or a malloc failure) has occurred +** on a different connection that shares the BtShared cache with this +** cursor. The error has left the cache in an inconsistent state. +** Do nothing else with this cursor. Any attempt to use the cursor +** should return the error code stored in BtCursor.skipNext +*/ +#define CURSOR_VALID 0 +#define CURSOR_INVALID 1 +#define CURSOR_SKIPNEXT 2 +#define CURSOR_REQUIRESEEK 3 +#define CURSOR_FAULT 4 + +/* +** The database page the PENDING_BYTE occupies. This page is never used. +*/ +# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) + +/* +** These macros define the location of the pointer-map entry for a +** database page. The first argument to each is the number of usable +** bytes on each page of the database (often 1024). The second is the +** page number to look up in the pointer map. +** +** PTRMAP_PAGENO returns the database page number of the pointer-map +** page that stores the required pointer. PTRMAP_PTROFFSET returns +** the offset of the requested map entry. +** +** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, +** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be +** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements +** this test. +*/ +#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) +#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) +#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) + +/* +** The pointer map is a lookup table that identifies the parent page for +** each child page in the database file. The parent page is the page that +** contains a pointer to the child. Every page in the database contains +** 0 or 1 parent pages. (In this context 'database page' refers +** to any page that is not part of the pointer map itself.) Each pointer map +** entry consists of a single byte 'type' and a 4 byte parent page number. +** The PTRMAP_XXX identifiers below are the valid types. +** +** The purpose of the pointer map is to facility moving pages from one +** position in the file to another as part of autovacuum. When a page +** is moved, the pointer in its parent must be updated to point to the +** new location. The pointer map is used to locate the parent page quickly. +** +** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not +** used in this case. +** +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** is not used in this case. +** +** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** overflow pages. The page number identifies the page that +** contains the cell with a pointer to this overflow page. +** +** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of +** overflow pages. The page-number identifies the previous +** page in the overflow page list. +** +** PTRMAP_BTREE: The database page is a non-root btree page. The page number +** identifies the parent page in the btree. +*/ +#define PTRMAP_ROOTPAGE 1 +#define PTRMAP_FREEPAGE 2 +#define PTRMAP_OVERFLOW1 3 +#define PTRMAP_OVERFLOW2 4 +#define PTRMAP_BTREE 5 + +/* A bunch of assert() statements to check the transaction state variables +** of handle p (type Btree*) are internally consistent. +*/ +#define btreeIntegrity(p) \ + assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ + assert( p->pBt->inTransaction>=p->inTrans ); + + +/* +** The ISAUTOVACUUM macro is used within balance_nonroot() to determine +** if the database supports auto-vacuum or not. Because it is used +** within an expression that is an argument to another macro +** (sqliteMallocRaw), it is not possible to use conditional compilation. +** So, this macro is defined instead. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM (pBt->autoVacuum) +#else +#define ISAUTOVACUUM 0 +#endif + + +/* +** This structure is passed around through all the sanity checking routines +** in order to keep track of some global state information. +** +** The aRef[] array is allocated so that there is 1 bit for each page in +** the database. As the integrity-check proceeds, for each page used in +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which +** indicate corruption). +*/ +typedef struct IntegrityCk IntegrityCk; +struct IntegrityCk { + BtShared *pBt; /* The tree being checked out */ + Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ + u8 *aPgRef; /* 1 bit per page in the db (see above) */ + Pgno nPage; /* Number of pages in the database */ + int mxErr; /* Stop accumulating errors when this reaches zero */ + int nErr; /* Number of messages written to zErrMsg so far */ + int bOomFault; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + Pgno v1; /* Value for first %u substitution in zPfx */ + int v2; /* Value for second %d substitution in zPfx */ + StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ + sqlite3 *db; /* Database connection running the check */ +}; + +/* +** Routines to read or write a two- and four-byte big-endian integer values. +*/ +#define get2byte(x) ((x)[0]<<8 | (x)[1]) +#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) +#define get4byte sqlite3Get4byte +#define put4byte sqlite3Put4byte + +/* +** get2byteAligned(), unlike get2byte(), requires that its argument point to a +** two-byte aligned address. get2bytea() is only used for accessing the +** cell addresses in a btree header. +*/ +#if SQLITE_BYTEORDER==4321 +# define get2byteAligned(x) (*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 +# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) +#else +# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) +#endif diff --git a/third_party/sqlite3/build.c b/third_party/sqlite3/build.c new file mode 100644 index 000000000..288864ab5 --- /dev/null +++ b/third_party/sqlite3/build.c @@ -0,0 +1,5335 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the SQLite parser +** when syntax rules are reduced. The routines in this file handle the +** following kinds of SQL syntax: +** +** CREATE TABLE +** DROP TABLE +** CREATE INDEX +** DROP INDEX +** creating ID lists +** BEGIN TRANSACTION +** COMMIT +** ROLLBACK +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** The TableLock structure is only used by the sqlite3TableLock() and +** codeTableLocks() functions. +*/ +struct TableLock { + int iDb; /* The database containing the table to be locked */ + Pgno iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zLockName; /* Name of the table */ +}; + +/* +** Record the fact that we want to lock a table at run-time. +** +** The table to be locked has root page iTab and is found in database iDb. +** A read or a write lock can be taken depending on isWritelock. +** +** This routine just records the fact that the lock is desired. The +** code to make the lock occur is generated by a later call to +** codeTableLocks() which occurs during sqlite3FinishCoding(). +*/ +void sqlite3TableLock( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + Pgno iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + Parse *pToplevel; + int i; + int nBytes; + TableLock *p; + assert( iDb>=0 ); + + if( iDb==1 ) return; + if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; + pToplevel = sqlite3ParseToplevel(pParse); + for(i=0; inTableLock; i++){ + p = &pToplevel->aTableLock[i]; + if( p->iDb==iDb && p->iTab==iTab ){ + p->isWriteLock = (p->isWriteLock || isWriteLock); + return; + } + } + + nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); + pToplevel->aTableLock = + sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); + if( pToplevel->aTableLock ){ + p = &pToplevel->aTableLock[pToplevel->nTableLock++]; + p->iDb = iDb; + p->iTab = iTab; + p->isWriteLock = isWriteLock; + p->zLockName = zName; + }else{ + pToplevel->nTableLock = 0; + sqlite3OomFault(pToplevel->db); + } +} + +/* +** Code an OP_TableLock instruction for each table locked by the +** statement (configured by calls to sqlite3TableLock()). +*/ +static void codeTableLocks(Parse *pParse){ + int i; + Vdbe *pVdbe = pParse->pVdbe; + assert( pVdbe!=0 ); + + for(i=0; inTableLock; i++){ + TableLock *p = &pParse->aTableLock[i]; + int p1 = p->iDb; + sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, + p->zLockName, P4_STATIC); + } +} +#else + #define codeTableLocks(x) +#endif + +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; ipToplevel==0 ); + db = pParse->db; + if( pParse->nested ) return; + if( db->mallocFailed || pParse->nErr ){ + if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + return; + } + + /* Begin by generating some termination code at the end of the + ** vdbe program + */ + v = pParse->pVdbe; + if( v==0 ){ + if( db->init.busy ){ + pParse->rc = SQLITE_DONE; + return; + } + v = sqlite3GetVdbe(pParse); + if( v==0 ) pParse->rc = SQLITE_ERROR; + } + assert( !pParse->isMultiWrite + || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); + if( v ){ + if( pParse->bReturning ){ + Returning *pReturning = pParse->u1.pReturning; + int addrRewind; + int i; + int reg; + + addrRewind = + sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); + VdbeCoverage(v); + reg = pReturning->iRetReg; + for(i=0; inRetCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i); + sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrRewind); + } + sqlite3VdbeAddOp0(v, OP_Halt); + +#if SQLITE_USER_AUTHENTICATION + if( pParse->nTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + return; + } + } +#endif + + /* The cookie mask contains one bit for each database file open. + ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are + ** set for each database that is used. Generate code to start a + ** transaction on each used database and to verify the schema cookie + ** on each used database. + */ + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ + int iDb, i; + assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + for(iDb=0; iDbnDb; iDb++){ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; +#endif + + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); + + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); + + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + int iReg = pEL->a[i].u.iConstExprReg; + if( iReg>0 ){ + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); + } + } + } + + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); + } + } + + /* Get the VDBE program ready for execution + */ + if( v && pParse->nErr==0 && !db->mallocFailed ){ + /* A minimum of one cursor is required if autoincrement is used + * See ticket [a696379c1f08866] */ + assert( pParse->pAinc==0 || pParse->nTab>0 ); + sqlite3VdbeMakeReady(v, pParse); + pParse->rc = SQLITE_DONE; + }else{ + pParse->rc = SQLITE_ERROR; + } +} + +/* +** Run the parser and code generator recursively in order to generate +** code for the SQL statement given onto the end of the pParse context +** currently under construction. When the parser is run recursively +** this way, the final OP_Halt is not appended and other initialization +** and finalization steps are omitted because those are handling by the +** outermost parser. +** +** Not everything is nestable. This facility is designed to permit +** INSERT, UPDATE, and DELETE operations against the schema table. Use +** care if you decide to try to use this routine for some other purposes. +*/ +void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ + va_list ap; + char *zSql; + char *zErrMsg = 0; + sqlite3 *db = pParse->db; + char saveBuf[PARSE_TAIL_SZ]; + + if( pParse->nErr ) return; + assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ + va_start(ap, zFormat); + zSql = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( zSql==0 ){ + /* This can result either from an OOM or because the formatted string + ** exceeds SQLITE_LIMIT_LENGTH. In the latter case, we need to set + ** an error */ + if( !db->mallocFailed ) pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; + return; + } + pParse->nested++; + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3DbFree(db, zErrMsg); + sqlite3DbFree(db, zSql); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); + pParse->nested--; +} + +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the table and the +** first matching table is returned. (No checking for duplicate table +** names is done.) The search order is TEMP first, then MAIN, then any +** auxiliary databases added using the ATTACH command. +** +** See also sqlite3LocateTable(). +*/ +Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ + Table *p = 0; + int i; + + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( db->auth.authLevelnDb; i++){ + if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; + } + if( i>=db->nDb ){ + /* No match against the official names. But always match "main" + ** to schema 0 as a legacy fallback. */ + if( sqlite3StrICmp(zDatabase,"main")==0 ){ + i = 0; + }else{ + return 0; + } + } + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( i==1 ){ + if( sqlite3StrICmp(zName+7, &ALT_TEMP_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &DFLT_SCHEMA_TABLE[7])==0 + ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + DFLT_TEMP_SCHEMA_TABLE); + } + }else{ + if( sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, + DFLT_SCHEMA_TABLE); + } + } + } + }else{ + /* Match against TEMP first */ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, zName); + if( p ) return p; + /* The main database is second */ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, zName); + if( p ) return p; + /* Attached databases are in order of attachment */ + for(i=2; inDb; i++){ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p ) break; + } + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, DFLT_SCHEMA_TABLE); + }else if( sqlite3StrICmp(zName+7, &ALT_TEMP_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + DFLT_TEMP_SCHEMA_TABLE); + } + } + } + return p; +} + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. Also leave an +** error message in pParse->zErrMsg. +** +** The difference between this routine and sqlite3FindTable() is that this +** routine leaves an error message in pParse->zErrMsg where +** sqlite3FindTable() does not. +*/ +Table *sqlite3LocateTable( + Parse *pParse, /* context in which to report errors */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ + const char *zName, /* Name of the table we are looking for */ + const char *zDbase /* Name of the database. Might be NULL */ +){ + Table *p; + sqlite3 *db = pParse->db; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 + && SQLITE_OK!=sqlite3ReadSchema(pParse) + ){ + return 0; + } + + p = sqlite3FindTable(db, zName, zDbase); + if( p==0 ){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + if( pParse->disableVtab==0 ){ + Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); + if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ + pMod = sqlite3PragmaVtabRegister(db, zName); + } + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } + } +#endif + if( flags & LOCATE_NOERR ) return 0; + pParse->checkSchema = 1; + }else if( IsVirtual(p) && pParse->disableVtab ){ + p = 0; + } + + if( p==0 ){ + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + } + + return p; +} + +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +Table *sqlite3LocateTableItem( + Parse *pParse, + u32 flags, + SrcItem *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zDbSName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, flags, p->zName, zDb); +} + +/* +** Locate the in-memory structure that describes +** a particular index given the name of that index +** and the name of the database that contains the index. +** Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the +** table and the first matching index is returned. (No checking +** for duplicate index names is done.) The search order is +** TEMP first, then MAIN, then any auxiliary databases added +** using the ATTACH command. +*/ +Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ + Index *p = 0; + int i; + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + Schema *pSchema = db->aDb[j].pSchema; + assert( pSchema ); + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&pSchema->idxHash, zName); + if( p ) break; + } + return p; +} + +/* +** Reclaim the memory used by an index +*/ +void sqlite3FreeIndex(sqlite3 *db, Index *p){ +#ifndef SQLITE_OMIT_ANALYZE + sqlite3DeleteIndexSamples(db, p); +#endif + sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); + sqlite3DbFree(db, p->zColAff); + if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(p->aiRowEst); +#endif + sqlite3DbFree(db, p); +} + +/* +** For the index called zIdxName which is found in the database iDb, +** unlike that index from its Table then remove the index from +** the index hash table and free all memory structures associated +** with the index. +*/ +void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ + Index *pIndex; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &db->aDb[iDb].pSchema->idxHash; + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); + if( ALWAYS(pIndex) ){ + if( pIndex->pTable->pIndex==pIndex ){ + pIndex->pTable->pIndex = pIndex->pNext; + }else{ + Index *p; + /* Justification of ALWAYS(); The index must be on the list of + ** indices. */ + p = pIndex->pTable->pIndex; + while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } + if( ALWAYS(p && p->pNext==pIndex) ){ + p->pNext = pIndex->pNext; + } + } + sqlite3FreeIndex(db, pIndex); + } + db->mDbFlags |= DBFLAG_SchemaChange; +} + +/* +** Look through the list of open database files in db->aDb[] and if +** any have been closed, remove them from the list. Reallocate the +** db->aDb[] structure to a smaller size, if possible. +** +** Entry 0 (the "main" database) and entry 1 (the "temp" database) +** are never candidates for being collapsed. +*/ +void sqlite3CollapseDatabaseArray(sqlite3 *db){ + int i, j; + for(i=j=2; inDb; i++){ + struct Db *pDb = &db->aDb[i]; + if( pDb->pBt==0 ){ + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; + continue; + } + if( jaDb[j] = db->aDb[i]; + } + j++; + } + db->nDb = j; + if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ + memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); + sqlite3DbFree(db, db->aDb); + db->aDb = db->aDbStatic; + } +} + +/* +** Reset the schema for the database at index iDb. Also reset the +** TEMP schema. The reset is deferred if db->nSchemaLock is not zero. +** Deferred resets may be run by calling with iDb<0. +*/ +void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ + int i; + assert( iDbnDb ); + + if( iDb>=0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + DbSetProperty(db, iDb, DB_ResetWanted); + DbSetProperty(db, 1, DB_ResetWanted); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + } + + if( db->nSchemaLock==0 ){ + for(i=0; inDb; i++){ + if( DbHasProperty(db, i, DB_ResetWanted) ){ + sqlite3SchemaClear(db->aDb[i].pSchema); + } + } + } +} + +/* +** Erase all schema information from all attached databases (including +** "main" and "temp") for a single database connection. +*/ +void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pSchema ){ + if( db->nSchemaLock==0 ){ + sqlite3SchemaClear(pDb->pSchema); + }else{ + DbSetProperty(db, i, DB_ResetWanted); + } + } + } + db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk); + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); + if( db->nSchemaLock==0 ){ + sqlite3CollapseDatabaseArray(db); + } +} + +/* +** This routine is called when a commit occurs. +*/ +void sqlite3CommitInternalChanges(sqlite3 *db){ + db->mDbFlags &= ~DBFLAG_SchemaChange; +} + +/* +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). +*/ +void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ + int i; + Column *pCol; + assert( pTable!=0 ); + if( (pCol = pTable->aCol)!=0 ){ + for(i=0; inCol; i++, pCol++){ + assert( pCol->zName==0 || pCol->hName==sqlite3StrIHash(pCol->zName) ); + sqlite3DbFree(db, pCol->zName); + sqlite3ExprDelete(db, pCol->pDflt); + sqlite3DbFree(db, pCol->zColl); + } + sqlite3DbFree(db, pTable->aCol); + } +} + +/* +** Remove the memory data structures associated with the given +** Table. No changes are made to disk by this routine. +** +** This routine just deletes the data structure. It does not unlink +** the table data structure from the hash table. But it does destroy +** memory structures of the indices and foreign keys associated with +** the table. +** +** The db parameter is optional. It is needed if the Table object +** contains lookaside memory. (Table objects in the schema do not use +** lookaside memory, but some ephemeral Table objects do.) Or the +** db parameter can be used with db->pnBytesFreed to measure the memory +** used by the Table object. +*/ +static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ + Index *pIndex, *pNext; + +#ifdef SQLITE_DEBUG + /* Record the number of outstanding lookaside allocations in schema Tables + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. + ** + ** If malloc has already failed, it may be that it failed while allocating + ** a Table object that was going to be marked ephemeral. So do not check + ** that no lookaside memory is used in this case either. */ + int nLookaside = 0; + if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + nLookaside = sqlite3LookasideUsed(db, 0); + } +#endif + + /* Delete all indices associated with this table. */ + for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ + pNext = pIndex->pNext; + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, 0 + ); + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + assert( pOld==pIndex || pOld==0 ); + } + sqlite3FreeIndex(db, pIndex); + } + + /* Delete any foreign keys attached to this table. */ + sqlite3FkDelete(db, pTable); + + /* Delete the Table structure itself. + */ + sqlite3DeleteColumnNames(db, pTable); + sqlite3DbFree(db, pTable->zName); + sqlite3DbFree(db, pTable->zColAff); + sqlite3SelectDelete(db, pTable->pSelect); + sqlite3ExprListDelete(db, pTable->pCheck); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); +#endif + sqlite3DbFree(db, pTable); + + /* Verify that no lookaside memory was used by schema tables */ + assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) ); +} +void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + deleteTable(db, pTable); +} + + +/* +** Unlink the given table from the hash tables and the delete the +** table structure with all its indices and foreign keys. +*/ +void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ + Table *p; + Db *pDb; + + assert( db!=0 ); + assert( iDb>=0 && iDbnDb ); + assert( zTabName ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ + pDb = &db->aDb[iDb]; + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); + sqlite3DeleteTable(db, p); + db->mDbFlags |= DBFLAG_SchemaChange; +} + +/* +** Given a token, return a string that consists of the text of that +** token. Space to hold the returned string +** is obtained from sqliteMalloc() and must be freed by the calling +** function. +** +** Any quotation marks (ex: "name", 'name', [name], or `name`) that +** surround the body of the token are removed. +** +** Tokens are often just pointers into the original SQL text and so +** are not \000 terminated and are not persistent. The returned string +** is \000 terminated and is persistent. +*/ +char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ + char *zName; + if( pName ){ + zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); + sqlite3Dequote(zName); + }else{ + zName = 0; + } + return zName; +} + +/* +** Open the sqlite_schema table stored in database number iDb for +** writing. The table is opened using cursor 0. +*/ +void sqlite3OpenSchemaTable(Parse *p, int iDb){ + Vdbe *v = sqlite3GetVdbe(p); + sqlite3TableLock(p, iDb, SCHEMA_ROOT, 1, DFLT_SCHEMA_TABLE); + sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, SCHEMA_ROOT, iDb, 5); + if( p->nTab==0 ){ + p->nTab = 1; + } +} + +/* +** Parameter zName points to a nul-terminated buffer containing the name +** of a database ("main", "temp" or the name of an attached db). This +** function returns the index of the named database in db->aDb[], or +** -1 if the named db cannot be found. +*/ +int sqlite3FindDbName(sqlite3 *db, const char *zName){ + int i = -1; /* Database number */ + if( zName ){ + Db *pDb; + for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ + if( 0==sqlite3_stricmp(pDb->zDbSName, zName) ) break; + /* "main" is always an acceptable alias for the primary database + ** even if it has been renamed using SQLITE_DBCONFIG_MAINDBNAME. */ + if( i==0 && 0==sqlite3_stricmp("main", zName) ) break; + } + } + return i; +} + +/* +** The token *pName contains the name of a database (either "main" or +** "temp" or the name of an attached db). This routine returns the +** index of the named database in db->aDb[], or -1 if the named db +** does not exist. +*/ +int sqlite3FindDb(sqlite3 *db, Token *pName){ + int i; /* Database number */ + char *zName; /* Name we are searching for */ + zName = sqlite3NameFromToken(db, pName); + i = sqlite3FindDbName(db, zName); + sqlite3DbFree(db, zName); + return i; +} + +/* The table or view or trigger name is passed to this routine via tokens +** pName1 and pName2. If the table name was fully qualified, for example: +** +** CREATE TABLE xxx.yyy (...); +** +** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if +** the table name is not fully qualified, i.e.: +** +** CREATE TABLE yyy(...); +** +** Then pName1 is set to "yyy" and pName2 is "". +** +** This routine sets the *ppUnqual pointer to point at the token (pName1 or +** pName2) that stores the unqualified table name. The index of the +** database "xxx" is returned. +*/ +int sqlite3TwoPartName( + Parse *pParse, /* Parsing and code generating context */ + Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */ + Token *pName2, /* The "yyy" in the name "xxx.yyy" */ + Token **pUnqual /* Write the unqualified object name here */ +){ + int iDb; /* Database holding the object */ + sqlite3 *db = pParse->db; + + assert( pName2!=0 ); + if( pName2->n>0 ){ + if( db->init.busy ) { + sqlite3ErrorMsg(pParse, "corrupt database"); + return -1; + } + *pUnqual = pName2; + iDb = sqlite3FindDb(db, pName1); + if( iDb<0 ){ + sqlite3ErrorMsg(pParse, "unknown database %T", pName1); + return -1; + } + }else{ + assert( db->init.iDb==0 || db->init.busy || IN_RENAME_OBJECT + || (db->mDbFlags & DBFLAG_Vacuum)!=0); + iDb = db->init.iDb; + *pUnqual = pName1; + } + return iDb; +} + +/* +** True if PRAGMA writable_schema is ON +*/ +int sqlite3WritableSchema(sqlite3 *db){ + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==0 ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_WriteSchema ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_Defensive ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + (SQLITE_WriteSchema|SQLITE_Defensive) ); + return (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==SQLITE_WriteSchema; +} + +/* +** This routine is used to check if the UTF-8 string zName is a legal +** unqualified name for a new schema object (table, index, view or +** trigger). All names are legal except those that begin with the string +** "sqlite_" (in upper, lower or mixed case). This portion of the namespace +** is reserved for internal use. +** +** When parsing the sqlite_schema table, this routine also checks to +** make sure the "type", "name", and "tbl_name" columns are consistent +** with the SQL. +*/ +int sqlite3CheckObjectName( + Parse *pParse, /* Parsing context */ + const char *zName, /* Name of the object to check */ + const char *zType, /* Type of this object */ + const char *zTblName /* Parent table name for triggers and indexes */ +){ + sqlite3 *db = pParse->db; + if( sqlite3WritableSchema(db) + || db->init.imposterTable + || !sqlite3Config.bExtraSchemaChecks + ){ + /* Skip these error checks for writable_schema=ON */ + return SQLITE_OK; + } + if( db->init.busy ){ + if( sqlite3_stricmp(zType, db->init.azInit[0]) + || sqlite3_stricmp(zName, db->init.azInit[1]) + || sqlite3_stricmp(zTblName, db->init.azInit[2]) + ){ + sqlite3ErrorMsg(pParse, ""); /* corruptSchema() will supply the error */ + return SQLITE_ERROR; + } + }else{ + if( (pParse->nested==0 && 0==sqlite3StrNICmp(zName, "sqlite_", 7)) + || (sqlite3ReadOnlyShadowTables(db) && sqlite3ShadowTableName(db, zName)) + ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", + zName); + return SQLITE_ERROR; + } + + } + return SQLITE_OK; +} + +/* +** Return the PRIMARY KEY index of a table +*/ +Index *sqlite3PrimaryKeyIndex(Table *pTab){ + Index *p; + for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} + return p; +} + +/* +** Convert an table column number into a index column number. That is, +** for the column iCol in the table (as defined by the CREATE TABLE statement) +** find the (first) offset of that column in index pIdx. Or return -1 +** if column iCol is not used in index pIdx. +*/ +i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ + int i; + for(i=0; inColumn; i++){ + if( iCol==pIdx->aiColumn[i] ) return i; + } + return -1; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a storage column number into a table column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. The true column number +** is the index (0,1,2,...) of the column in the CREATE TABLE statement. +** +** The storage column number is less than the table column number if +** and only there are VIRTUAL columns to the left. +** +** If SQLITE_OMIT_GENERATED_COLUMNS, this routine is a no-op macro. +*/ +i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ + if( pTab->tabFlags & TF_HasVirtual ){ + int i; + for(i=0; i<=iCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) iCol++; + } + } + return iCol; +} +#endif + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a table column number into a storage column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. Or, if the input column is +** the N-th virtual column (zero-based) then the storage number is +** the number of non-virtual columns in the table plus N. +** +** The true column number is the index (0,1,2,...) of the column in +** the CREATE TABLE statement. +** +** If the input column is a VIRTUAL column, then it should not appear +** in storage. But the value sometimes is cached in registers that +** follow the range of registers used to construct storage. This +** avoids computing the same VIRTUAL column multiple times, and provides +** values for use by OP_Param opcodes in triggers. Hence, if the +** input column is a VIRTUAL table, put it after all the other columns. +** +** In the following, N means "normal column", S means STORED, and +** V means VIRTUAL. Suppose the CREATE TABLE has columns like this: +** +** CREATE TABLE ex(N,S,V,N,S,V,N,S,V); +** -- 0 1 2 3 4 5 6 7 8 +** +** Then the mapping from this function is as follows: +** +** INPUTS: 0 1 2 3 4 5 6 7 8 +** OUTPUTS: 0 1 6 2 3 7 4 5 8 +** +** So, in other words, this routine shifts all the virtual columns to +** the end. +** +** If SQLITE_OMIT_GENERATED_COLUMNS then there are no virtual columns and +** this routine is a no-op macro. If the pTab does not have any virtual +** columns, then this routine is no-op that always return iCol. If iCol +** is negative (indicating the ROWID column) then this routine return iCol. +*/ +i16 sqlite3TableColumnToStorage(Table *pTab, i16 iCol){ + int i; + i16 n; + assert( iColnCol ); + if( (pTab->tabFlags & TF_HasVirtual)==0 || iCol<0 ) return iCol; + for(i=0, n=0; iaCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) n++; + } + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ){ + /* iCol is a virtual column itself */ + return pTab->nNVCol + i - n; + }else{ + /* iCol is a normal or stored column */ + return n; + } +} +#endif + +/* +** Begin constructing a new table representation in memory. This is +** the first of several action routines that get called in response +** to a CREATE TABLE statement. In particular, this routine is called +** after seeing tokens "CREATE" and "TABLE" and the table name. The isTemp +** flag is true if the table should be stored in the auxiliary database +** file instead of in the main database file. This is normally the case +** when the "TEMP" or "TEMPORARY" keyword occurs in between +** CREATE and TABLE. +** +** The new table record is initialized and put in pParse->pNewTable. +** As more of the CREATE TABLE statement is parsed, additional action +** routines will be called to add more information to this record. +** At the end of the CREATE TABLE statement, the sqlite3EndTable() routine +** is called to complete the construction of the new table record. +*/ +void sqlite3StartTable( + Parse *pParse, /* Parser context */ + Token *pName1, /* First part of the name of the table or view */ + Token *pName2, /* Second part of the name of the table or view */ + int isTemp, /* True if this is a TEMP table */ + int isView, /* True if this is a VIEW */ + int isVirtual, /* True if this is a VIRTUAL table */ + int noErr /* Do nothing if table already exists */ +){ + Table *pTable; + char *zName = 0; /* The name of the new table */ + sqlite3 *db = pParse->db; + Vdbe *v; + int iDb; /* Database number to create the table in */ + Token *pName; /* Unqualified name of the table to create */ + + if( db->init.busy && db->init.newTnum==1 ){ + /* Special case: Parsing the sqlite_schema or sqlite_temp_schema schema */ + iDb = db->init.iDb; + zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); + pName = pName1; + }else{ + /* The common case */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + zName = sqlite3NameFromToken(db, pName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)zName, pName); + } + } + pParse->sNameToken = *pName; + if( zName==0 ) return; + if( sqlite3CheckObjectName(pParse, zName, isView?"view":"table", zName) ){ + goto begin_table_error; + } + if( db->init.iDb==1 ) isTemp = 1; +#ifndef SQLITE_OMIT_AUTHORIZATION + assert( isTemp==0 || isTemp==1 ); + assert( isView==0 || isView==1 ); + { + static const u8 aCode[] = { + SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_VIEW, + SQLITE_CREATE_TEMP_VIEW + }; + char *zDb = db->aDb[iDb].zDbSName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ + goto begin_table_error; + } + if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], + zName, 0, zDb) ){ + goto begin_table_error; + } + } +#endif + + /* Make sure the new table name does not collide with an existing + ** index or table name in the same database. Issue an error message if + ** it does. The exception is if the statement being parsed was passed + ** to an sqlite3_declare_vtab() call. In that case only the column names + ** and types will be used, so there is no need to test for namespace + ** collisions. + */ + if( !IN_SPECIAL_PARSE ){ + char *zDb = db->aDb[iDb].zDbSName; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto begin_table_error; + } + pTable = sqlite3FindTable(db, zName, zDb); + if( pTable ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "table %T already exists", pName); + }else{ + assert( !db->init.busy || CORRUPT_DB ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto begin_table_error; + } + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); + goto begin_table_error; + } + } + + pTable = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTable==0 ){ + assert( db->mallocFailed ); + pParse->rc = SQLITE_NOMEM_BKPT; + pParse->nErr++; + goto begin_table_error; + } + pTable->zName = zName; + pTable->iPKey = -1; + pTable->pSchema = db->aDb[iDb].pSchema; + pTable->nTabRef = 1; +#ifdef SQLITE_DEFAULT_ROWEST + pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST); +#else + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#endif + assert( pParse->pNewTable==0 ); + pParse->pNewTable = pTable; + + /* If this is the magic sqlite_sequence table used by autoincrement, + ** then record a pointer to this table in the main database structure + ** so that INSERT can find the table easily. + */ +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( !pParse->nested && strcmp(zName, "sqlite_sequence")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTable->pSchema->pSeqTab = pTable; + } +#endif + + /* Begin generating the code that will insert the table record into + ** the schema table. Note in particular that we must go ahead + ** and allocate the record number for the table entry now. Before any + ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause + ** indices to be created and the table record must come before the + ** indices. Hence, the record number for the table must be allocated + ** now. + */ + if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ + int addr1; + int fileFormat; + int reg1, reg2, reg3; + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( isVirtual ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* If the file format and encoding in the database have not been set, + ** set them now. + */ + reg1 = pParse->regRowid = ++pParse->nMem; + reg2 = pParse->regRoot = ++pParse->nMem; + reg3 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? + 1 : SQLITE_MAX_FILE_FORMAT; + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, fileFormat); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); + sqlite3VdbeJumpHere(v, addr1); + + /* This just creates a place-holder record in the sqlite_schema table. + ** The record created does not contain anything yet. It will be replaced + ** by the real entry in code generated at sqlite3EndTable(). + ** + ** The rowid for the new entry is left in register pParse->regRowid. + ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid and root page number values are needed by the code that + ** sqlite3EndTable will generate. + */ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( isView || isVirtual ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); + }else +#endif + { + assert( !pParse->bReturning ); + pParse->u1.addrCrTab = + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); + } + sqlite3OpenSchemaTable(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); + } + + /* Normal (non-error) return. */ + return; + + /* If an error occurs, we jump here */ +begin_table_error: + sqlite3DbFree(db, zName); + return; +} + +/* Set properties of a table column based on the (magical) +** name of the column. +*/ +#if SQLITE_ENABLE_HIDDEN_COLUMNS +void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ + if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ + pCol->colFlags |= COLFLAG_HIDDEN; + if( pTab ) pTab->tabFlags |= TF_HasHidden; + }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ + pTab->tabFlags |= TF_OOOHidden; + } +} +#endif + +/* +** Name of the special TEMP trigger used to implement RETURNING. The +** name begins with "sqlite_" so that it is guaranteed not to collide +** with any application-generated triggers. +*/ +#define RETURNING_TRIGGER_NAME "sqlite_returning" + +/* +** Clean up the data structures associated with the RETURNING clause. +*/ +static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ + Hash *pHash; + pHash = &(db->aDb[1].pSchema->trigHash); + sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, 0); + sqlite3ExprListDelete(db, pRet->pReturnEL); + sqlite3DbFree(db, pRet); +} + +/* +** Add the RETURNING clause to the parse currently underway. +** +** This routine creates a special TEMP trigger that will fire for each row +** of the DML statement. That TEMP trigger contains a single SELECT +** statement with a result set that is the argument of the RETURNING clause. +** The trigger has the Trigger.bReturning flag and an opcode of +** TK_RETURNING instead of TK_SELECT, so that the trigger code generator +** knows to handle it specially. The TEMP trigger is automatically +** removed at the end of the parse. +** +** When this routine is called, we do not yet know if the RETURNING clause +** is attached to a DELETE, INSERT, or UPDATE, so construct it as a +** RETURNING trigger instead. It will then be converted into the appropriate +** type on the first call to sqlite3TriggersExist(). +*/ +void sqlite3AddReturning(Parse *pParse, ExprList *pList){ + Returning *pRet; + Hash *pHash; + sqlite3 *db = pParse->db; + if( pParse->pNewTrigger ){ + sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); + }else{ + assert( pParse->bReturning==0 ); + } + pParse->bReturning = 1; + pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); + if( pRet==0 ){ + sqlite3ExprListDelete(db, pList); + return; + } + pParse->u1.pReturning = pRet; + pRet->pParse = pParse; + pRet->pReturnEL = pList; + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); + testcase( pParse->earlyCleanup ); + if( db->mallocFailed ) return; + pRet->retTrig.zName = RETURNING_TRIGGER_NAME; + pRet->retTrig.op = TK_RETURNING; + pRet->retTrig.tr_tm = TRIGGER_AFTER; + pRet->retTrig.bReturning = 1; + pRet->retTrig.pSchema = db->aDb[1].pSchema; + pRet->retTrig.step_list = &pRet->retTStep; + pRet->retTStep.op = TK_RETURNING; + pRet->retTStep.pTrig = &pRet->retTrig; + pRet->retTStep.pExprList = pList; + pHash = &(db->aDb[1].pSchema->trigHash); + assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); + if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) + ==&pRet->retTrig ){ + sqlite3OomFault(db); + } +} + +/* +** Add a new column to the table currently being constructed. +** +** The parser calls this routine once for each column declaration +** in a CREATE TABLE statement. sqlite3StartTable() gets called +** first to get things going. Then this routine is called for each +** column. +*/ +void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ + Table *p; + int i; + char *z; + char *zType; + Column *pCol; + sqlite3 *db = pParse->db; + u8 hName; + + if( (p = pParse->pNewTable)==0 ) return; + if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); + return; + } + z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); + if( z==0 ) return; + if( IN_RENAME_OBJECT ) sqlite3RenameTokenMap(pParse, (void*)z, pName); + memcpy(z, pName->z, pName->n); + z[pName->n] = 0; + sqlite3Dequote(z); + hName = sqlite3StrIHash(z); + for(i=0; inCol; i++){ + if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; + } + } + if( (p->nCol & 0x7)==0 ){ + Column *aNew; + aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); + if( aNew==0 ){ + sqlite3DbFree(db, z); + return; + } + p->aCol = aNew; + } + pCol = &p->aCol[p->nCol]; + memset(pCol, 0, sizeof(p->aCol[0])); + pCol->zName = z; + pCol->hName = hName; + sqlite3ColumnPropertiesFromName(p, pCol); + + if( pType->n==0 ){ + /* If there is no type specified, columns have the default affinity + ** 'BLOB' with a default size of 4 bytes. */ + pCol->affinity = SQLITE_AFF_BLOB; + pCol->szEst = 1; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( 4>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif + }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, pType->z, pType->n); + zType[pType->n] = 0; + sqlite3Dequote(zType); + pCol->affinity = sqlite3AffinityType(zType, pCol); + pCol->colFlags |= COLFLAG_HASTYPE; + } + p->nCol++; + p->nNVCol++; + pParse->constraintName.n = 0; +} + +/* +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. A "NOT NULL" constraint has +** been seen on a column. This routine sets the notNull flag on +** the column currently under construction. +*/ +void sqlite3AddNotNull(Parse *pParse, int onError){ + Table *p; + Column *pCol; + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + pCol = &p->aCol[p->nCol-1]; + pCol->notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; + + /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created + ** on this column. */ + if( pCol->colFlags & COLFLAG_UNIQUE ){ + Index *pIdx; + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None ); + if( pIdx->aiColumn[0]==p->nCol-1 ){ + pIdx->uniqNotNull = 1; + } + } + } +} + +/* +** Scan the column type name zType (length nType) and return the +** associated affinity type. +** +** This routine does a case-independent search of zType for the +** substrings in the following table. If one of the substrings is +** found, the corresponding affinity is returned. If zType contains +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', +** SQLITE_AFF_INTEGER is returned. +** +** Substring | Affinity +** -------------------------------- +** 'INT' | SQLITE_AFF_INTEGER +** 'CHAR' | SQLITE_AFF_TEXT +** 'CLOB' | SQLITE_AFF_TEXT +** 'TEXT' | SQLITE_AFF_TEXT +** 'BLOB' | SQLITE_AFF_BLOB +** 'REAL' | SQLITE_AFF_REAL +** 'FLOA' | SQLITE_AFF_REAL +** 'DOUB' | SQLITE_AFF_REAL +** +** If none of the substrings in the above table are found, +** SQLITE_AFF_NUMERIC is returned. +*/ +char sqlite3AffinityType(const char *zIn, Column *pCol){ + u32 h = 0; + char aff = SQLITE_AFF_NUMERIC; + const char *zChar = 0; + + assert( zIn!=0 ); + while( zIn[0] ){ + h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + zIn++; + if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ + aff = SQLITE_AFF_TEXT; + zChar = zIn; + }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ + && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ + aff = SQLITE_AFF_BLOB; + if( zIn[0]=='(' ) zChar = zIn; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('f'<<24)+('l'<<16)+('o'<<8)+'a') /* FLOA */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('d'<<24)+('o'<<16)+('u'<<8)+'b') /* DOUB */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; +#endif + }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){ /* INT */ + aff = SQLITE_AFF_INTEGER; + break; + } + } + + /* If pCol is not NULL, store an estimate of the field size. The + ** estimate is scaled so that the size of an integer is 1. */ + if( pCol ){ + int v = 0; /* default size is approx 4 bytes */ + if( aff r=(k/4+1) */ + sqlite3GetInt32(zChar, &v); + break; + } + zChar++; + } + }else{ + v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + } + } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( v>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif + v = v/4 + 1; + if( v>255 ) v = 255; + pCol->szEst = v; + } + return aff; +} + +/* +** The expression is the default value for the most recently added column +** of the table currently under construction. +** +** Default value expressions must be constant. Raise an exception if this +** is not the case. +** +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. +*/ +void sqlite3AddDefaultValue( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The parsed expression of the default value */ + const char *zStart, /* Start of the default value text */ + const char *zEnd /* First character past end of defaut value text */ +){ + Table *p; + Column *pCol; + sqlite3 *db = pParse->db; + p = pParse->pNewTable; + if( p!=0 ){ + int isInit = db->init.busy && db->init.iDb!=1; + pCol = &(p->aCol[p->nCol-1]); + if( !sqlite3ExprIsConstantOrFunction(pExpr, isInit) ){ + sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", + pCol->zName); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, "cannot use DEFAULT on a generated column"); +#endif + }else{ + /* A copy of pExpr is used instead of the original, as pExpr contains + ** tokens that point to volatile memory. + */ + Expr x; + sqlite3ExprDelete(db, pCol->pDflt); + memset(&x, 0, sizeof(x)); + x.op = TK_SPAN; + x.u.zToken = sqlite3DbSpanDup(db, zStart, zEnd); + x.pLeft = pExpr; + x.flags = EP_Skip; + pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + sqlite3DbFree(db, x.u.zToken); + } + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, pExpr); + } + sqlite3ExprDelete(db, pExpr); +} + +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the expression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + +/* +** Tag the given column as being part of the PRIMARY KEY +*/ +static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ + pCol->colFlags |= COLFLAG_PRIMKEY; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "generated columns cannot be part of the PRIMARY KEY"); + } +#endif +} + +/* +** Designate the PRIMARY KEY for the table. pList is a list of names +** of columns that form the primary key. If pList is NULL, then the +** most recently added column of the table is the primary key. +** +** A table can have at most one primary key. If the table already has +** a primary key (and this is the second primary key) then create an +** error. +** +** If the PRIMARY KEY is on a single column whose datatype is INTEGER, +** then we will try to use that column as the rowid. Set the Table.iPKey +** field of the table under construction to be the index of the +** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is +** no INTEGER PRIMARY KEY. +** +** If the key is not an INTEGER PRIMARY KEY, then create a unique +** index for the key. No index is created for INTEGER PRIMARY KEYs. +*/ +void sqlite3AddPrimaryKey( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List of field names to be indexed */ + int onError, /* What to do with a uniqueness conflict */ + int autoInc, /* True if the AUTOINCREMENT keyword is present */ + int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ +){ + Table *pTab = pParse->pNewTable; + Column *pCol = 0; + int iCol = -1, i; + int nTerm; + if( pTab==0 ) goto primary_key_exit; + if( pTab->tabFlags & TF_HasPrimaryKey ){ + sqlite3ErrorMsg(pParse, + "table \"%s\" has more than one primary key", pTab->zName); + goto primary_key_exit; + } + pTab->tabFlags |= TF_HasPrimaryKey; + if( pList==0 ){ + iCol = pTab->nCol - 1; + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); + nTerm = 1; + }else{ + nTerm = pList->nExpr; + for(i=0; ia[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); + break; + } + } + } + } + } + if( nTerm==1 + && pCol + && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 + && sortOrder!=SQLITE_SO_DESC + ){ + if( IN_RENAME_OBJECT && pList ){ + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[0].pExpr); + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pCExpr); + } + pTab->iPKey = iCol; + pTab->keyConf = (u8)onError; + assert( autoInc==0 || autoInc==1 ); + pTab->tabFlags |= autoInc*TF_Autoincrement; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags; + (void)sqlite3HasExplicitNulls(pParse, pList); + }else if( autoInc ){ +#ifndef SQLITE_OMIT_AUTOINCREMENT + sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " + "INTEGER PRIMARY KEY"); +#endif + }else{ + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); + pList = 0; + } + +primary_key_exit: + sqlite3ExprListDelete(pParse->db, pList); + return; +} + +/* +** Add a new CHECK constraint to the table currently under construction. +*/ +void sqlite3AddCheckConstraint( + Parse *pParse, /* Parsing context */ + Expr *pCheckExpr, /* The check expression */ + const char *zStart, /* Opening "(" */ + const char *zEnd /* Closing ")" */ +){ +#ifndef SQLITE_OMIT_CHECK + Table *pTab = pParse->pNewTable; + sqlite3 *db = pParse->db; + if( pTab && !IN_DECLARE_VTAB + && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) + ){ + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + }else{ + Token t; + for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} + while( sqlite3Isspace(zEnd[-1]) ){ zEnd--; } + t.z = zStart; + t.n = (int)(zEnd - t.z); + sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1); + } + }else +#endif + { + sqlite3ExprDelete(pParse->db, pCheckExpr); + } +} + +/* +** Set the collation function of the most recently parsed table column +** to the CollSeq given. +*/ +void sqlite3AddCollateType(Parse *pParse, Token *pToken){ + Table *p; + int i; + char *zColl; /* Dequoted name of collation sequence */ + sqlite3 *db; + + if( (p = pParse->pNewTable)==0 || IN_RENAME_OBJECT ) return; + i = p->nCol-1; + db = pParse->db; + zColl = sqlite3NameFromToken(db, pToken); + if( !zColl ) return; + + if( sqlite3LocateCollSeq(pParse, zColl) ){ + Index *pIdx; + sqlite3DbFree(db, p->aCol[i].zColl); + p->aCol[i].zColl = zColl; + + /* If the column is declared as " PRIMARY KEY COLLATE ", + ** then an index may have been created on this column before the + ** collation type was added. Correct this if it is the case. + */ + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 ); + if( pIdx->aiColumn[0]==i ){ + pIdx->azColl[0] = p->aCol[i].zColl; + } + } + }else{ + sqlite3DbFree(db, zColl); + } +} + +/* Change the most recently parsed column to be a GENERATED ALWAYS AS +** column. +*/ +void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + u8 eType = COLFLAG_VIRTUAL; + Table *pTab = pParse->pNewTable; + Column *pCol; + if( pTab==0 ){ + /* generated column in an CREATE TABLE IF NOT EXISTS that already exists */ + goto generated_done; + } + pCol = &(pTab->aCol[pTab->nCol-1]); + if( IN_DECLARE_VTAB ){ + sqlite3ErrorMsg(pParse, "virtual tables cannot use computed columns"); + goto generated_done; + } + if( pCol->pDflt ) goto generated_error; + if( pType ){ + if( pType->n==7 && sqlite3StrNICmp("virtual",pType->z,7)==0 ){ + /* no-op */ + }else if( pType->n==6 && sqlite3StrNICmp("stored",pType->z,6)==0 ){ + eType = COLFLAG_STORED; + }else{ + goto generated_error; + } + } + if( eType==COLFLAG_VIRTUAL ) pTab->nNVCol--; + pCol->colFlags |= eType; + assert( TF_HasVirtual==COLFLAG_VIRTUAL ); + assert( TF_HasStored==COLFLAG_STORED ); + pTab->tabFlags |= eType; + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ + } + pCol->pDflt = pExpr; + pExpr = 0; + goto generated_done; + +generated_error: + sqlite3ErrorMsg(pParse, "error in generated column \"%s\"", + pCol->zName); +generated_done: + sqlite3ExprDelete(pParse->db, pExpr); +#else + /* Throw and error for the GENERATED ALWAYS AS clause if the + ** SQLITE_OMIT_GENERATED_COLUMNS compile-time option is used. */ + sqlite3ErrorMsg(pParse, "generated columns not supported"); + sqlite3ExprDelete(pParse->db, pExpr); +#endif +} + +/* +** Generate code that will increment the schema cookie. +** +** The schema cookie is used to determine when the schema for the +** database changes. After each schema change, the cookie value +** changes. When a process first reads the schema it records the +** cookie. Thereafter, whenever it goes to access the database, +** it checks the cookie to make sure the schema has not changed +** since it was last read. +** +** This plan is not completely bullet-proof. It is possible for +** the schema to change multiple times and for the cookie to be +** set back to prior value. But schema changes are infrequent +** and the probability of hitting the same cookie value is only +** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. +*/ +void sqlite3ChangeCookie(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + (int)(1+(unsigned)db->aDb[iDb].pSchema->schema_cookie)); +} + +/* +** Measure the number of characters needed to output the given +** identifier. The number returned includes any quotes used +** but does not include the null terminator. +** +** The estimate is conservative. It might be larger that what is +** really needed. +*/ +static int identLength(const char *z){ + int n; + for(n=0; *z; n++, z++){ + if( *z=='"' ){ n++; } + } + return n + 2; +} + +/* +** The first parameter is a pointer to an output buffer. The second +** parameter is a pointer to an integer that contains the offset at +** which to write into the output buffer. This function copies the +** nul-terminated string pointed to by the third parameter, zSignedIdent, +** to the specified offset in the buffer and updates *pIdx to refer +** to the first byte after the last byte written before returning. +** +** If the string zSignedIdent consists entirely of alpha-numeric +** characters, does not begin with a digit and is not an SQL keyword, +** then it is copied to the output buffer exactly as it is. Otherwise, +** it is quoted using double-quotes. +*/ +static void identPut(char *z, int *pIdx, char *zSignedIdent){ + unsigned char *zIdent = (unsigned char*)zSignedIdent; + int i, j, needQuote; + i = *pIdx; + + for(j=0; zIdent[j]; j++){ + if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; + } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; + + if( needQuote ) z[i++] = '"'; + for(j=0; zIdent[j]; j++){ + z[i++] = zIdent[j]; + if( zIdent[j]=='"' ) z[i++] = '"'; + } + if( needQuote ) z[i++] = '"'; + z[i] = 0; + *pIdx = i; +} + +/* +** Generate a CREATE TABLE statement appropriate for the given +** table. Memory to hold the text of the statement is obtained +** from sqliteMalloc() and must be freed by the calling function. +*/ +static char *createTableStmt(sqlite3 *db, Table *p){ + int i, k, n; + char *zStmt; + char *zSep, *zSep2, *zEnd; + Column *pCol; + n = 0; + for(pCol = p->aCol, i=0; inCol; i++, pCol++){ + n += identLength(pCol->zName) + 5; + } + n += identLength(p->zName); + if( n<50 ){ + zSep = ""; + zSep2 = ","; + zEnd = ")"; + }else{ + zSep = "\n "; + zSep2 = ",\n "; + zEnd = "\n)"; + } + n += 35 + 6*p->nCol; + zStmt = sqlite3DbMallocRaw(0, n); + if( zStmt==0 ){ + sqlite3OomFault(db); + return 0; + } + sqlite3_snprintf(n, zStmt, "CREATE TABLE "); + k = sqlite3Strlen30(zStmt); + identPut(zStmt, &k, p->zName); + zStmt[k++] = '('; + for(pCol=p->aCol, i=0; inCol; i++, pCol++){ + static const char * const azType[] = { + /* SQLITE_AFF_BLOB */ "", + /* SQLITE_AFF_TEXT */ " TEXT", + /* SQLITE_AFF_NUMERIC */ " NUM", + /* SQLITE_AFF_INTEGER */ " INT", + /* SQLITE_AFF_REAL */ " REAL" + }; + int len; + const char *zType; + + sqlite3_snprintf(n-k, &zStmt[k], zSep); + k += sqlite3Strlen30(&zStmt[k]); + zSep = zSep2; + identPut(zStmt, &k, pCol->zName); + assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); + assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_BLOB ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); + testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); + testcase( pCol->affinity==SQLITE_AFF_INTEGER ); + testcase( pCol->affinity==SQLITE_AFF_REAL ); + + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; + len = sqlite3Strlen30(zType); + assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==sqlite3AffinityType(zType, 0) ); + memcpy(&zStmt[k], zType, len); + k += len; + assert( k<=n ); + } + sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); + return zStmt; +} + +/* +** Resize an Index object to hold N columns total. Return SQLITE_OK +** on success and SQLITE_NOMEM on an OOM error. +*/ +static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ + char *zExtra; + int nByte; + if( pIdx->nColumn>=N ) return SQLITE_OK; + assert( pIdx->isResized==0 ); + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; + zExtra = sqlite3DbMallocZero(db, nByte); + if( zExtra==0 ) return SQLITE_NOMEM_BKPT; + memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); + pIdx->azColl = (const char**)zExtra; + zExtra += sizeof(char*)*N; + memcpy(zExtra, pIdx->aiRowLogEst, sizeof(LogEst)*(pIdx->nKeyCol+1)); + pIdx->aiRowLogEst = (LogEst*)zExtra; + zExtra += sizeof(LogEst)*N; + memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); + pIdx->aiColumn = (i16*)zExtra; + zExtra += sizeof(i16)*N; + memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); + pIdx->aSortOrder = (u8*)zExtra; + pIdx->nColumn = N; + pIdx->isResized = 1; + return SQLITE_OK; +} + +/* +** Estimate the total row width for a table. +*/ +static void estimateTableWidth(Table *pTab){ + unsigned wTable = 0; + const Column *pTabCol; + int i; + for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){ + wTable += pTabCol->szEst; + } + if( pTab->iPKey<0 ) wTable++; + pTab->szTabRow = sqlite3LogEst(wTable*4); +} + +/* +** Estimate the average size of a row for an index. +*/ +static void estimateIndexWidth(Index *pIdx){ + unsigned wIndex = 0; + int i; + const Column *aCol = pIdx->pTable->aCol; + for(i=0; inColumn; i++){ + i16 x = pIdx->aiColumn[i]; + assert( xpTable->nCol ); + wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + } + pIdx->szIdxRow = sqlite3LogEst(wIndex*4); +} + +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. +*/ +static int hasColumn(const i16 *aiCol, int nCol, int x){ + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } + return 0; +} + +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table. The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** 2019-10-24: For the purpose of this computation, virtual columns are +** not considered to be covered by the index, even if they are in the +** index, because we do not trust the logic in whereIndexExprTrans() to be +** able to find all instances of a reference to the indexed table column +** and convert them into references to the index. Hence we always want +** the actual table at hand in order to recompute the virtual column, if +** necessary. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + Table *pTab = pIdx->pTable; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); +} + +/* +** This routine runs at the end of parsing a CREATE TABLE statement that +** has a WITHOUT ROWID clause. The job of this routine is to convert both +** internal schema data structures and the generated VDBE code so that they +** are appropriate for a WITHOUT ROWID table instead of a rowid table. +** Changes include: +** +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY +** into BTREE_BLOBKEY. +** (3) Bypass the creation of the sqlite_schema table entry +** for the PRIMARY KEY as the primary key index is now +** identified by the sqlite_schema table entry of the table itself. +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the +** schema to the rootpage from the main table. +** (5) Add all table columns to the PRIMARY KEY Index object +** so that the PRIMARY KEY is a covering index. The surplus +** columns are part of KeyInfo.nAllField and are not used for +** sorting or lookup or uniqueness checks. +** (6) Replace the rowid tail on all automatically generated UNIQUE +** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. +*/ +static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ + Index *pIdx; + Index *pPk; + int nPk; + int nExtra; + int i, j; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) + */ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + pTab->tabFlags |= TF_HasNotNull; + } + + /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY + ** into BTREE_BLOBKEY. + */ + assert( !pParse->bReturning ); + if( pParse->u1.addrCrTab ){ + assert( v ); + sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); + } + + /* Locate the PRIMARY KEY index. Or, if this table was originally + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + */ + if( pTab->iPKey>=0 ){ + ExprList *pList; + Token ipkToken; + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); + if( pList==0 ) return; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); + } + pList->a[0].sortFlags = pParse->iPkSortOrder; + assert( pParse->pNewTable==pTab ); + pTab->iPKey = -1; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( db->mallocFailed || pParse->nErr ) return; + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk->nKeyCol==1 ); + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( isDupColumn(pPk, j, pPk, i) ){ + pPk->nColumn--; + }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); + pPk->azColl[j] = pPk->azColl[i]; + pPk->aSortOrder[j] = pPk->aSortOrder[i]; + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; + } + assert( pPk!=0 ); + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; + nPk = pPk->nColumn = pPk->nKeyCol; + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_schema + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v && pPk->tnum>0 ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, (int)pPk->tnum, OP_Goto); + } + + /* The root page of the PRIMARY KEY is the table root page */ + pPk->tnum = pTab->tnum; + + /* Update the in-memory representation of all UNIQUE indices by converting + ** the final rowid column into one or more columns of the PRIMARY KEY. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int n; + if( IsPrimaryKeyIndex(pIdx) ) continue; + for(i=n=0; inKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } + } + if( n==0 ){ + /* This index is a superset of the primary key */ + pIdx->nColumn = pIdx->nKeyCol; + continue; + } + if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + for(i=0, j=pIdx->nKeyCol; inKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + pIdx->aiColumn[j] = pPk->aiColumn[i]; + pIdx->azColl[j] = pPk->azColl[i]; + if( pPk->aSortOrder[i] ){ + /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + pIdx->bAscKeyBug = 1; + } + j++; + } + } + assert( pIdx->nColumn>=pIdx->nKeyCol+n ); + assert( pIdx->nColumn>=j ); + } + + /* Add all table columns to the PRIMARY KEY index + */ + nExtra = 0; + for(i=0; inCol; i++){ + if( !hasColumn(pPk->aiColumn, nPk, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; + } + if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 + ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = sqlite3StrBINARY; + j++; + } + } + assert( pPk->nColumn==j ); + assert( pTab->nNVCol<=j ); + recomputeColumnsNotIndexed(pPk); +} + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTab is a virtual table and zName is a shadow table name +** for that virtual table. +*/ +int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){ + int nName; /* Length of zName */ + Module *pMod; /* Module for the virtual table */ + + if( !IsVirtual(pTab) ) return 0; + nName = sqlite3Strlen30(pTab->zName); + if( sqlite3_strnicmp(zName, pTab->zName, nName)!=0 ) return 0; + if( zName[nName]!='_' ) return 0; + pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->azModuleArg[0]); + if( pMod==0 ) return 0; + if( pMod->pModule->iVersion<3 ) return 0; + if( pMod->pModule->xShadowName==0 ) return 0; + return pMod->pModule->xShadowName(zName+nName+1); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if zName is a shadow table name in the current database +** connection. +** +** zName is temporarily modified while this routine is running, but is +** restored to its original value prior to this routine returning. +*/ +int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ + char *zTail; /* Pointer to the last "_" in zName */ + Table *pTab; /* Table that zName is a shadow of */ + zTail = strrchr(zName, '_'); + if( zTail==0 ) return 0; + *zTail = 0; + pTab = sqlite3FindTable(db, zName, 0); + *zTail = '_'; + if( pTab==0 ) return 0; + if( !IsVirtual(pTab) ) return 0; + return sqlite3IsShadowTableOf(db, pTab, zName); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef SQLITE_DEBUG +/* +** Mark all nodes of an expression as EP_Immutable, indicating that +** they should not be changed. Expressions attached to a table or +** index definition are tagged this way to help ensure that we do +** not pass them into code generator routines by mistake. +*/ +static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + ExprSetVVAProperty(pExpr, EP_Immutable); + return WRC_Continue; +} +static void markExprListImmutable(ExprList *pList){ + if( pList ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = markImmutableExprStep; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + sqlite3WalkExprList(&w, pList); + } +} +#else +#define markExprListImmutable(X) /* no-op */ +#endif /* SQLITE_DEBUG */ + + +/* +** This routine is called to report the final ")" that terminates +** a CREATE TABLE statement. +** +** The table structure that other action routines have been building +** is added to the internal hash tables, assuming no errors have +** occurred. +** +** An entry for the table is made in the schema table on disk, unless +** this is a temporary table or db->init.busy==1. When db->init.busy==1 +** it means we are reading the sqlite_schema table because we just +** connected to the database or because the sqlite_schema table has +** recently changed, so the entry for this table already exists in +** the sqlite_schema table. We do not want to create it again. +** +** If the pSelect argument is not NULL, it means that this routine +** was called to create a table generated from a +** "CREATE TABLE ... AS SELECT ..." statement. The column names of +** the new table will match the result set of the SELECT. +*/ +void sqlite3EndTable( + Parse *pParse, /* Parse context */ + Token *pCons, /* The ',' token after the last column defn. */ + Token *pEnd, /* The ')' before options in the CREATE TABLE */ + u8 tabOpts, /* Extra table options. Usually 0. */ + Select *pSelect /* Select from a "CREATE ... AS SELECT" */ +){ + Table *p; /* The new table */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* Database in which the table lives */ + Index *pIdx; /* An implied index of the table */ + + if( pEnd==0 && pSelect==0 ){ + return; + } + assert( !db->mallocFailed ); + p = pParse->pNewTable; + if( p==0 ) return; + + if( pSelect==0 && sqlite3ShadowTableName(db, p->zName) ){ + p->tabFlags |= TF_Shadow; + } + + /* If the db->init.busy is 1 it means we are reading the SQL off the + ** "sqlite_schema" or "sqlite_temp_schema" table on the disk. + ** So do not write to the disk again. Extract the root page number + ** for the table from the db->init.newTnum field. (The page number + ** should have been put there by the sqliteOpenCb routine.) + ** + ** If the root page number is 1, that means this is the sqlite_schema + ** table itself. So mark it read-only. + */ + if( db->init.busy ){ + if( pSelect ){ + sqlite3ErrorMsg(pParse, ""); + return; + } + p->tnum = db->init.newTnum; + if( p->tnum==1 ) p->tabFlags |= TF_Readonly; + } + + assert( (p->tabFlags & TF_HasPrimaryKey)==0 + || p->iPKey>=0 || sqlite3PrimaryKeyIndex(p)!=0 ); + assert( (p->tabFlags & TF_HasPrimaryKey)!=0 + || (p->iPKey<0 && sqlite3PrimaryKeyIndex(p)==0) ); + + /* Special processing for WITHOUT ROWID Tables */ + if( tabOpts & TF_WithoutRowid ){ + if( (p->tabFlags & TF_Autoincrement) ){ + sqlite3ErrorMsg(pParse, + "AUTOINCREMENT not allowed on WITHOUT ROWID tables"); + return; + } + if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ + sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); + return; + } + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + convertToWithoutRowidTable(pParse, p); + } + iDb = sqlite3SchemaToIndex(db, p->pSchema); + +#ifndef SQLITE_OMIT_CHECK + /* Resolve names in all CHECK constraint expressions. + */ + if( p->pCheck ){ + sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + if( pParse->nErr ){ + /* If errors are seen, delete the CHECK constraints now, else they might + ** actually be used if PRAGMA writable_schema=ON is set. */ + sqlite3ExprListDelete(db, p->pCheck); + p->pCheck = 0; + }else{ + markExprListImmutable(p->pCheck); + } + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( p->tabFlags & TF_HasGenerated ){ + int ii, nNG = 0; + testcase( p->tabFlags & TF_HasVirtual ); + testcase( p->tabFlags & TF_HasStored ); + for(ii=0; iinCol; ii++){ + u32 colFlags = p->aCol[ii].colFlags; + if( (colFlags & COLFLAG_GENERATED)!=0 ){ + Expr *pX = p->aCol[ii].pDflt; + testcase( colFlags & COLFLAG_VIRTUAL ); + testcase( colFlags & COLFLAG_STORED ); + if( sqlite3ResolveSelfReference(pParse, p, NC_GenCol, pX, 0) ){ + /* If there are errors in resolving the expression, change the + ** expression to a NULL. This prevents code generators that operate + ** on the expression from inserting extra parts into the expression + ** tree that have been allocated from lookaside memory, which is + ** illegal in a schema and will lead to errors or heap corruption + ** when the database connection closes. */ + sqlite3ExprDelete(db, pX); + p->aCol[ii].pDflt = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + }else{ + nNG++; + } + } + if( nNG==0 ){ + sqlite3ErrorMsg(pParse, "must have at least one non-generated column"); + return; + } + } +#endif + + /* Estimate the average row size for the table and for all implied indices */ + estimateTableWidth(p); + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + estimateIndexWidth(pIdx); + } + + /* If not initializing, then create a record for the new table + ** in the schema table of the database. + ** + ** If this is a TEMPORARY table, write the entry into the auxiliary + ** file instead of into the main database file. + */ + if( !db->init.busy ){ + int n; + Vdbe *v; + char *zType; /* "view" or "table" */ + char *zType2; /* "VIEW" or "TABLE" */ + char *zStmt; /* Text of the CREATE TABLE or CREATE VIEW statement */ + + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + + sqlite3VdbeAddOp1(v, OP_Close, 0); + + /* + ** Initialize zType for the new view or table. + */ + if( p->pSelect==0 ){ + /* A regular table */ + zType = "table"; + zType2 = "TABLE"; +#ifndef SQLITE_OMIT_VIEW + }else{ + /* A view */ + zType = "view"; + zType2 = "VIEW"; +#endif + } + + /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT + ** statement to populate the new table. The root-page number for the + ** new table is in register pParse->regRoot. + ** + ** Once the SELECT has been coded by sqlite3Select(), it is in a + ** suitable state to query for the column names and types to be used + ** by the new table. + ** + ** A shared-cache write-lock is not required to write to the new table, + ** as a schema-lock must have already been obtained to create it. Since + ** a schema-lock excludes all other database users, the write-lock would + ** be redundant. + */ + if( pSelect ){ + SelectDest dest; /* Where the SELECT should store results */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int regRec; /* A record to be insert into the new table */ + int regRowid; /* Rowid of the next row to insert */ + int addrInsLoop; /* Top of the loop for inserting rows */ + Table *pSelTab; /* A table that describes the SELECT results */ + + regYield = ++pParse->nMem; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + assert(pParse->nTab==1); + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); + pParse->nTab = 2; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + if( pParse->nErr ) return; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = p->nNVCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + sqlite3Select(pParse, pSelect, &dest); + if( pParse->nErr ) return; + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); + addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); + sqlite3TableAffinity(v, p, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeGoto(v, addrInsLoop); + sqlite3VdbeJumpHere(v, addrInsLoop); + sqlite3VdbeAddOp1(v, OP_Close, 1); + } + + /* Compute the complete text of the CREATE statement */ + if( pSelect ){ + zStmt = createTableStmt(db, p); + }else{ + Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; + n = (int)(pEnd2->z - pParse->sNameToken.z); + if( pEnd2->z[0]!=';' ) n += pEnd2->n; + zStmt = sqlite3MPrintf(db, + "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + ); + } + + /* A slot for the record has already been allocated in the + ** schema table. We just need to update that slot with all + ** the information we've collected. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE + " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" + " WHERE rowid=#%d", + db->aDb[iDb].zDbSName, + zType, + p->zName, + p->zName, + pParse->regRoot, + zStmt, + pParse->regRowid + ); + sqlite3DbFree(db, zStmt); + sqlite3ChangeCookie(pParse, iDb); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Check to see if we need to create an sqlite_sequence table for + ** keeping track of autoincrement keys. + */ + if( (p->tabFlags & TF_Autoincrement)!=0 ){ + Db *pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->pSeqTab==0 ){ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.sqlite_sequence(name,seq)", + pDb->zDbSName + ); + } + } +#endif + + /* Reparse everything to update our internal data structures */ + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + } + + /* Add the table to the in-memory representation of the database. + */ + if( db->init.busy ){ + Table *pOld; + Schema *pSchema = p->pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); + if( pOld ){ + assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ + sqlite3OomFault(db); + return; + } + pParse->pNewTable = 0; + db->mDbFlags |= DBFLAG_SchemaChange; + } + +#ifndef SQLITE_OMIT_ALTERTABLE + if( !pSelect && !p->pSelect ){ + assert( pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + p->addColOffset = 13 + (int)(pCons->z - pParse->sNameToken.z); + } +#endif +} + +#ifndef SQLITE_OMIT_VIEW +/* +** The parser calls this routine in order to create a new VIEW +*/ +void sqlite3CreateView( + Parse *pParse, /* The parsing context */ + Token *pBegin, /* The CREATE token that begins the statement */ + Token *pName1, /* The token that holds the name of the view */ + Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ + Select *pSelect, /* A SELECT statement that will become the new view */ + int isTemp, /* TRUE for a TEMPORARY view */ + int noErr /* Suppress error messages if VIEW already exists */ +){ + Table *p; + int n; + const char *z; + Token sEnd; + DbFixer sFix; + Token *pName = 0; + int iDb; + sqlite3 *db = pParse->db; + + if( pParse->nVar>0 ){ + sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); + goto create_view_fail; + } + sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); + p = pParse->pNewTable; + if( p==0 || pParse->nErr ) goto create_view_fail; + sqlite3TwoPartName(pParse, pName1, pName2, &pName); + iDb = sqlite3SchemaToIndex(db, p->pSchema); + sqlite3FixInit(&sFix, pParse, iDb, "view", pName); + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; + + /* Make a copy of the entire SELECT statement that defines the view. + ** This will force all the Expr.token.z values to be dynamically + ** allocated rather than point to the input string - which means that + ** they will persist after the current sqlite3_exec() call returns. + */ + pSelect->selFlags |= SF_View; + if( IN_RENAME_OBJECT ){ + p->pSelect = pSelect; + pSelect = 0; + }else{ + p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; + + /* Locate the end of the CREATE VIEW statement. Make sEnd point to + ** the end. + */ + sEnd = pParse->sLastToken; + assert( sEnd.z[0]!=0 || sEnd.n==0 ); + if( sEnd.z[0]!=';' ){ + sEnd.z += sEnd.n; + } + sEnd.n = 0; + n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); + z = pBegin->z; + while( sqlite3Isspace(z[n-1]) ){ n--; } + sEnd.z = &z[n-1]; + sEnd.n = 1; + + /* Use sqlite3EndTable() to add the view to the schema table */ + sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprlistUnmap(pParse, pCNames); + } + sqlite3ExprListDelete(db, pCNames); + return; +} +#endif /* SQLITE_OMIT_VIEW */ + +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +/* +** The Table structure pTable is really a VIEW. Fill in the names of +** the columns of the view in the pTable structure. Return the number +** of errors. If an error is seen leave an error message in pParse->zErrMsg. +*/ +int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + Table *pSelTab; /* A fake table from which we get the result set */ + Select *pSel; /* Copy of the SELECT that implements the view */ + int nErr = 0; /* Number of errors encountered */ + int n; /* Temporarily holds the number of cursors assigned */ + sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; +#endif +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif + + assert( pTable ); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + db->nSchemaLock++; + rc = sqlite3VtabCallConnect(pParse, pTable); + db->nSchemaLock--; + if( rc ){ + return 1; + } + if( IsVirtual(pTable) ) return 0; +#endif + +#ifndef SQLITE_OMIT_VIEW + /* A positive nCol means the columns names for this view are + ** already known. + */ + if( pTable->nCol>0 ) return 0; + + /* A negative nCol is a special marker meaning that we are currently + ** trying to compute the column names. If we enter this routine with + ** a negative nCol, it means two or more views form a loop, like this: + ** + ** CREATE VIEW one AS SELECT * FROM two; + ** CREATE VIEW two AS SELECT * FROM one; + ** + ** Actually, the error above is now caught prior to reaching this point. + ** But the following test is still important as it does come up + ** in the following: + ** + ** CREATE TABLE main.ex1(a); + ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; + ** SELECT * FROM temp.ex1; + */ + if( pTable->nCol<0 ){ + sqlite3ErrorMsg(pParse, "view %s is circularly defined", pTable->zName); + return 1; + } + assert( pTable->nCol>=0 ); + + /* If we get this far, it means we need to compute the table names. + ** Note that the call to sqlite3ResultSetOfSelect() will expand any + ** "*" elements in the results set of the view and will assign cursors + ** to the elements of the FROM clause. But we do not want these changes + ** to be permanent. So the computation is done on a copy of the SELECT + ** statement that defines the view. + */ + assert( pTable->pSelect ); + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + u8 eParseMode = pParse->eParseMode; + pParse->eParseMode = PARSE_MODE_NORMAL; + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + DisableLookaside; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); + db->xAuth = xAuth; +#else + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); +#endif + pParse->nTab = n; + if( pSelTab==0 ){ + pTable->nCol = 0; + nErr++; + }else if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, + SQLITE_AFF_NONE); + } + }else{ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pTable->tabFlags |= (pSelTab->tabFlags & COLFLAG_NOINSERT); + pSelTab->nCol = 0; + pSelTab->aCol = 0; + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + } + pTable->nNVCol = pTable->nCol; + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDelete(db, pSel); + EnableLookaside; + pParse->eParseMode = eParseMode; + } else { + nErr++; + } + pTable->pSchema->schemaFlags |= DB_UnresetViews; + if( db->mallocFailed ){ + sqlite3DeleteColumnNames(db, pTable); + pTable->aCol = 0; + pTable->nCol = 0; + } +#endif /* SQLITE_OMIT_VIEW */ + return nErr; +} +#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ + +#ifndef SQLITE_OMIT_VIEW +/* +** Clear the column names from every VIEW in database idx. +*/ +static void sqliteViewResetAll(sqlite3 *db, int idx){ + HashElem *i; + assert( sqlite3SchemaMutexHeld(db, idx, 0) ); + if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; + for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + if( pTab->pSelect ){ + sqlite3DeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; + } + } + DbClearProperty(db, idx, DB_UnresetViews); +} +#else +# define sqliteViewResetAll(A,B) +#endif /* SQLITE_OMIT_VIEW */ + +/* +** This function is called by the VDBE to adjust the internal schema +** used by SQLite when the btree layer moves a table root page. The +** root-page of a table or index in database iDb has changed from iFrom +** to iTo. +** +** Ticket #1728: The symbol table might still contain information +** on tables and/or indices that are the process of being deleted. +** If you are unlucky, one of those deleted indices or tables might +** have the same rootpage number as the real table or index that is +** being moved. So we cannot stop searching after the first match +** because the first match might be for one of the deleted indices +** or tables and not the table/index that is actually being moved. +** We must continue looping until all tables and indices with +** rootpage==iFrom have been converted to have a rootpage of iTo +** in order to be certain that we got the right one. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +void sqlite3RootPageMoved(sqlite3 *db, int iDb, Pgno iFrom, Pgno iTo){ + HashElem *pElem; + Hash *pHash; + Db *pDb; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb = &db->aDb[iDb]; + pHash = &pDb->pSchema->tblHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + if( pTab->tnum==iFrom ){ + pTab->tnum = iTo; + } + } + pHash = &pDb->pSchema->idxHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Index *pIdx = sqliteHashData(pElem); + if( pIdx->tnum==iFrom ){ + pIdx->tnum = iTo; + } + } +} +#endif + +/* +** Write code to erase the table with root-page iTable from database iDb. +** Also write code to modify the sqlite_schema table and internal schema +** if a root-page of another table is moved by the btree-layer whilst +** erasing iTable (this can happen with an auto-vacuum database). +*/ +static void destroyRootPage(Parse *pParse, int iTable, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + int r1 = sqlite3GetTempReg(pParse); + if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema"); + sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); + sqlite3MayAbort(pParse); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* OP_Destroy stores an in integer r1. If this integer + ** is non-zero, then it is the root page number of a table moved to + ** location iTable. The following code modifies the sqlite_schema table to + ** reflect this. + ** + ** The "#NNN" in the SQL is a special constant that means whatever value + ** is in register NNN. See grammar rules associated with the TK_REGISTER + ** token for additional information. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE + " SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zDbSName, iTable, r1, r1); +#endif + sqlite3ReleaseTempReg(pParse, r1); +} + +/* +** Write VDBE code to erase table pTab and all associated indices on disk. +** Code to update the sqlite_schema tables and internal schema definitions +** in case a root-page belonging to another table is moved by the btree layer +** is also added (this can happen with an auto-vacuum database). +*/ +static void destroyTable(Parse *pParse, Table *pTab){ + /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM + ** is not defined), then it is important to call OP_Destroy on the + ** table and index root-pages in order, starting with the numerically + ** largest root-page number. This guarantees that none of the root-pages + ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the + ** following were coded: + ** + ** OP_Destroy 4 0 + ** ... + ** OP_Destroy 5 0 + ** + ** and root page 5 happened to be the largest root-page number in the + ** database, then root page 5 would be moved to page 4 by the + ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit + ** a free-list page. + */ + Pgno iTab = pTab->tnum; + Pgno iDestroyed = 0; + + while( 1 ){ + Index *pIdx; + Pgno iLargest = 0; + + if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ + Pgno iIdx = pIdx->tnum; + assert( pIdx->pSchema==pTab->pSchema ); + if( (iDestroyed==0 || (iIdxiLargest ){ + iLargest = iIdx; + } + } + if( iLargest==0 ){ + return; + }else{ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); + destroyRootPage(pParse, iLargest, iDb); + iDestroyed = iLargest; + } + } +} + +/* +** Remove entries from the sqlite_statN tables (for N in (1,2,3)) +** after a DROP INDEX or DROP TABLE command. +*/ +static void sqlite3ClearStatTables( + Parse *pParse, /* The parsing context */ + int iDb, /* The database number */ + const char *zType, /* "idx" or "tbl" */ + const char *zName /* Name of index or table */ +){ + int i; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; + for(i=1; i<=4; i++){ + char zTab[24]; + sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); + if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + zDbName, zTab, zType, zName + ); + } + } +} + +/* +** Generate code to drop a table. +*/ +void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ + Vdbe *v; + sqlite3 *db = pParse->db; + Trigger *pTrigger; + Db *pDb = &db->aDb[iDb]; + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_schema and/or + ** sqlite_temp_schema if required. + */ + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", + pDb->zDbSName, pTab->zName + ); + } +#endif + + /* Drop all entries in the schema table that refer to the + ** table. The program name loops through the schema table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled separately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q." DFLT_SCHEMA_TABLE + " WHERE tbl_name=%Q and type!='trigger'", + pDb->zDbSName, pTab->zName); + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); + sqliteViewResetAll(db, iDb); +} + +/* +** Return TRUE if shadow tables should be read-only in the current +** context. +*/ +int sqlite3ReadOnlyShadowTables(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (db->flags & SQLITE_Defensive)!=0 + && db->pVtabCtx==0 + && db->nVdbeExec==0 + ){ + return 1; + } +#endif + return 0; +} + +/* +** Return true if it is not allowed to drop the given table +*/ +static int tableMayNotBeDropped(sqlite3 *db, Table *pTab){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName+7, "stat", 4)==0 ) return 0; + if( sqlite3StrNICmp(pTab->zName+7, "parameters", 10)==0 ) return 0; + return 1; + } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + return 1; + } + return 0; +} + +/* +** This routine is called to do the work of a DROP TABLE statement. +** pName is the name of the table to be dropped. +*/ +void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ + Table *pTab; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + if( db->mallocFailed ){ + goto exit_drop_table; + } + assert( pParse->nErr==0 ); + assert( pName->nSrc==1 ); + if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; + if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); + if( noErr ) db->suppressErr--; + + if( pTab==0 ){ + if( noErr ) sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + goto exit_drop_table; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbnDb ); + + /* If pTab is a virtual table, call ViewGetColumnNames() to ensure + ** it is initialized. + */ + if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_drop_table; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code; + const char *zTab = SCHEMA_TABLE(iDb); + const char *zDb = db->aDb[iDb].zDbSName; + const char *zArg2 = 0; + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ + goto exit_drop_table; + } + if( isView ){ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_VIEW; + }else{ + code = SQLITE_DROP_VIEW; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( IsVirtual(pTab) ){ + code = SQLITE_DROP_VTABLE; + zArg2 = sqlite3GetVTable(db, pTab)->pMod->zName; +#endif + }else{ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_TABLE; + }else{ + code = SQLITE_DROP_TABLE; + } + } + if( sqlite3AuthCheck(pParse, code, pTab->zName, zArg2, zDb) ){ + goto exit_drop_table; + } + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ + goto exit_drop_table; + } + } +#endif + if( tableMayNotBeDropped(db, pTab) ){ + sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); + goto exit_drop_table; + } + +#ifndef SQLITE_OMIT_VIEW + /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used + ** on a table. + */ + if( isView && pTab->pSelect==0 ){ + sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); + goto exit_drop_table; + } + if( !isView && pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); + goto exit_drop_table; + } +#endif + + /* Generate code to remove the table from the schema table + ** on disk. + */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + if( !isView ){ + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + } + sqlite3CodeDropTable(pParse, pTab, iDb, isView); + } + +exit_drop_table: + sqlite3SrcListDelete(db, pName); +} + +/* +** This routine is called to create a new foreign key on the table +** currently under construction. pFromCol determines which columns +** in the current table point to the foreign key. If pFromCol==0 then +** connect the key to the last column inserted. pTo is the name of +** the table referred to (a.k.a the "parent" table). pToCol is a list +** of tables in the parent pTo table. flags contains all +** information about the conflict resolution algorithms specified +** in the ON DELETE, ON UPDATE and ON INSERT clauses. +** +** An FKey structure is created and added to the table currently +** under construction in the pParse->pNewTable field. +** +** The foreign key is set for IMMEDIATE processing. A subsequent call +** to sqlite3DeferForeignKey() might change this to DEFERRED. +*/ +void sqlite3CreateForeignKey( + Parse *pParse, /* Parsing context */ + ExprList *pFromCol, /* Columns in this table that point to other table */ + Token *pTo, /* Name of the other table */ + ExprList *pToCol, /* Columns in the other table */ + int flags /* Conflict resolution algorithms. */ +){ + sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_FOREIGN_KEY + FKey *pFKey = 0; + FKey *pNextTo; + Table *p = pParse->pNewTable; + int nByte; + int i; + int nCol; + char *z; + + assert( pTo!=0 ); + if( p==0 || IN_DECLARE_VTAB ) goto fk_end; + if( pFromCol==0 ){ + int iCol = p->nCol-1; + if( NEVER(iCol<0) ) goto fk_end; + if( pToCol && pToCol->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "foreign key on %s" + " should reference only one column of table %T", + p->aCol[iCol].zName, pTo); + goto fk_end; + } + nCol = 1; + }else if( pToCol && pToCol->nExpr!=pFromCol->nExpr ){ + sqlite3ErrorMsg(pParse, + "number of columns in foreign key does not match the number of " + "columns in the referenced table"); + goto fk_end; + }else{ + nCol = pFromCol->nExpr; + } + nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + if( pToCol ){ + for(i=0; inExpr; i++){ + nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; + } + } + pFKey = sqlite3DbMallocZero(db, nByte ); + if( pFKey==0 ){ + goto fk_end; + } + pFKey->pFrom = p; + pFKey->pNextFrom = p->pFKey; + z = (char*)&pFKey->aCol[nCol]; + pFKey->zTo = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)z, pTo); + } + memcpy(z, pTo->z, pTo->n); + z[pTo->n] = 0; + sqlite3Dequote(z); + z += pTo->n+1; + pFKey->nCol = nCol; + if( pFromCol==0 ){ + pFKey->aCol[0].iFrom = p->nCol-1; + }else{ + for(i=0; inCol; j++){ + if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zEName)==0 ){ + pFKey->aCol[i].iFrom = j; + break; + } + } + if( j>=p->nCol ){ + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", + pFromCol->a[i].zEName); + goto fk_end; + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, &pFKey->aCol[i], pFromCol->a[i].zEName); + } + } + } + if( pToCol ){ + for(i=0; ia[i].zEName); + pFKey->aCol[i].zCol = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, z, pToCol->a[i].zEName); + } + memcpy(z, pToCol->a[i].zEName, n); + z[n] = 0; + z += n+1; + } + } + pFKey->isDeferred = 0; + pFKey->aAction[0] = (u8)(flags & 0xff); /* ON DELETE action */ + pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ + + assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pFKey->zTo, (void *)pFKey + ); + if( pNextTo==pFKey ){ + sqlite3OomFault(db); + goto fk_end; + } + if( pNextTo ){ + assert( pNextTo->pPrevTo==0 ); + pFKey->pNextTo = pNextTo; + pNextTo->pPrevTo = pFKey; + } + + /* Link the foreign key to the table as the last step. + */ + p->pFKey = pFKey; + pFKey = 0; + +fk_end: + sqlite3DbFree(db, pFKey); +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + sqlite3ExprListDelete(db, pFromCol); + sqlite3ExprListDelete(db, pToCol); +} + +/* +** This routine is called when an INITIALLY IMMEDIATE or INITIALLY DEFERRED +** clause is seen as part of a foreign key definition. The isDeferred +** parameter is 1 for INITIALLY DEFERRED and 0 for INITIALLY IMMEDIATE. +** The behavior of the most recently created foreign key is adjusted +** accordingly. +*/ +void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ +#ifndef SQLITE_OMIT_FOREIGN_KEY + Table *pTab; + FKey *pFKey; + if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; + assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ + pFKey->isDeferred = (u8)isDeferred; +#endif +} + +/* +** Generate code that will erase and refill index *pIdx. This is +** used to initialize a newly created index or to recompute the +** content of an index in response to a REINDEX command. +** +** if memRootPage is not negative, it means that the index is newly +** created. The register specified by memRootPage contains the +** root page number of the index. If memRootPage is negative, then +** the index already exists and must be cleared before being refilled and +** the root page number of the index is taken from pIndex->tnum. +*/ +static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ + Table *pTab = pIndex->pTable; /* The table that is indexed */ + int iTab = pParse->nTab++; /* Btree cursor used for pTab */ + int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ + int iSorter; /* Cursor opened by OpenSorter (if in use) */ + int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ + Pgno tnum; /* Root page of index */ + int iPartIdxLabel; /* Jump to this label to skip a row */ + Vdbe *v; /* Generate code into this virtual machine */ + KeyInfo *pKey; /* KeyInfo for index */ + int regRecord; /* Register holding assembled index record */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, + db->aDb[iDb].zDbSName ) ){ + return; + } +#endif + + /* Require a write-lock on the table to perform this operation */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + + v = sqlite3GetVdbe(pParse); + if( v==0 ) return; + if( memRootPage>=0 ){ + tnum = (Pgno)memRootPage; + }else{ + tnum = pIndex->tnum; + } + pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + + /* Open the sorter cursor if we are to use one. */ + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) + sqlite3KeyInfoRef(pKey), P4_KEYINFO); + + /* Open the table. Loop through all rows of the table, inserting index + ** records into the sorter. */ + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); + regRecord = sqlite3GetTempReg(pParse); + sqlite3MultiWrite(pParse); + + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, (int)tnum, iDb, + (char *)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); + + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); + if( IsUniqueIndex(pIndex) ){ + int j2 = sqlite3VdbeGoto(v, 1); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyAbortable(v, OE_Abort); + sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, + pIndex->nKeyCol); VdbeCoverage(v); + sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); + sqlite3VdbeJumpHere(v, j2); + }else{ + /* Most CREATE INDEX and REINDEX statements that are not UNIQUE can not + ** abort. The exception is if one of the indexed expressions contains a + ** user function that throws an exception when it is evaluated. But the + ** overhead of adding a statement journal to a CREATE INDEX statement is + ** very small (since most of the pages written do not contain content that + ** needs to be restored if the statement aborts), so we call + ** sqlite3MayAbort() for all CREATE INDEX statements. */ + sqlite3MayAbort(pParse); + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); + if( !pIndex->bAscKeyBug ){ + /* This OP_SeekEnd opcode makes index insert for a REINDEX go much + ** faster by avoiding unnecessary seeks. But the optimization does + ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables + ** with DESC primary keys, since those indexes have there keys in + ** a different order from the main table. + ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + */ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + + sqlite3VdbeAddOp1(v, OP_Close, iTab); + sqlite3VdbeAddOp1(v, OP_Close, iIdx); + sqlite3VdbeAddOp1(v, OP_Close, iSorter); +} + +/* +** Allocate heap space to hold an Index object with nCol columns. +** +** Increase the allocation size to provide an extra nExtra bytes +** of 8-byte aligned space after the Index object and return a +** pointer to this extra space in *ppExtra. +*/ +Index *sqlite3AllocateIndexObject( + sqlite3 *db, /* Database connection */ + i16 nCol, /* Total number of columns in the index */ + int nExtra, /* Number of bytes of extra space to alloc */ + char **ppExtra /* Pointer to the "extra" space */ +){ + Index *p; /* Allocated index object */ + int nByte; /* Bytes of space for Index object + arrays */ + + nByte = ROUND8(sizeof(Index)) + /* Index structure */ + ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ + sizeof(i16)*nCol + /* Index.aiColumn */ + sizeof(u8)*nCol); /* Index.aSortOrder */ + p = sqlite3DbMallocZero(db, nByte + nExtra); + if( p ){ + char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); + p->azColl = (const char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->aSortOrder = (u8*)pExtra; + p->nColumn = nCol; + p->nKeyCol = nCol - 1; + *ppExtra = ((char*)p) + nByte; + } + return p; +} + +/* +** If expression list pList contains an expression that was parsed with +** an explicit "NULLS FIRST" or "NULLS LAST" clause, leave an error in +** pParse and return non-zero. Otherwise, return zero. +*/ +int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].bNulls ){ + u8 sf = pList->a[i].sortFlags; + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + (sf==0 || sf==3) ? "FIRST" : "LAST" + ); + return 1; + } + } + } + return 0; +} + +/* +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will +** be NULL for a primary key or an index that is created to satisfy a +** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable +** as the table to be indexed. pParse->pNewTable is a table that is +** currently being constructed by a CREATE TABLE statement. +** +** pList is a list of columns to be indexed. pList will be NULL if this +** is a primary key or unique-constraint on the most recent column added +** to the table currently under construction. +*/ +void sqlite3CreateIndex( + Parse *pParse, /* All information about this parse */ + Token *pName1, /* First part of index name. May be NULL */ + Token *pName2, /* Second part of index name. May be NULL */ + SrcList *pTblName, /* Table to index. Use pParse->pNewTable if 0 */ + ExprList *pList, /* A list of columns to be indexed */ + int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + Token *pStart, /* The CREATE token that begins this statement */ + Expr *pPIWhere, /* WHERE clause for partial indices */ + int sortOrder, /* Sort order of primary key when pList==NULL */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ +){ + Table *pTab = 0; /* Table to be indexed */ + Index *pIndex = 0; /* The index to be created */ + char *zName = 0; /* Name of the index */ + int nName; /* Number of characters in zName */ + int i, j; + DbFixer sFix; /* For assigning database names to pTable */ + int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ + sqlite3 *db = pParse->db; + Db *pDb; /* The specific table containing the indexed database */ + int iDb; /* Index of the database that is being written */ + Token *pName = 0; /* Unqualified name of the index to create */ + struct ExprList_item *pListItem; /* For looping over pList */ + int nExtra = 0; /* Space allocated for zExtra[] */ + int nExtraCol; /* Number of extra columns needed */ + char *zExtra = 0; /* Extra space after the Index object */ + Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ + + if( db->mallocFailed || pParse->nErr>0 ){ + goto exit_create_index; + } + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ + goto exit_create_index; + } + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_create_index; + } + if( sqlite3HasExplicitNulls(pParse, pList) ){ + goto exit_create_index; + } + + /* + ** Find the table that is to be indexed. Return early if not found. + */ + if( pTblName!=0 ){ + + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. + */ + assert( pName1 && pName2 ); + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); + +#ifndef SQLITE_OMIT_TEMPDB + /* If the index name was unqualified, check if the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. + */ + if( !db->init.busy ){ + pTab = sqlite3SrcListLookup(pParse, pTblName); + if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + } +#endif + + sqlite3FixInit(&sFix, pParse, iDb, "index", pName); + if( sqlite3FixSrcList(&sFix, pTblName) ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite3FixSrcList can never fail. */ + assert(0); + } + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; + if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ + sqlite3ErrorMsg(pParse, + "cannot create a TEMP index on non-TEMP table \"%s\"", + pTab->zName); + goto exit_create_index; + } + if( !HasRowid(pTab) ) pPk = sqlite3PrimaryKeyIndex(pTab); + }else{ + assert( pName==0 ); + assert( pStart==0 ); + pTab = pParse->pNewTable; + if( !pTab ) goto exit_create_index; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + } + pDb = &db->aDb[iDb]; + + assert( pTab!=0 ); + assert( pParse->nErr==0 ); + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && db->init.busy==0 + && pTblName!=0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; + } +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "views may not be indexed"); + goto exit_create_index; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be indexed"); + goto exit_create_index; + } +#endif + + /* + ** Find the name of the index. Make sure there is not already another + ** index or table with the same name. + ** + ** Exception: If we are reading the names of permanent indices from the + ** sqlite_schema table (because some other process changed the schema) and + ** one of the index names collides with the name of a temporary table or + ** index, then we will continue to process this index. + ** + ** If pName==0 it means that we are + ** dealing with a primary key or UNIQUE constraint. We have to invent our + ** own name. + */ + if( pName ){ + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName,"index",pTab->zName) ){ + goto exit_create_index; + } + if( !IN_RENAME_OBJECT ){ + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto exit_create_index; + } + } + }else{ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + if( zName==0 ){ + goto exit_create_index; + } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_SPECIAL_PARSE ) zName[7]++; + } + + /* Check for authorization to create an index. + */ +#ifndef SQLITE_OMIT_AUTHORIZATION + if( !IN_RENAME_OBJECT ){ + const char *zDb = pDb->zDbSName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + goto exit_create_index; + } + i = SQLITE_CREATE_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE_CREATE_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ + goto exit_create_index; + } + } +#endif + + /* If pList==0, it means this routine was called to make a primary + ** key out of the last column added to the table under construction. + ** So create a fake list to simulate this. + */ + if( pList==0 ){ + Token prevCol; + Column *pCol = &pTab->aCol[pTab->nCol-1]; + pCol->colFlags |= COLFLAG_UNIQUE; + sqlite3TokenInit(&prevCol, pCol->zName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); + if( pList==0 ) goto exit_create_index; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder, SQLITE_SO_UNDEFINED); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); + if( pParse->nErr ) goto exit_create_index; + } + + /* Figure out how many bytes of space are required to store explicitly + ** specified collation sequence names. + */ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); + } + } + + /* + ** Allocate the index structure. + */ + nName = sqlite3Strlen30(zName); + nExtraCol = pPk ? pPk->nKeyCol : 1; + assert( pList->nExpr + nExtraCol <= 32767 /* Fits in i16 */ ); + pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, + nName + nExtra + 1, &zExtra); + if( db->mallocFailed ){ + goto exit_create_index; + } + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); + pIndex->zName = zExtra; + zExtra += nName + 1; + memcpy(pIndex->zName, zName, nName+1); + pIndex->pTable = pTab; + pIndex->onError = (u8)onError; + pIndex->uniqNotNull = onError!=OE_None; + pIndex->idxType = idxType; + pIndex->pSchema = db->aDb[iDb].pSchema; + pIndex->nKeyCol = pList->nExpr; + if( pPIWhere ){ + sqlite3ResolveSelfReference(pParse, pTab, NC_PartIdx, pPIWhere, 0); + pIndex->pPartIdxWhere = pPIWhere; + pPIWhere = 0; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + + /* Check to see if we should honor DESC requests on index columns + */ + if( pDb->pSchema->file_format>=4 ){ + sortOrderMask = -1; /* Honor DESC */ + }else{ + sortOrderMask = 0; /* Ignore DESC */ + } + + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. + ** + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. + */ + pListItem = pList->a; + if( IN_RENAME_OBJECT ){ + pIndex->aColExpr = pList; + pList = 0; + } + for(i=0; inKeyCol; i++, pListItem++){ + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ + const char *zColl; /* Collation sequence name */ + + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + pIndex->aColExpr = pList; + pList = 0; + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else{ + if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ + pIndex->bHasVCol = 1; + } + } + pIndex->aiColumn[i] = (i16)j; + } + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ + int nColl; + zColl = pListItem->pExpr->u.zToken; + nColl = sqlite3Strlen30(zColl) + 1; + assert( nExtra>=nColl ); + memcpy(zExtra, zColl, nColl); + zColl = zExtra; + zExtra += nColl; + nExtra -= nColl; + }else if( j>=0 ){ + zColl = pTab->aCol[j].zColl; + } + if( !zColl ) zColl = sqlite3StrBINARY; + if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ + goto exit_create_index; + } + pIndex->azColl[i] = zColl; + requestedSortOrder = pListItem->sortFlags & sortOrderMask; + pIndex->aSortOrder[i] = (u8)requestedSortOrder; + } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ + if( pPk ){ + for(j=0; jnKeyCol; j++){ + int x = pPk->aiColumn[j]; + assert( x>=0 ); + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ + pIndex->nColumn--; + }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); + pIndex->aiColumn[i] = x; + pIndex->azColl[i] = pPk->azColl[j]; + pIndex->aSortOrder[i] = pPk->aSortOrder[j]; + i++; + } + } + assert( i==pIndex->nColumn ); + }else{ + pIndex->aiColumn[i] = XN_ROWID; + pIndex->azColl[i] = sqlite3StrBINARY; + } + sqlite3DefaultRowEst(pIndex); + if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + + /* If this index contains every column of its table, then mark + ** it as a covering index */ + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); + if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ + pIndex->isCovering = 1; + for(j=0; jnCol; j++){ + if( j==pTab->iPKey ) continue; + if( sqlite3TableColumnToIndex(pIndex,j)>=0 ) continue; + pIndex->isCovering = 0; + break; + } + } + + if( pTab==pParse->pNewTable ){ + /* This routine has been called to create an automatic index as a + ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or + ** a PRIMARY KEY or UNIQUE clause following the column definitions. + ** i.e. one of: + ** + ** CREATE TABLE t(x PRIMARY KEY, y); + ** CREATE TABLE t(x, y, UNIQUE(x, y)); + ** + ** Either way, check to see if the table already has such an index. If + ** so, don't bother creating this one. This only applies to + ** automatically created indices. Users can do as they wish with + ** explicit indices. + ** + ** Two UNIQUE or PRIMARY KEY constraints are considered equivalent + ** (and thus suppressing the second one) even if they have different + ** sort orders. + ** + ** If there are different collating sequences or if the columns of + ** the constraint occur in different orders, then the constraints are + ** considered distinct and both result in separate indices. + */ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int k; + assert( IsUniqueIndex(pIdx) ); + assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); + assert( IsUniqueIndex(pIndex) ); + + if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; + for(k=0; knKeyCol; k++){ + const char *z1; + const char *z2; + assert( pIdx->aiColumn[k]>=0 ); + if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; + z1 = pIdx->azColl[k]; + z2 = pIndex->azColl[k]; + if( sqlite3StrICmp(z1, z2) ) break; + } + if( k==pIdx->nKeyCol ){ + if( pIdx->onError!=pIndex->onError ){ + /* This constraint creates the same index as a previous + ** constraint specified somewhere in the CREATE TABLE statement. + ** However the ON CONFLICT clauses are different. If both this + ** constraint and the previous equivalent constraint have explicit + ** ON CONFLICT clauses this is an error. Otherwise, use the + ** explicitly specified behavior for the index. + */ + if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ + sqlite3ErrorMsg(pParse, + "conflicting ON CONFLICT clauses specified", 0); + } + if( pIdx->onError==OE_Default ){ + pIdx->onError = pIndex->onError; + } + } + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; + if( IN_RENAME_OBJECT ){ + pIndex->pNext = pParse->pNewIndex; + pParse->pNewIndex = pIndex; + pIndex = 0; + } + goto exit_create_index; + } + } + } + + if( !IN_RENAME_OBJECT ){ + + /* Link the new Index structure to its table and to the other + ** in-memory database structures. + */ + assert( pParse->nErr==0 ); + if( db->init.busy ){ + Index *p; + assert( !IN_SPECIAL_PARSE ); + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + if( sqlite3IndexHasDuplicateRootPage(pIndex) ){ + sqlite3ErrorMsg(pParse, "invalid rootpage"); + pParse->rc = SQLITE_CORRUPT_BKPT; + goto exit_create_index; + } + } + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + sqlite3OomFault(db); + goto exit_create_index; + } + db->mDbFlags |= DBFLAG_SchemaChange; + } + + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_schema table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_schema + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. + */ + else if( HasRowid(pTab) || pTblName!=0 ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; + + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = (Pgno)sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + assert( pName!=0 || pStart==0 ); + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_schema for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q." DFLT_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), 0); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); + } + + sqlite3VdbeJumpHere(v, (int)pIndex->tnum); + } + } + if( db->init.busy || pTblName==0 ){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; + pIndex = 0; + } + else if( IN_RENAME_OBJECT ){ + assert( pParse->pNewIndex==0 ); + pParse->pNewIndex = pIndex; + pIndex = 0; + } + + /* Clean up before exiting */ +exit_create_index: + if( pIndex ) sqlite3FreeIndex(db, pIndex); + if( pTab ){ + /* Ensure all REPLACE indexes on pTab are at the end of the pIndex list. + ** The list was already ordered when this routine was entered, so at this + ** point at most a single index (the newly added index) will be out of + ** order. So we have to reorder at most one index. */ + Index **ppFrom = &pTab->pIndex; + Index *pThis; + for(ppFrom=&pTab->pIndex; (pThis = *ppFrom)!=0; ppFrom=&pThis->pNext){ + Index *pNext; + if( pThis->onError!=OE_Replace ) continue; + while( (pNext = pThis->pNext)!=0 && pNext->onError!=OE_Replace ){ + *ppFrom = pNext; + pThis->pNext = pNext->pNext; + pNext->pNext = pThis; + ppFrom = &pNext->pNext; + } + break; + } +#ifdef SQLITE_DEBUG + /* Verify that all REPLACE indexes really are now at the end + ** of the index list. In other words, no other index type ever + ** comes after a REPLACE index on the list. */ + for(pThis = pTab->pIndex; pThis; pThis=pThis->pNext){ + assert( pThis->onError!=OE_Replace + || pThis->pNext==0 + || pThis->pNext->onError==OE_Replace ); + } +#endif + } + sqlite3ExprDelete(db, pPIWhere); + sqlite3ExprListDelete(db, pList); + sqlite3SrcListDelete(db, pTblName); + sqlite3DbFree(db, zName); +} + +/* +** Fill the Index.aiRowEst[] array with default information - information +** to be used when we have not run the ANALYZE command. +** +** aiRowEst[0] is supposed to contain the number of elements in the index. +** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the +** number of rows in the table that match any particular value of the +** first column of the index. aiRowEst[2] is an estimate of the number +** of rows that match any particular combination of the first 2 columns +** of the index. And so forth. It must always be the case that +* +** aiRowEst[N]<=aiRowEst[N-1] +** aiRowEst[N]>=1 +** +** Apart from that, we have little to go on besides intuition as to +** how aiRowEst[] should be initialized. The numbers generated here +** are based on typical values found in actual indices. +*/ +void sqlite3DefaultRowEst(Index *pIdx){ + /* 10, 9, 8, 7, 6 */ + static const LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + LogEst x; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. + ** + ** 2020-05-27: If some of the stat data is coming from the sqlite_stat1 + ** table but other parts we are having to guess at, then do not let the + ** estimated number of rows in the table be less than 1000 (LogEst 99). + ** Failure to do this can cause the indexes for which we do not have + ** stat1 data to be ignored by the query planner. + */ + x = pIdx->pTable->nRowLogEst; + assert( 99==sqlite3LogEst(1000) ); + if( x<99 ){ + pIdx->pTable->nRowLogEst = x = 99; + } + if( pIdx->pPartIdxWhere!=0 ) x -= 10; assert( 10==sqlite3LogEst(2) ); + a[0] = x; + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); + } + + assert( 0==sqlite3LogEst(1) ); + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; +} + +/* +** This routine will drop an existing named index. This routine +** implements the DROP INDEX statement. +*/ +void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ + Index *pIndex; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed ){ + goto exit_drop_index; + } + assert( pName->nSrc==1 ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_drop_index; + } + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + if( pIndex==0 ){ + if( !ifExists ){ + sqlite3ErrorMsg(pParse, "no such index: %S", pName, 0); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + } + pParse->checkSchema = 1; + goto exit_drop_index; + } + if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){ + sqlite3ErrorMsg(pParse, "index associated with UNIQUE " + "or PRIMARY KEY constraint cannot be dropped", 0); + goto exit_drop_index; + } + iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_INDEX; + Table *pTab = pIndex->pTable; + const char *zDb = db->aDb[iDb].zDbSName; + const char *zTab = SCHEMA_TABLE(iDb); + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + goto exit_drop_index; + } + if( !OMIT_TEMPDB && iDb ) code = SQLITE_DROP_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ + goto exit_drop_index; + } + } +#endif + + /* Generate code to remove the index and from the schema table */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3NestedParse(pParse, + "DELETE FROM %Q." DFLT_SCHEMA_TABLE " WHERE name=%Q AND type='index'", + db->aDb[iDb].zDbSName, pIndex->zName + ); + sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); + sqlite3ChangeCookie(pParse, iDb); + destroyRootPage(pParse, pIndex->tnum, iDb); + sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); + } + +exit_drop_index: + sqlite3SrcListDelete(db, pName); +} + +/* +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. +** +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). +** +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. +** +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. +*/ +void *sqlite3ArrayAllocate( + sqlite3 *db, /* Connection to notify of malloc failures */ + void *pArray, /* Array of objects. Might be reallocated */ + int szEntry, /* Size of each object in the array */ + int *pnEntry, /* Number of objects currently in use */ + int *pIdx /* Write the index of a new slot here */ +){ + char *z; + sqlite3_int64 n = *pIdx = *pnEntry; + if( (n & (n-1))==0 ){ + sqlite3_int64 sz = (n==0) ? 1 : 2*n; + void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); + if( pNew==0 ){ + *pIdx = -1; + return pArray; + } + pArray = pNew; + } + z = (char*)pArray; + memset(&z[n * szEntry], 0, szEntry); + ++*pnEntry; + return pArray; +} + +/* +** Append a new element to the given IdList. Create a new IdList if +** need be. +** +** A new IdList is returned, or NULL if malloc() fails. +*/ +IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ + sqlite3 *db = pParse->db; + int i; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + if( pList==0 ) return 0; + } + pList->a = sqlite3ArrayAllocate( + db, + pList->a, + sizeof(pList->a[0]), + &pList->nId, + &i + ); + if( i<0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList->a[i].zName = sqlite3NameFromToken(db, pToken); + if( IN_RENAME_OBJECT && pList->a[i].zName ){ + sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); + } + return pList; +} + +/* +** Delete an IdList. +*/ +void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ + int i; + if( pList==0 ) return; + for(i=0; inId; i++){ + sqlite3DbFree(db, pList->a[i].zName); + } + sqlite3DbFree(db, pList->a); + sqlite3DbFreeNN(db, pList); +} + +/* +** Return the index in pList of the identifier named zId. Return -1 +** if not found. +*/ +int sqlite3IdListIndex(IdList *pList, const char *zName){ + int i; + if( pList==0 ) return -1; + for(i=0; inId; i++){ + if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; + } + return -1; +} + +/* +** Maximum size of a SrcList object. +** The SrcList object is used to represent the FROM clause of a +** SELECT statement, and the query planner cannot deal with more +** than 64 tables in a join. So any value larger than 64 here +** is sufficient for most uses. Smaller values, like say 10, are +** appropriate for small and memory-limited applications. +*/ +#ifndef SQLITE_MAX_SRCLIST +# define SQLITE_MAX_SRCLIST 200 +#endif + +/* +** Expand the space allocated for the given SrcList object by +** creating nExtra new slots beginning at iStart. iStart is zero based. +** New slots are zeroed. +** +** For example, suppose a SrcList initially contains two entries: A,B. +** To append 3 new entries onto the end, do this: +** +** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); +** +** After the call above it would contain: A, B, nil, nil, nil. +** If the iStart argument had been 1 instead of 2, then the result +** would have been: A, nil, nil, nil, B. To prepend the new slots, +** the iStart value would be 0. The result then would +** be: nil, nil, nil, A, B. +** +** If a memory allocation fails or the SrcList becomes too large, leave +** the original SrcList unchanged, return NULL, and leave an error message +** in pParse. +*/ +SrcList *sqlite3SrcListEnlarge( + Parse *pParse, /* Parsing context into which errors are reported */ + SrcList *pSrc, /* The SrcList to be enlarged */ + int nExtra, /* Number of new slots to add to pSrc->a[] */ + int iStart /* Index in pSrc->a[] of first new slot */ +){ + int i; + + /* Sanity checking on calling parameters */ + assert( iStart>=0 ); + assert( nExtra>=1 ); + assert( pSrc!=0 ); + assert( iStart<=pSrc->nSrc ); + + /* Allocate additional space if needed */ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ + SrcList *pNew; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; + sqlite3 *db = pParse->db; + + if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ + sqlite3ErrorMsg(pParse, "too many FROM clause terms, max: %d", + SQLITE_MAX_SRCLIST); + return 0; + } + if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; + pNew = sqlite3DbRealloc(db, pSrc, + sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + if( pNew==0 ){ + assert( db->mallocFailed ); + return 0; + } + pSrc = pNew; + pSrc->nAlloc = nAlloc; + } + + /* Move existing slots that come after the newly inserted slots + ** out of the way */ + for(i=pSrc->nSrc-1; i>=iStart; i--){ + pSrc->a[i+nExtra] = pSrc->a[i]; + } + pSrc->nSrc += nExtra; + + /* Zero the newly allocated slots */ + memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); + for(i=iStart; ia[i].iCursor = -1; + } + + /* Return a pointer to the enlarged SrcList */ + return pSrc; +} + + +/* +** Append a new table name to the given SrcList. Create a new SrcList if +** need be. A new entry is created in the SrcList even if pTable is NULL. +** +** A SrcList is returned, or NULL if there is an OOM error or if the +** SrcList grows to large. The returned +** SrcList might be the same as the SrcList that was input or it might be +** a new one. If an OOM error does occurs, then the prior value of pList +** that is input to this routine is automatically freed. +** +** If pDatabase is not null, it means that the table has an optional +** database name prefix. Like this: "database.table". The pDatabase +** points to the table name and the pTable points to the database name. +** The SrcList.a[].zName field is filled with the table name which might +** come from pTable (if pDatabase is NULL) or from pDatabase. +** SrcList.a[].zDatabase is filled with the database name from pTable, +** or with NULL if no database is specified. +** +** In other words, if call like this: +** +** sqlite3SrcListAppend(D,A,B,0); +** +** Then B is a table name and the database name is unspecified. If called +** like this: +** +** sqlite3SrcListAppend(D,A,B,C); +** +** Then C is the table name and B is the database name. If C is defined +** then so is B. In other words, we never have a case where: +** +** sqlite3SrcListAppend(D,A,0,C); +** +** Both pTable and pDatabase are assumed to be quoted. They are dequoted +** before being added to the SrcList. +*/ +SrcList *sqlite3SrcListAppend( + Parse *pParse, /* Parsing context, in which errors are reported */ + SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ + Token *pTable, /* Table to append */ + Token *pDatabase /* Database of the table */ +){ + SrcItem *pItem; + sqlite3 *db; + assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + assert( pParse!=0 ); + assert( pParse->db!=0 ); + db = pParse->db; + if( pList==0 ){ + pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + if( pList==0 ) return 0; + pList->nAlloc = 1; + pList->nSrc = 1; + memset(&pList->a[0], 0, sizeof(pList->a[0])); + pList->a[0].iCursor = -1; + }else{ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, pList, 1, pList->nSrc); + if( pNew==0 ){ + sqlite3SrcListDelete(db, pList); + return 0; + }else{ + pList = pNew; + } + } + pItem = &pList->a[pList->nSrc-1]; + if( pDatabase && pDatabase->z==0 ){ + pDatabase = 0; + } + if( pDatabase ){ + pItem->zName = sqlite3NameFromToken(db, pDatabase); + pItem->zDatabase = sqlite3NameFromToken(db, pTable); + }else{ + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->zDatabase = 0; + } + return pList; +} + +/* +** Assign VdbeCursor index numbers to all tables in a SrcList +*/ +void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ + int i; + SrcItem *pItem; + assert(pList || pParse->db->mallocFailed ); + if( pList ){ + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pItem->iCursor>=0 ) continue; + pItem->iCursor = pParse->nTab++; + if( pItem->pSelect ){ + sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + } + } + } +} + +/* +** Delete an entire SrcList including all its substructure. +*/ +void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ + int i; + SrcItem *pItem; + if( pList==0 ) return; + for(pItem=pList->a, i=0; inSrc; i++, pItem++){ + if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); + sqlite3DeleteTable(db, pItem->pTab); + if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + if( pItem->pOn ) sqlite3ExprDelete(db, pItem->pOn); + if( pItem->pUsing ) sqlite3IdListDelete(db, pItem->pUsing); + } + sqlite3DbFreeNN(db, pList); +} + +/* +** This routine is called by the parser to add a new term to the +** end of a growing FROM clause. The "p" parameter is the part of +** the FROM clause that has already been constructed. "p" is NULL +** if this is the first term of the FROM clause. pTable and pDatabase +** are the name of the table and database named in the FROM clause term. +** pDatabase is NULL if the database name qualifier is missing - the +** usual case. If the term has an alias, then pAlias points to the +** alias token. If the term is a subquery, then pSubquery is the +** SELECT statement that the subquery encodes. The pTable and +** pDatabase parameters are NULL for subqueries. The pOn and pUsing +** parameters are the content of the ON and USING clauses. +** +** Return a new SrcList which encodes is the FROM with the new +** term added. +*/ +SrcList *sqlite3SrcListAppendFromTerm( + Parse *pParse, /* Parsing context */ + SrcList *p, /* The left part of the FROM clause already seen */ + Token *pTable, /* Name of the table to add to the FROM clause */ + Token *pDatabase, /* Name of the database containing pTable */ + Token *pAlias, /* The right-hand side of the AS subexpression */ + Select *pSubquery, /* A subquery used in place of a table name */ + Expr *pOn, /* The ON clause of a join */ + IdList *pUsing /* The USING clause of a join */ +){ + SrcItem *pItem; + sqlite3 *db = pParse->db; + if( !p && (pOn || pUsing) ){ + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + (pOn ? "ON" : "USING") + ); + goto append_from_error; + } + p = sqlite3SrcListAppend(pParse, p, pTable, pDatabase); + if( p==0 ){ + goto append_from_error; + } + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; + assert( (pTable==0)==(pDatabase==0) ); + assert( pItem->zName==0 || pDatabase!=0 ); + if( IN_RENAME_OBJECT && pItem->zName ){ + Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable; + sqlite3RenameTokenMap(pParse, pItem->zName, pToken); + } + assert( pAlias!=0 ); + if( pAlias->n ){ + pItem->zAlias = sqlite3NameFromToken(db, pAlias); + } + pItem->pSelect = pSubquery; + pItem->pOn = pOn; + pItem->pUsing = pUsing; + return p; + + append_from_error: + assert( p==0 ); + sqlite3ExprDelete(db, pOn); + sqlite3IdListDelete(db, pUsing); + sqlite3SelectDelete(db, pSubquery); + return 0; +} + +/* +** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** element of the source-list passed as the second argument. +*/ +void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ + assert( pIndexedBy!=0 ); + if( p && pIndexedBy->n>0 ){ + SrcItem *pItem; + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + if( pIndexedBy->n==1 && !pIndexedBy->z ){ + /* A "NOT INDEXED" clause was supplied. See parse.y + ** construct "indexed_opt" for details. */ + pItem->fg.notIndexed = 1; + }else{ + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = 1; + } + } +} + +/* +** Append the contents of SrcList p2 to SrcList p1 and return the resulting +** SrcList. Or, if an error occurs, return NULL. In all cases, p1 and p2 +** are deleted by this function. +*/ +SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ + assert( p1 && p1->nSrc==1 ); + if( p2 ){ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + if( pNew==0 ){ + sqlite3SrcListDelete(pParse->db, p2); + }else{ + p1 = pNew; + memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + sqlite3DbFree(pParse->db, p2); + } + } + return p1; +} + +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p ){ + SrcItem *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + +/* +** When building up a FROM clause in the parser, the join operator +** is initially attached to the left operand. But the code generator +** expects the join operator to be on the right operand. This routine +** Shifts all join operators from left to right for an entire FROM +** clause. +** +** Example: Suppose the join is like this: +** +** A natural cross join B +** +** The operator is "natural cross join". The A and B operands are stored +** in p->a[0] and p->a[1], respectively. The parser initially stores the +** operator with A. This routine shifts that operator over to B. +*/ +void sqlite3SrcListShiftJoinType(SrcList *p){ + if( p ){ + int i; + for(i=p->nSrc-1; i>0; i--){ + p->a[i].fg.jointype = p->a[i-1].fg.jointype; + } + p->a[0].fg.jointype = 0; + } +} + +/* +** Generate VDBE code for a BEGIN statement. +*/ +void sqlite3BeginTransaction(Parse *pParse, int type){ + sqlite3 *db; + Vdbe *v; + int i; + + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( !v ) return; + if( type!=TK_DEFERRED ){ + for(i=0; inDb; i++){ + int eTxnType; + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsReadonly(pBt) ){ + eTxnType = 0; /* Read txn */ + }else if( type==TK_EXCLUSIVE ){ + eTxnType = 2; /* Exclusive txn */ + }else{ + eTxnType = 1; /* Write txn */ + } + sqlite3VdbeAddOp2(v, OP_Transaction, i, eTxnType); + sqlite3VdbeUsesBtree(v, i); + } + } + sqlite3VdbeAddOp0(v, OP_AutoCommit); +} + +/* +** Generate VDBE code for a COMMIT or ROLLBACK statement. +** Code for ROLLBACK is generated if eType==TK_ROLLBACK. Otherwise +** code is generated for a COMMIT. +*/ +void sqlite3EndTransaction(Parse *pParse, int eType){ + Vdbe *v; + int isRollback; + + assert( pParse!=0 ); + assert( pParse->db!=0 ); + assert( eType==TK_COMMIT || eType==TK_END || eType==TK_ROLLBACK ); + isRollback = eType==TK_ROLLBACK; + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, + isRollback ? "ROLLBACK" : "COMMIT", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, isRollback); + } +} + +/* +** This function is called by the parser when it parses a command to create, +** release or rollback an SQL savepoint. +*/ +void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + if( zName ){ + Vdbe *v = sqlite3GetVdbe(pParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + static const char * const az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; + assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); +#endif + if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ + sqlite3DbFree(pParse->db, zName); + return; + } + sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); + } +} + +/* +** Make sure the TEMP database is open and available for use. Return +** the number of errors. Leave any error messages in the pParse structure. +*/ +int sqlite3OpenTempDatabase(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt==0 && !pParse->explain ){ + int rc; + Btree *pBt; + static const int flags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TEMP_DB; + + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "unable to open a temporary database " + "file for storing temporary tables"); + pParse->rc = rc; + return 1; + } + db->aDb[1].pBt = pBt; + assert( db->aDb[1].pSchema ); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, 0, 0) ){ + sqlite3OomFault(db); + return 1; + } + } + return 0; +} + +/* +** Record the fact that the schema cookie will need to be verified +** for database iDb. The code to actually verify the schema cookie +** will occur at the end of the top-level VDBE and will be generated +** later, by sqlite3FinishCoding(). +*/ +static void sqlite3CodeVerifySchemaAtToplevel(Parse *pToplevel, int iDb){ + assert( iDb>=0 && iDbdb->nDb ); + assert( pToplevel->db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbdb, iDb, 0) ); + if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); + } + } +} +void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ + sqlite3CodeVerifySchemaAtToplevel(sqlite3ParseToplevel(pParse), iDb); +} + + +/* +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** attached database. Otherwise, invoke it for the database named zDb only. +*/ +void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + int i; + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ + sqlite3CodeVerifySchema(pParse, i); + } + } +} + +/* +** Generate VDBE code that prepares for doing an operation that +** might change the database. +** +** This routine starts a new transaction if we are not already within +** a transaction. If we are already within a transaction, then a checkpoint +** is set if the setStatement parameter is true. A checkpoint should +** be set for operations that might fail (due to a constraint) part of +** the way through and which will need to undo some writes without having to +** rollback the whole transaction. For operations where all constraints +** can be checked before any changes are made to the database, it is never +** necessary to undo a write and the checkpoint should not be set. +*/ +void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3CodeVerifySchemaAtToplevel(pToplevel, iDb); + DbMaskSet(pToplevel->writeMask, iDb); + pToplevel->isMultiWrite |= setStatement; +} + +/* +** Indicate that the statement currently under construction might write +** more than one entry (example: deleting one row then inserting another, +** inserting multiple rows in a table, or inserting a row and index entries.) +** If an abort occurs after some of these writes have completed, then it will +** be necessary to undo the completed writes. +*/ +void sqlite3MultiWrite(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->isMultiWrite = 1; +} + +/* +** The code generator calls this routine if is discovers that it is +** possible to abort a statement prior to completion. In order to +** perform this abort without corrupting the database, we need to make +** sure that the statement is protected by a statement transaction. +** +** Technically, we only need to set the mayAbort flag if the +** isMultiWrite flag was previously set. There is a time dependency +** such that the abort must occur after the multiwrite. This makes +** some statements involving the REPLACE conflict resolution algorithm +** go a little faster. But taking advantage of this time dependency +** makes it more difficult to prove that the code is correct (in +** particular, it prevents us from writing an effective +** implementation of sqlite3AssertMayAbort()) and so we have chosen +** to take the safe route and skip the optimization. +*/ +void sqlite3MayAbort(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->mayAbort = 1; +} + +/* +** Code an OP_Halt that causes the vdbe to return an SQLITE_CONSTRAINT +** error. The onError parameter determines which (if any) of the statement +** and/or current transaction is rolled back. +*/ +void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + i8 p4type, /* P4_STATIC or P4_TRANSIENT */ + u8 p5Errmsg /* P5_ErrMsg type */ +){ + Vdbe *v; + assert( pParse->pVdbe!=0 ); + v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT || pParse->nested ); + if( onError==OE_Abort ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); + sqlite3VdbeChangeP5(v, p5Errmsg); +} + +/* +** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation. +*/ +void sqlite3UniqueConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Constraint type */ + Index *pIdx /* The index that triggers the constraint */ +){ + char *zErr; + int j; + StrAccum errMsg; + Table *pTab = pIdx->pTable; + + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); + if( pIdx->aColExpr ){ + sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3_str_append(&errMsg, ", ", 2); + sqlite3_str_appendall(&errMsg, pTab->zName); + sqlite3_str_append(&errMsg, ".", 1); + sqlite3_str_appendall(&errMsg, zCol); + } + } + zErr = sqlite3StrAccumFinish(&errMsg); + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + : SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); +} + + +/* +** Code an OP_Halt due to non-unique rowid. +*/ +void sqlite3RowidConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Conflict resolution algorithm */ + Table *pTab /* The table with the non-unique rowid */ +){ + char *zMsg; + int rc; + if( pTab->iPKey>=0 ){ + zMsg = sqlite3MPrintf(pParse->db, "%s.%s", pTab->zName, + pTab->aCol[pTab->iPKey].zName); + rc = SQLITE_CONSTRAINT_PRIMARYKEY; + }else{ + zMsg = sqlite3MPrintf(pParse->db, "%s.rowid", pTab->zName); + rc = SQLITE_CONSTRAINT_ROWID; + } + sqlite3HaltConstraint(pParse, rc, onError, zMsg, P4_DYNAMIC, + P5_ConstraintUnique); +} + +/* +** Check to see if pIndex uses the collating sequence pColl. Return +** true if it does and false if it does not. +*/ +#ifndef SQLITE_OMIT_REINDEX +static int collationMatch(const char *zColl, Index *pIndex){ + int i; + assert( zColl!=0 ); + for(i=0; inColumn; i++){ + const char *z = pIndex->azColl[i]; + assert( z!=0 || pIndex->aiColumn[i]<0 ); + if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + return 1; + } + } + return 0; +} +#endif + +/* +** Recompute all indices of pTab that use the collating sequence pColl. +** If pColl==0 then recompute all indices of pTab. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ + if( !IsVirtual(pTab) ){ + Index *pIndex; /* An index associated with pTab */ + + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + } + } + } +} +#endif + +/* +** Recompute all indices of all tables in all databases where the +** indices use the collating sequence pColl. If pColl==0 then recompute +** all indices everywhere. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexDatabases(Parse *pParse, char const *zColl){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ + + assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + assert( pDb!=0 ); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + reindexTable(pParse, pTab, zColl); + } + } +} +#endif + +/* +** Generate code for the REINDEX command. +** +** REINDEX -- 1 +** REINDEX -- 2 +** REINDEX ?.? -- 3 +** REINDEX ?.? -- 4 +** +** Form 1 causes all indices in all attached databases to be rebuilt. +** Form 2 rebuilds all indices in all databases that use the named +** collating function. Forms 3 and 4 rebuild the named index or all +** indices associated with the named table. +*/ +#ifndef SQLITE_OMIT_REINDEX +void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ + CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ + char *z; /* Name of a table or index */ + const char *zDb; /* Name of the database */ + Table *pTab; /* A table in the database */ + Index *pIndex; /* An index associated with pTab */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + Token *pObjName; /* Name of the table or index to be reindexed */ + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + if( pName1==0 ){ + reindexDatabases(pParse, 0); + return; + }else if( NEVER(pName2==0) || pName2->z==0 ){ + char *zColl; + assert( pName1->z ); + zColl = sqlite3NameFromToken(pParse->db, pName1); + if( !zColl ) return; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + if( pColl ){ + reindexDatabases(pParse, zColl); + sqlite3DbFree(db, zColl); + return; + } + sqlite3DbFree(db, zColl); + } + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iDb].zDbSName; + pTab = sqlite3FindTable(db, z, zDb); + if( pTab ){ + reindexTable(pParse, pTab, 0); + sqlite3DbFree(db, z); + return; + } + pIndex = sqlite3FindIndex(db, z, zDb); + sqlite3DbFree(db, z); + if( pIndex ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + return; + } + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); +} +#endif + +/* +** Return a KeyInfo structure that is appropriate for the given Index. +** +** The caller should invoke sqlite3KeyInfoUnref() on the returned object +** when it has finished using it. +*/ +KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; + if( pParse->nErr ) return 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + } + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortFlags[i] = pIdx->aSortOrder[i]; + assert( 0==(pKey->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ); + } + if( pParse->nErr ){ + assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); + if( pIdx->bNoQuery==0 ){ + /* Deactivate the index because it contains an unknown collating + ** sequence. The only way to reactive the index is to reload the + ** schema. Adding the missing collating sequence later does not + ** reactive the index. The application had the chance to register + ** the missing index using the collation-needed callback. For + ** simplicity, SQLite will not give the application a second chance. + */ + pIdx->bNoQuery = 1; + pParse->rc = SQLITE_ERROR_RETRY; + } + sqlite3KeyInfoUnref(pKey); + pKey = 0; + } + } + return pKey; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Create a new CTE object +*/ +Cte *sqlite3CteNew( + Parse *pParse, /* Parsing context */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery, /* Query used to initialize the table */ + u8 eM10d /* The MATERIALIZED flag */ +){ + Cte *pNew; + sqlite3 *db = pParse->db; + + pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); + assert( pNew!=0 || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + }else{ + pNew->pSelect = pQuery; + pNew->pCols = pArglist; + pNew->zName = sqlite3NameFromToken(pParse->db, pName); + pNew->eM10d = eM10d; + } + return pNew; +} + +/* +** Clear information from a Cte object, but do not deallocate storage +** for the object itself. +*/ +static void cteClear(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); +} + +/* +** Free the contents of the CTE object passed as the second argument. +*/ +void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + cteClear(db, pCte); + sqlite3DbFree(db, pCte); +} + +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. The CTE described by teh third argument is added to +** the WITH clause of the second argument. If the second argument is +** NULL, then a new WITH argument is created. +*/ +With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Cte *pCte /* CTE to add to the WITH clause */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + if( pCte==0 ){ + return pWith; + } + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = pCte->zName; + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } + } + } + + if( pWith ){ + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( (pNew!=0 && zName!=0) || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3CteDelete(db, pCte); + pNew = pWith; + }else{ + pNew->a[pNew->nCte++] = *pCte; + sqlite3DbFree(db, pCte); + } + + return pNew; +} + +/* +** Free the contents of the With object passed as the second argument. +*/ +void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + cteClear(db, &pWith->a[i]); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/third_party/sqlite3/callback.c b/third_party/sqlite3/callback.c new file mode 100644 index 000000000..cf429bb22 --- /dev/null +++ b/third_party/sqlite3/callback.c @@ -0,0 +1,537 @@ +/* +** 2005 May 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains functions used to access the internal hash tables +** of user defined functions and collation sequences. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +/* +** Invoke the 'collation needed' callback to request a collation sequence +** in the encoding enc of name zName, length nName. +*/ +static void callCollNeeded(sqlite3 *db, int enc, const char *zName){ + assert( !db->xCollNeeded || !db->xCollNeeded16 ); + if( db->xCollNeeded ){ + char *zExternal = sqlite3DbStrDup(db, zName); + if( !zExternal ) return; + db->xCollNeeded(db->pCollNeededArg, db, enc, zExternal); + sqlite3DbFree(db, zExternal); + } +#ifndef SQLITE_OMIT_UTF16 + if( db->xCollNeeded16 ){ + char const *zExternal; + sqlite3_value *pTmp = sqlite3ValueNew(db); + sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC); + zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); + if( zExternal ){ + db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); + } + sqlite3ValueFree(pTmp); + } +#endif +} + +/* +** This routine is called if the collation factory fails to deliver a +** collation function in the best encoding but there may be other versions +** of this collation function (for other text encodings) available. Use one +** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if +** possible. +*/ +static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ + CollSeq *pColl2; + char *z = pColl->zName; + int i; + static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; + for(i=0; i<3; i++){ + pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, 0); + if( pColl2->xCmp!=0 ){ + memcpy(pColl, pColl2, sizeof(CollSeq)); + pColl->xDel = 0; /* Do not copy the destructor */ + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +/* +** This routine is called on a collation sequence before it is used to +** check that it is defined. An undefined collation sequence exists when +** a database is loaded that contains references to collation sequences +** that have not been defined by sqlite3_create_collation() etc. +** +** If required, this routine calls the 'collation needed' callback to +** request a definition of the collating sequence. If this doesn't work, +** an equivalent collating sequence that uses a text encoding different +** from the main database is substituted, if one is available. +*/ +int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ + if( pColl && pColl->xCmp==0 ){ + const char *zName = pColl->zName; + sqlite3 *db = pParse->db; + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); + if( !p ){ + return SQLITE_ERROR; + } + assert( p==pColl ); + } + return SQLITE_OK; +} + + + +/* +** Locate and return an entry from the db.aCollSeq hash table. If the entry +** specified by zName and nName is not found and parameter 'create' is +** true, then create a new entry. Otherwise return NULL. +** +** Each pointer stored in the sqlite3.aCollSeq hash table contains an +** array of three CollSeq structures. The first is the collation sequence +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** +** Stored immediately after the three collation sequences is a copy of +** the collation sequence name. A pointer to this string is stored in +** each collation sequence structure. +*/ +static CollSeq *findCollSeqEntry( + sqlite3 *db, /* Database connection */ + const char *zName, /* Name of the collating sequence */ + int create /* Create a new entry if true */ +){ + CollSeq *pColl; + pColl = sqlite3HashFind(&db->aCollSeq, zName); + + if( 0==pColl && create ){ + int nName = sqlite3Strlen30(zName) + 1; + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName); + if( pColl ){ + CollSeq *pDel = 0; + pColl[0].zName = (char*)&pColl[3]; + pColl[0].enc = SQLITE_UTF8; + pColl[1].zName = (char*)&pColl[3]; + pColl[1].enc = SQLITE_UTF16LE; + pColl[2].zName = (char*)&pColl[3]; + pColl[2].enc = SQLITE_UTF16BE; + memcpy(pColl[0].zName, zName, nName); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); + + /* If a malloc() failure occurred in sqlite3HashInsert(), it will + ** return the pColl pointer to be deleted (because it wasn't added + ** to the hash table). + */ + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + sqlite3OomFault(db); + sqlite3DbFree(db, pDel); + pColl = 0; + } + } + } + return pColl; +} + +/* +** Parameter zName points to a UTF-8 encoded string nName bytes long. +** Return the CollSeq* pointer for the collation sequence named zName +** for the encoding 'enc' from the database 'db'. +** +** If the entry specified is not found and 'create' is true, then create a +** new entry. Otherwise return NULL. +** +** A separate function sqlite3LocateCollSeq() is a wrapper around +** this routine. sqlite3LocateCollSeq() invokes the collation factory +** if necessary and generates an error message if the collating sequence +** cannot be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +*/ +CollSeq *sqlite3FindCollSeq( + sqlite3 *db, /* Database connection to search */ + u8 enc, /* Desired text encoding */ + const char *zName, /* Name of the collating sequence. Might be NULL */ + int create /* True to create CollSeq if doesn't already exist */ +){ + CollSeq *pColl; + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); + if( zName ){ + pColl = findCollSeqEntry(db, zName, create); + if( pColl ) pColl += enc-1; + }else{ + pColl = db->pDfltColl; + } + return pColl; +} + +/* +** Change the text encoding for a database connection. This means that +** the pDfltColl must change as well. +*/ +void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + db->enc = enc; + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); +} + +/* +** This function is responsible for invoking the collation factory callback +** or substituting a collation sequence of a different encoding when the +** requested collation sequence is not available in the desired encoding. +** +** If it is not NULL, then pColl must point to the database native encoding +** collation sequence with name zName, length nName. +** +** The return value is either the collation sequence to be used in database +** db for collation type name zName, length nName, or NULL, if no collation +** sequence can be found. If no collation is found, leave an error message. +** +** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() +*/ +CollSeq *sqlite3GetCollSeq( + Parse *pParse, /* Parsing context */ + u8 enc, /* The desired encoding for the collating sequence */ + CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ + const char *zName /* Collating sequence name */ +){ + CollSeq *p; + sqlite3 *db = pParse->db; + + p = pColl; + if( !p ){ + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( !p || !p->xCmp ){ + /* No collation sequence of this type for this encoding is registered. + ** Call the collation factory to see if it can supply us with one. + */ + callCollNeeded(db, enc, zName); + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( p && !p->xCmp && synthCollSeq(db, p) ){ + p = 0; + } + assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + pParse->rc = SQLITE_ERROR_MISSING_COLLSEQ; + } + return p; +} + +/* +** This function returns the collation sequence for database native text +** encoding identified by the string zName. +** +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. +** +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. +** +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +*/ +CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; + CollSeq *pColl; + + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); + } + + return pColl; +} + +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** +** The returned value is always between 0 and 6, as follows: +** +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. +** +** If nArg==(-2) then any function with a non-null xSFunc is +** a perfect match and any function with xSFunc NULL is +** a non-match. +*/ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + assert( p->nArg>=-1 ); + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg ){ + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( p->nArg>=0 ) return 0; + } + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ + match = 1; + } + + /* Bonus points if the text encoding matches */ + if( enc==(p->funcFlags & SQLITE_FUNC_ENCMASK) ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->funcFlags & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + + return match; +} + +/* +** Search a FuncDefHash for a function with the given name. Return +** a pointer to the matching FuncDef if found, or 0 if there is no match. +*/ +FuncDef *sqlite3FunctionSearch( + int h, /* Hash of the name */ + const char *zFunc /* Name of function */ +){ + FuncDef *p; + for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + if( sqlite3StrICmp(p->zName, zFunc)==0 ){ + return p; + } + } + return 0; +} + +/* +** Insert a new FuncDef into a FuncDefHash hash table. +*/ +void sqlite3InsertBuiltinFuncs( + FuncDef *aDef, /* List of global functions to be inserted */ + int nDef /* Length of the apDef[] list */ +){ + int i; + for(i=0; i='a' && zName[0]<='z' ); + pOther = sqlite3FunctionSearch(h, zName); + if( pOther ){ + assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); + aDef[i].pNext = pOther->pNext; + pOther->pNext = &aDef[i]; + }else{ + aDef[i].pNext = 0; + aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; + sqlite3BuiltinFunctions.a[h] = &aDef[i]; + } + } +} + + + +/* +** Locate a user function given a name, a number of arguments and a flag +** indicating whether the function prefers UTF-16 over UTF-8. Return a +** pointer to the FuncDef structure that defines that function, or return +** NULL if the function does not exist. +** +** If the createFlag argument is true, then a new (blank) FuncDef +** structure is created and liked into the "db" structure if a +** no matching function previously existed. +** +** If nArg is -2, then the first valid function found is returned. A +** function is valid if xSFunc is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. +** +** If createFlag is false, then a function with the required name and +** number of arguments may be returned even if the eTextRep flag does not +** match that requested. +*/ +FuncDef *sqlite3FindFunction( + sqlite3 *db, /* An open database */ + const char *zName, /* Name of the function. zero-terminated */ + int nArg, /* Number of arguments. -1 means any number */ + u8 enc, /* Preferred text encoding */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ +){ + FuncDef *p; /* Iterator variable */ + FuncDef *pBest = 0; /* Best match found so far */ + int bestScore = 0; /* Score of best match */ + int h; /* Hash value */ + int nName; /* Length of the name */ + + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); + nName = sqlite3Strlen30(zName); + + /* First search for a match amongst the application-defined functions. + */ + p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + + /* If no match is found, search the built-in functions. + ** + ** If the DBFLAG_PreferBuiltin flag is set, then search the built-in + ** functions even if a prior app-defined function was found. And give + ** priority to built-in functions. + ** + ** Except, if createFlag is true, that means that we are trying to + ** install a new function. Whatever FuncDef structure is returned it will + ** have fields overwritten with new information appropriate for the + ** new function. But the FuncDefs for built-in functions are read-only. + ** So we must not search for built-ins when creating a new function. + */ + if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ + bestScore = 0; + h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); + p = sqlite3FunctionSearch(h, zName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + } + + /* If the createFlag parameter is true and the search did not reveal an + ** exact match for the name, number of arguments and encoding, then add a + ** new entry to the hash table and return it. + */ + if( createFlag && bestScorezName = (const char*)&pBest[1]; + pBest->nArg = (u16)nArg; + pBest->funcFlags = enc; + memcpy((char*)&pBest[1], zName, nName+1); + for(z=(u8*)pBest->zName; *z; z++) *z = sqlite3UpperToLower[*z]; + pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); + if( pOther==pBest ){ + sqlite3DbFree(db, pBest); + sqlite3OomFault(db); + return 0; + }else{ + pBest->pNext = pOther; + } + } + + if( pBest && (pBest->xSFunc || createFlag) ){ + return pBest; + } + return 0; +} + +/* +** Free all resources held by the schema structure. The void* argument points +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the +** pointer itself, it just cleans up subsidiary resources (i.e. the contents +** of the schema hash tables). +** +** The Schema.cache_size variable is not cleared. +*/ +void sqlite3SchemaClear(void *p){ + Hash temp1; + Hash temp2; + HashElem *pElem; + Schema *pSchema = (Schema *)p; + + temp1 = pSchema->tblHash; + temp2 = pSchema->trigHash; + sqlite3HashInit(&pSchema->trigHash); + sqlite3HashClear(&pSchema->idxHash); + for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ + sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + } + sqlite3HashClear(&temp2); + sqlite3HashInit(&pSchema->tblHash); + for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + sqlite3DeleteTable(0, pTab); + } + sqlite3HashClear(&temp1); + sqlite3HashClear(&pSchema->fkeyHash); + pSchema->pSeqTab = 0; + if( pSchema->schemaFlags & DB_SchemaLoaded ){ + pSchema->iGeneration++; + } + pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); +} + +/* +** Find and return the schema associated with a BTree. Create +** a new one if necessary. +*/ +Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ + Schema * p; + if( pBt ){ + p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaClear); + }else{ + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); + } + if( !p ){ + sqlite3OomFault(db); + }else if ( 0==p->file_format ){ + sqlite3HashInit(&p->tblHash); + sqlite3HashInit(&p->idxHash); + sqlite3HashInit(&p->trigHash); + sqlite3HashInit(&p->fkeyHash); + p->enc = SQLITE_UTF8; + } + return p; +} diff --git a/third_party/sqlite3/complete.c b/third_party/sqlite3/complete.c new file mode 100644 index 000000000..dd626aa91 --- /dev/null +++ b/third_party/sqlite3/complete.c @@ -0,0 +1,293 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +extern const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +int sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +int sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3ValueFree(pVal); + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ diff --git a/third_party/sqlite3/ctime.c b/third_party/sqlite3/ctime.c new file mode 100644 index 000000000..cc42ed757 --- /dev/null +++ b/third_party/sqlite3/ctime.c @@ -0,0 +1,757 @@ +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ +/* clang-format off */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ + +/* These macros are provided to "stringify" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const sqlite3azCompileOpt[] = { + +/* +** BEGIN CODE GENERATED BY tool/mkctime.tcl +*/ +#if SQLITE_32BIT_ROWID + "32BIT_ROWID", +#endif +#if SQLITE_4_BYTE_ALIGNED_MALLOC + "4_BYTE_ALIGNED_MALLOC", +#endif +#if SQLITE_64BIT_STATS + "64BIT_STATS", +#endif +#if SQLITE_ALLOW_COVERING_INDEX_SCAN + "ALLOW_COVERING_INDEX_SCAN", +#endif +#if SQLITE_ALLOW_URI_AUTHORITY + "ALLOW_URI_AUTHORITY", +#endif +#ifdef SQLITE_BITMASK_TYPE + "BITMASK_TYPE=" CTIMEOPT_VAL(SQLITE_BITMASK_TYPE), +#endif +#if SQLITE_BUG_COMPATIBLE_20160819 + "BUG_COMPATIBLE_20160819", +#endif +#if SQLITE_CASE_SENSITIVE_LIKE + "CASE_SENSITIVE_LIKE", +#endif +#if SQLITE_CHECK_PAGES + "CHECK_PAGES", +#endif +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif +#if SQLITE_COVERAGE_TEST + "COVERAGE_TEST", +#endif +#if SQLITE_DEBUG + "DEBUG", +#endif +#if SQLITE_DEFAULT_AUTOMATIC_INDEX + "DEFAULT_AUTOMATIC_INDEX", +#endif +#if SQLITE_DEFAULT_AUTOVACUUM + "DEFAULT_AUTOVACUUM", +#endif +#ifdef SQLITE_DEFAULT_CACHE_SIZE + "DEFAULT_CACHE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_CACHE_SIZE), +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + "DEFAULT_CKPTFULLFSYNC", +#endif +#ifdef SQLITE_DEFAULT_FILE_FORMAT + "DEFAULT_FILE_FORMAT=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_FORMAT), +#endif +#ifdef SQLITE_DEFAULT_FILE_PERMISSIONS + "DEFAULT_FILE_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_PERMISSIONS), +#endif +#if SQLITE_DEFAULT_FOREIGN_KEYS + "DEFAULT_FOREIGN_KEYS", +#endif +#ifdef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + "DEFAULT_JOURNAL_SIZE_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT), +#endif +#ifdef SQLITE_DEFAULT_LOCKING_MODE + "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), +#endif +#ifdef SQLITE_DEFAULT_LOOKASIDE + "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), +#endif +#if SQLITE_DEFAULT_MEMSTATUS + "DEFAULT_MEMSTATUS", +#endif +#ifdef SQLITE_DEFAULT_MMAP_SIZE + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PAGE_SIZE + "DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PCACHE_INITSZ + "DEFAULT_PCACHE_INITSZ=" CTIMEOPT_VAL(SQLITE_DEFAULT_PCACHE_INITSZ), +#endif +#ifdef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS + "DEFAULT_PROXYDIR_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_PROXYDIR_PERMISSIONS), +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + "DEFAULT_RECURSIVE_TRIGGERS", +#endif +#ifdef SQLITE_DEFAULT_ROWEST + "DEFAULT_ROWEST=" CTIMEOPT_VAL(SQLITE_DEFAULT_ROWEST), +#endif +#ifdef SQLITE_DEFAULT_SECTOR_SIZE + "DEFAULT_SECTOR_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_SECTOR_SIZE), +#endif +#ifdef SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT + "DEFAULT_WAL_AUTOCHECKPOINT=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_AUTOCHECKPOINT), +#endif +#ifdef SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WORKER_THREADS + "DEFAULT_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WORKER_THREADS), +#endif +#if SQLITE_DIRECT_OVERFLOW_READ + "DIRECT_OVERFLOW_READ", +#endif +#if SQLITE_DISABLE_DIRSYNC + "DISABLE_DIRSYNC", +#endif +#if SQLITE_DISABLE_FTS3_UNICODE + "DISABLE_FTS3_UNICODE", +#endif +#if SQLITE_DISABLE_FTS4_DEFERRED + "DISABLE_FTS4_DEFERRED", +#endif +#if SQLITE_DISABLE_INTRINSIC + "DISABLE_INTRINSIC", +#endif +#if SQLITE_DISABLE_LFS + "DISABLE_LFS", +#endif +#if SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + "DISABLE_PAGECACHE_OVERFLOW_STATS", +#endif +#if SQLITE_DISABLE_SKIPAHEAD_DISTINCT + "DISABLE_SKIPAHEAD_DISTINCT", +#endif +#ifdef SQLITE_ENABLE_8_3_NAMES + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), +#endif +#if SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif +#if SQLITE_ENABLE_ATOMIC_WRITE + "ENABLE_ATOMIC_WRITE", +#endif +#if SQLITE_ENABLE_BATCH_ATOMIC_WRITE + "ENABLE_BATCH_ATOMIC_WRITE", +#endif +#if SQLITE_ENABLE_BYTECODE_VTAB + "ENABLE_BYTECODE_VTAB", +#endif +#if SQLITE_ENABLE_CEROD + "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), +#endif +#if SQLITE_ENABLE_COLUMN_METADATA + "ENABLE_COLUMN_METADATA", +#endif +#if SQLITE_ENABLE_COLUMN_USED_MASK + "ENABLE_COLUMN_USED_MASK", +#endif +#if SQLITE_ENABLE_COSTMULT + "ENABLE_COSTMULT", +#endif +#if SQLITE_ENABLE_CURSOR_HINTS + "ENABLE_CURSOR_HINTS", +#endif +#if SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", +#endif +#if SQLITE_ENABLE_EXPENSIVE_ASSERT + "ENABLE_EXPENSIVE_ASSERT", +#endif +#if SQLITE_ENABLE_FTS1 + "ENABLE_FTS1", +#endif +#if SQLITE_ENABLE_FTS2 + "ENABLE_FTS2", +#endif +#if SQLITE_ENABLE_FTS3 + "ENABLE_FTS3", +#endif +#if SQLITE_ENABLE_FTS3_PARENTHESIS + "ENABLE_FTS3_PARENTHESIS", +#endif +#if SQLITE_ENABLE_FTS3_TOKENIZER + "ENABLE_FTS3_TOKENIZER", +#endif +#if SQLITE_ENABLE_FTS4 + "ENABLE_FTS4", +#endif +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif +#if SQLITE_ENABLE_GEOPOLY + "ENABLE_GEOPOLY", +#endif +#if SQLITE_ENABLE_HIDDEN_COLUMNS + "ENABLE_HIDDEN_COLUMNS", +#endif +#if SQLITE_ENABLE_ICU + "ENABLE_ICU", +#endif +#if SQLITE_ENABLE_IOTRACE + "ENABLE_IOTRACE", +#endif +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif +#if SQLITE_ENABLE_LOAD_EXTENSION + "ENABLE_LOAD_EXTENSION", +#endif +#ifdef SQLITE_ENABLE_LOCKING_STYLE + "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), +#endif +#if SQLITE_ENABLE_MATH_FUNCTIONS + "ENABLE_MATH_FUNCTIONS", +#endif +#if SQLITE_ENABLE_MEMORY_MANAGEMENT + "ENABLE_MEMORY_MANAGEMENT", +#endif +#if SQLITE_ENABLE_MEMSYS3 + "ENABLE_MEMSYS3", +#endif +#if SQLITE_ENABLE_MEMSYS5 + "ENABLE_MEMSYS5", +#endif +#if SQLITE_ENABLE_MULTIPLEX + "ENABLE_MULTIPLEX", +#endif +#if SQLITE_ENABLE_NORMALIZE + "ENABLE_NORMALIZE", +#endif +#if SQLITE_ENABLE_NULL_TRIM + "ENABLE_NULL_TRIM", +#endif +#if SQLITE_ENABLE_OVERSIZE_CELL_CHECK + "ENABLE_OVERSIZE_CELL_CHECK", +#endif +#if SQLITE_ENABLE_PREUPDATE_HOOK + "ENABLE_PREUPDATE_HOOK", +#endif +#if SQLITE_ENABLE_QPSG + "ENABLE_QPSG", +#endif +#if SQLITE_ENABLE_RBU + "ENABLE_RBU", +#endif +#if SQLITE_ENABLE_RTREE + "ENABLE_RTREE", +#endif +#if SQLITE_ENABLE_SELECTTRACE + "ENABLE_SELECTTRACE", +#endif +#if SQLITE_ENABLE_SESSION + "ENABLE_SESSION", +#endif +#if SQLITE_ENABLE_SNAPSHOT + "ENABLE_SNAPSHOT", +#endif +#if SQLITE_ENABLE_SORTER_REFERENCES + "ENABLE_SORTER_REFERENCES", +#endif +#if SQLITE_ENABLE_SQLLOG + "ENABLE_SQLLOG", +#endif +#if defined(SQLITE_ENABLE_STAT4) + "ENABLE_STAT4", +#endif +#if SQLITE_ENABLE_STMTVTAB + "ENABLE_STMTVTAB", +#endif +#if SQLITE_ENABLE_STMT_SCANSTATUS + "ENABLE_STMT_SCANSTATUS", +#endif +#if SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + "ENABLE_UNKNOWN_SQL_FUNCTION", +#endif +#if SQLITE_ENABLE_UNLOCK_NOTIFY + "ENABLE_UNLOCK_NOTIFY", +#endif +#if SQLITE_ENABLE_UPDATE_DELETE_LIMIT + "ENABLE_UPDATE_DELETE_LIMIT", +#endif +#if SQLITE_ENABLE_URI_00_ERROR + "ENABLE_URI_00_ERROR", +#endif +#if SQLITE_ENABLE_VFSTRACE + "ENABLE_VFSTRACE", +#endif +#if SQLITE_ENABLE_WHERETRACE + "ENABLE_WHERETRACE", +#endif +#if SQLITE_ENABLE_ZIPVFS + "ENABLE_ZIPVFS", +#endif +#if SQLITE_EXPLAIN_ESTIMATED_ROWS + "EXPLAIN_ESTIMATED_ROWS", +#endif +#if SQLITE_EXTRA_IFNULLROW + "EXTRA_IFNULLROW", +#endif +#ifdef SQLITE_EXTRA_INIT + "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), +#endif +#ifdef SQLITE_EXTRA_SHUTDOWN + "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), +#endif +#ifdef SQLITE_FTS3_MAX_EXPR_DEPTH + "FTS3_MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_FTS3_MAX_EXPR_DEPTH), +#endif +#if SQLITE_FTS5_ENABLE_TEST_MI + "FTS5_ENABLE_TEST_MI", +#endif +#if SQLITE_FTS5_NO_WITHOUT_ROWID + "FTS5_NO_WITHOUT_ROWID", +#endif +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +#if SQLITE_HOMEGROWN_RECURSIVE_MUTEX + "HOMEGROWN_RECURSIVE_MUTEX", +#endif +#if SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", +#endif +#if SQLITE_IGNORE_FLOCK_LOCK_ERRORS + "IGNORE_FLOCK_LOCK_ERRORS", +#endif +#if SQLITE_INLINE_MEMCPY + "INLINE_MEMCPY", +#endif +#if SQLITE_INT64_TYPE + "INT64_TYPE", +#endif +#ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX + "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), +#endif +#if SQLITE_LIKE_DOESNT_MATCH_BLOBS + "LIKE_DOESNT_MATCH_BLOBS", +#endif +#if SQLITE_LOCK_TRACE + "LOCK_TRACE", +#endif +#if SQLITE_LOG_CACHE_SPILL + "LOG_CACHE_SPILL", +#endif +#ifdef SQLITE_MALLOC_SOFT_LIMIT + "MALLOC_SOFT_LIMIT=" CTIMEOPT_VAL(SQLITE_MALLOC_SOFT_LIMIT), +#endif +#ifdef SQLITE_MAX_ATTACHED + "MAX_ATTACHED=" CTIMEOPT_VAL(SQLITE_MAX_ATTACHED), +#endif +#ifdef SQLITE_MAX_COLUMN + "MAX_COLUMN=" CTIMEOPT_VAL(SQLITE_MAX_COLUMN), +#endif +#ifdef SQLITE_MAX_COMPOUND_SELECT + "MAX_COMPOUND_SELECT=" CTIMEOPT_VAL(SQLITE_MAX_COMPOUND_SELECT), +#endif +#ifdef SQLITE_MAX_DEFAULT_PAGE_SIZE + "MAX_DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_EXPR_DEPTH + "MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_EXPR_DEPTH), +#endif +#ifdef SQLITE_MAX_FUNCTION_ARG + "MAX_FUNCTION_ARG=" CTIMEOPT_VAL(SQLITE_MAX_FUNCTION_ARG), +#endif +#ifdef SQLITE_MAX_LENGTH + "MAX_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LENGTH), +#endif +#ifdef SQLITE_MAX_LIKE_PATTERN_LENGTH + "MAX_LIKE_PATTERN_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LIKE_PATTERN_LENGTH), +#endif +#ifdef SQLITE_MAX_MEMORY + "MAX_MEMORY=" CTIMEOPT_VAL(SQLITE_MAX_MEMORY), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE_ + "MAX_MMAP_SIZE_=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE_), +#endif +#ifdef SQLITE_MAX_PAGE_COUNT + "MAX_PAGE_COUNT=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_COUNT), +#endif +#ifdef SQLITE_MAX_PAGE_SIZE + "MAX_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif +#ifdef SQLITE_MAX_SQL_LENGTH + "MAX_SQL_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_SQL_LENGTH), +#endif +#ifdef SQLITE_MAX_TRIGGER_DEPTH + "MAX_TRIGGER_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_TRIGGER_DEPTH), +#endif +#ifdef SQLITE_MAX_VARIABLE_NUMBER + "MAX_VARIABLE_NUMBER=" CTIMEOPT_VAL(SQLITE_MAX_VARIABLE_NUMBER), +#endif +#ifdef SQLITE_MAX_VDBE_OP + "MAX_VDBE_OP=" CTIMEOPT_VAL(SQLITE_MAX_VDBE_OP), +#endif +#ifdef SQLITE_MAX_WORKER_THREADS + "MAX_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_MAX_WORKER_THREADS), +#endif +#if SQLITE_MEMDEBUG + "MEMDEBUG", +#endif +#if SQLITE_MIXED_ENDIAN_64BIT_FLOAT + "MIXED_ENDIAN_64BIT_FLOAT", +#endif +#if SQLITE_MMAP_READWRITE + "MMAP_READWRITE", +#endif +#if SQLITE_MUTEX_NOOP + "MUTEX_NOOP", +#endif +#if SQLITE_MUTEX_NREF + "MUTEX_NREF", +#endif +#if SQLITE_MUTEX_OMIT + "MUTEX_OMIT", +#endif +#if SQLITE_MUTEX_PTHREADS + "MUTEX_PTHREADS", +#endif +#if SQLITE_MUTEX_W32 + "MUTEX_W32", +#endif +#if SQLITE_NEED_ERR_NAME + "NEED_ERR_NAME", +#endif +#if SQLITE_NOINLINE + "NOINLINE", +#endif +#if SQLITE_NO_SYNC + "NO_SYNC", +#endif +#if SQLITE_OMIT_ALTERTABLE + "OMIT_ALTERTABLE", +#endif +#if SQLITE_OMIT_ANALYZE + "OMIT_ANALYZE", +#endif +#if SQLITE_OMIT_ATTACH + "OMIT_ATTACH", +#endif +#if SQLITE_OMIT_AUTHORIZATION + "OMIT_AUTHORIZATION", +#endif +#if SQLITE_OMIT_AUTOINCREMENT + "OMIT_AUTOINCREMENT", +#endif +#if SQLITE_OMIT_AUTOINIT + "OMIT_AUTOINIT", +#endif +#if SQLITE_OMIT_AUTOMATIC_INDEX + "OMIT_AUTOMATIC_INDEX", +#endif +#if SQLITE_OMIT_AUTORESET + "OMIT_AUTORESET", +#endif +#if SQLITE_OMIT_AUTOVACUUM + "OMIT_AUTOVACUUM", +#endif +#if SQLITE_OMIT_BETWEEN_OPTIMIZATION + "OMIT_BETWEEN_OPTIMIZATION", +#endif +#if SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", +#endif +#if SQLITE_OMIT_CAST + "OMIT_CAST", +#endif +#if SQLITE_OMIT_CHECK + "OMIT_CHECK", +#endif +#if SQLITE_OMIT_COMPLETE + "OMIT_COMPLETE", +#endif +#if SQLITE_OMIT_COMPOUND_SELECT + "OMIT_COMPOUND_SELECT", +#endif +#if SQLITE_OMIT_CONFLICT_CLAUSE + "OMIT_CONFLICT_CLAUSE", +#endif +#if SQLITE_OMIT_CTE + "OMIT_CTE", +#endif +#if SQLITE_OMIT_DATETIME_FUNCS + "OMIT_DATETIME_FUNCS", +#endif +#if SQLITE_OMIT_DECLTYPE + "OMIT_DECLTYPE", +#endif +#if SQLITE_OMIT_DEPRECATED + "OMIT_DEPRECATED", +#endif +#if SQLITE_OMIT_DISKIO + "OMIT_DISKIO", +#endif +#if SQLITE_OMIT_EXPLAIN + "OMIT_EXPLAIN", +#endif +#if SQLITE_OMIT_FLAG_PRAGMAS + "OMIT_FLAG_PRAGMAS", +#endif +#if SQLITE_OMIT_FLOATING_POINT + "OMIT_FLOATING_POINT", +#endif +#if SQLITE_OMIT_FOREIGN_KEY + "OMIT_FOREIGN_KEY", +#endif +#if SQLITE_OMIT_GET_TABLE + "OMIT_GET_TABLE", +#endif +#if SQLITE_OMIT_HEX_INTEGER + "OMIT_HEX_INTEGER", +#endif +#if SQLITE_OMIT_INCRBLOB + "OMIT_INCRBLOB", +#endif +#if SQLITE_OMIT_INTEGRITY_CHECK + "OMIT_INTEGRITY_CHECK", +#endif +#if SQLITE_OMIT_LIKE_OPTIMIZATION + "OMIT_LIKE_OPTIMIZATION", +#endif +#if SQLITE_OMIT_LOAD_EXTENSION + "OMIT_LOAD_EXTENSION", +#endif +#if SQLITE_OMIT_LOCALTIME + "OMIT_LOCALTIME", +#endif +#if SQLITE_OMIT_LOOKASIDE + "OMIT_LOOKASIDE", +#endif +#if SQLITE_OMIT_MEMORYDB + "OMIT_MEMORYDB", +#endif +#if SQLITE_OMIT_OR_OPTIMIZATION + "OMIT_OR_OPTIMIZATION", +#endif +#if SQLITE_OMIT_PAGER_PRAGMAS + "OMIT_PAGER_PRAGMAS", +#endif +#if SQLITE_OMIT_PARSER_TRACE + "OMIT_PARSER_TRACE", +#endif +#if SQLITE_OMIT_POPEN + "OMIT_POPEN", +#endif +#if SQLITE_OMIT_PRAGMA + "OMIT_PRAGMA", +#endif +#if SQLITE_OMIT_PROGRESS_CALLBACK + "OMIT_PROGRESS_CALLBACK", +#endif +#if SQLITE_OMIT_QUICKBALANCE + "OMIT_QUICKBALANCE", +#endif +#if SQLITE_OMIT_REINDEX + "OMIT_REINDEX", +#endif +#if SQLITE_OMIT_SCHEMA_PRAGMAS + "OMIT_SCHEMA_PRAGMAS", +#endif +#if SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + "OMIT_SCHEMA_VERSION_PRAGMAS", +#endif +#if SQLITE_OMIT_SHARED_CACHE + "OMIT_SHARED_CACHE", +#endif +#if SQLITE_OMIT_SHUTDOWN_DIRECTORIES + "OMIT_SHUTDOWN_DIRECTORIES", +#endif +#if SQLITE_OMIT_SUBQUERY + "OMIT_SUBQUERY", +#endif +#if SQLITE_OMIT_TCL_VARIABLE + "OMIT_TCL_VARIABLE", +#endif +#if SQLITE_OMIT_TEMPDB + "OMIT_TEMPDB", +#endif +#if SQLITE_OMIT_TEST_CONTROL + "OMIT_TEST_CONTROL", +#endif +#if SQLITE_OMIT_TRACE + "OMIT_TRACE", +#endif +#if SQLITE_OMIT_TRIGGER + "OMIT_TRIGGER", +#endif +#if SQLITE_OMIT_TRUNCATE_OPTIMIZATION + "OMIT_TRUNCATE_OPTIMIZATION", +#endif +#if SQLITE_OMIT_UTF16 + "OMIT_UTF16", +#endif +#if SQLITE_OMIT_VACUUM + "OMIT_VACUUM", +#endif +#if SQLITE_OMIT_VIEW + "OMIT_VIEW", +#endif +#if SQLITE_OMIT_VIRTUALTABLE + "OMIT_VIRTUALTABLE", +#endif +#if SQLITE_OMIT_WAL + "OMIT_WAL", +#endif +#if SQLITE_OMIT_WSD + "OMIT_WSD", +#endif +#if SQLITE_OMIT_XFER_OPT + "OMIT_XFER_OPT", +#endif +#if SQLITE_PCACHE_SEPARATE_HEADER + "PCACHE_SEPARATE_HEADER", +#endif +#if SQLITE_PERFORMANCE_TRACE + "PERFORMANCE_TRACE", +#endif +#if SQLITE_POWERSAFE_OVERWRITE + "POWERSAFE_OVERWRITE", +#endif +#if SQLITE_PREFER_PROXY_LOCKING + "PREFER_PROXY_LOCKING", +#endif +#if SQLITE_PROXY_DEBUG + "PROXY_DEBUG", +#endif +#if SQLITE_REVERSE_UNORDERED_SELECTS + "REVERSE_UNORDERED_SELECTS", +#endif +#if SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif +#if SQLITE_SECURE_DELETE + "SECURE_DELETE", +#endif +#if SQLITE_SMALL_STACK + "SMALL_STACK", +#endif +#ifdef SQLITE_SORTER_PMASZ + "SORTER_PMASZ=" CTIMEOPT_VAL(SQLITE_SORTER_PMASZ), +#endif +#if SQLITE_SOUNDEX + "SOUNDEX", +#endif +#ifdef SQLITE_STAT4_SAMPLES + "STAT4_SAMPLES=" CTIMEOPT_VAL(SQLITE_STAT4_SAMPLES), +#endif +#ifdef SQLITE_STMTJRNL_SPILL + "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), +#endif +#if SQLITE_SUBSTR_COMPATIBILITY + "SUBSTR_COMPATIBILITY", +#endif +#if SQLITE_SYSTEM_MALLOC + "SYSTEM_MALLOC", +#endif +#if SQLITE_TCL + "TCL", +#endif +#ifdef SQLITE_TEMP_STORE + "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), +#endif +#if SQLITE_TEST + "TEST", +#endif +#if defined(SQLITE_THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#elif defined(THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), +#else + "THREADSAFE=1", +#endif +#if SQLITE_UNLINK_AFTER_CLOSE + "UNLINK_AFTER_CLOSE", +#endif +#if SQLITE_UNTESTABLE + "UNTESTABLE", +#endif +#if SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif +#if SQLITE_USE_ALLOCA + "USE_ALLOCA", +#endif +#if SQLITE_USE_FCNTL_TRACE + "USE_FCNTL_TRACE", +#endif +#if SQLITE_USE_URI + "USE_URI", +#endif +#if SQLITE_VDBE_COVERAGE + "VDBE_COVERAGE", +#endif +#if SQLITE_WIN32_MALLOC + "WIN32_MALLOC", +#endif +#if SQLITE_ZERO_MALLOC + "ZERO_MALLOC", +#endif +/* +** END CODE GENERATED BY tool/mkctime.tcl +*/ +}; + +const char **sqlite3CompileOptions(int *pnOpt){ + *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); + return (const char**)sqlite3azCompileOpt; +} + +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ diff --git a/third_party/sqlite3/date.c b/third_party/sqlite3/date.c new file mode 100644 index 000000000..5daa0ce4e --- /dev/null +++ b/third_party/sqlite3/date.c @@ -0,0 +1,1254 @@ +/* +** 2003 October 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement date and time +** functions for SQLite. +** +** There is only one exported symbol in this file - the function +** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. +** All other code has file scope. +** +** SQLite processes all times and dates as julian day numbers. The +** dates and times are stored as the number of days since noon +** in Greenwich on November 24, 4714 B.C. according to the Gregorian +** calendar system. +** +** 1970-01-01 00:00:00 is JD 2440587.5 +** 2000-01-01 00:00:00 is JD 2451544.5 +** +** This implementation requires years to be expressed as a 4-digit number +** which means that only dates between 0000-01-01 and 9999-12-31 can +** be represented, even though julian day numbers allow a much wider +** range of dates. +** +** The Gregorian calendar system is used for all dates and times, +** even those that predate the Gregorian calendar. Historians usually +** use the julian calendar for dates prior to 1582-10-15 and for some +** dates afterwards, depending on locale. Beware of this difference. +** +** The conversion algorithms are implemented based on descriptions +** in the following text: +** +** Jean Meeus +** Astronomical Algorithms, 2nd Edition, 1998 +** ISBN 0-943396-61-1 +** Willmann-Bell, Inc +** Richmond, Virginia (USA) +*/ +#include "libc/assert.h" +#include "libc/calls/weirdtypes.h" +#include "libc/mem/mem.h" +#include "libc/time/struct/tm.h" +#include "libc/time/time.h" +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +#ifndef SQLITE_OMIT_DATETIME_FUNCS + +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So declare a substitute. The substitute function itself is +** defined in "os_win.c". +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +struct tm *__cdecl localtime(const time_t *); +#endif + +/* +** A structure for holding a single date and time. +*/ +typedef struct DateTime DateTime; +struct DateTime { + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validJD; /* True (1) if iJD is valid */ + char rawS; /* Raw numeric value stored in s */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char validTZ; /* True (1) if tz is valid */ + char tzSet; /* Timezone was set explicitly */ + char isError; /* An overflow has occurred */ +}; + + +/* +** Convert zDate into one or more integers according to the conversion +** specifier zFormat. +** +** zFormat[] contains 4 characters for each integer converted, except for +** the last integer which is specified by three characters. The meaning +** of a four-character format specifiers ABCD is: +** +** A: number of digits to convert. Always "2" or "4". +** B: minimum value. Always "0" or "1". +** C: maximum value, decoded as: +** a: 12 +** b: 14 +** c: 24 +** d: 31 +** e: 59 +** f: 9999 +** D: the separator character, or \000 to indicate this is the +** last number to convert. +** +** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would +** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". +** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates +** the 2-digit day which is the last integer in the set. +** +** The function returns the number of successful conversions. +*/ +static int getDigits(const char *zDate, const char *zFormat, ...){ + /* The aMx[] array translates the 3rd character of each format + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + va_list ap; + int cnt = 0; + char nextC; + va_start(ap, zFormat); + do{ + char N = zFormat[0] - '0'; + char min = zFormat[1] - '0'; + int val = 0; + u16 max; + + assert( zFormat[2]>='a' && zFormat[2]<='f' ); + max = aMx[zFormat[2] - 'a']; + nextC = zFormat[3]; + val = 0; + while( N-- ){ + if( !sqlite3Isdigit(*zDate) ){ + goto end_getDigits; + } + val = val*10 + *zDate - '0'; + zDate++; + } + if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ + goto end_getDigits; + } + *va_arg(ap,int*) = val; + zDate++; + cnt++; + zFormat += 4; + }while( nextC ); +end_getDigits: + va_end(ap); + return cnt; +} + +/* +** Parse a timezone extension on the end of a date-time. +** The extension is of the form: +** +** (+/-)HH:MM +** +** Or the "zulu" notation: +** +** Z +** +** If the parse is successful, write the number of minutes +** of change in p->tz and return 0. If a parser error occurs, +** return non-zero. +** +** A missing specifier is not considered an error. +*/ +static int parseTimezone(const char *zDate, DateTime *p){ + int sgn = 0; + int nHr, nMn; + int c; + while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tz = 0; + c = *zDate; + if( c=='-' ){ + sgn = -1; + }else if( c=='+' ){ + sgn = +1; + }else if( c=='Z' || c=='z' ){ + zDate++; + goto zulu_time; + }else{ + return c!=0; + } + zDate++; + if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ + return 1; + } + zDate += 5; + p->tz = sgn*(nMn + nHr*60); +zulu_time: + while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tzSet = 1; + return *zDate!=0; +} + +/* +** Parse times of the form HH:MM or HH:MM:SS or HH:MM:SS.FFFF. +** The HH, MM, and SS must each be exactly 2 digits. The +** fractional seconds FFFF can be one or more digits. +** +** Return 1 if there is a parsing error and 0 on success. +*/ +static int parseHhMmSs(const char *zDate, DateTime *p){ + int h, m, s; + double ms = 0.0; + if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ + return 1; + } + zDate += 5; + if( *zDate==':' ){ + zDate++; + if( getDigits(zDate, "20e", &s)!=1 ){ + return 1; + } + zDate += 2; + if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){ + double rScale = 1.0; + zDate++; + while( sqlite3Isdigit(*zDate) ){ + ms = ms*10.0 + *zDate - '0'; + rScale *= 10.0; + zDate++; + } + ms /= rScale; + } + }else{ + s = 0; + } + p->validJD = 0; + p->rawS = 0; + p->validHMS = 1; + p->h = h; + p->m = m; + p->s = s + ms; + if( parseTimezone(zDate, p) ) return 1; + p->validTZ = (p->tz!=0)?1:0; + return 0; +} + +/* +** Put the DateTime object into its error state. +*/ +static void datetimeError(DateTime *p){ + memset(p, 0, sizeof(*p)); + p->isError = 1; +} + +/* +** Convert from YYYY-MM-DD HH:MM:SS to julian day. We always assume +** that the YYYY-MM-DD is according to the Gregorian calendar. +** +** Reference: Meeus page 61 +*/ +static void computeJD(DateTime *p){ + int Y, M, D, A, B, X1, X2; + + if( p->validJD ) return; + if( p->validYMD ){ + Y = p->Y; + M = p->M; + D = p->D; + }else{ + Y = 2000; /* If no YMD specified, assume 2000-Jan-01 */ + M = 1; + D = 1; + } + if( Y<-4713 || Y>9999 || p->rawS ){ + datetimeError(p); + return; + } + if( M<=2 ){ + Y--; + M += 12; + } + A = Y/100; + B = 2 - A + (A/4); + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); + p->validJD = 1; + if( p->validHMS ){ + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + if( p->validTZ ){ + p->iJD -= p->tz*60000; + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; + } + } +} + +/* +** Parse dates of the form +** +** YYYY-MM-DD HH:MM:SS.FFF +** YYYY-MM-DD HH:MM:SS +** YYYY-MM-DD HH:MM +** YYYY-MM-DD +** +** Write the result into the DateTime structure and return 0 +** on success and 1 if the input string is not a well-formed +** date. +*/ +static int parseYyyyMmDd(const char *zDate, DateTime *p){ + int Y, M, D, neg; + + if( zDate[0]=='-' ){ + zDate++; + neg = 1; + }else{ + neg = 0; + } + if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ + return 1; + } + zDate += 10; + while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; } + if( parseHhMmSs(zDate, p)==0 ){ + /* We got the time */ + }else if( *zDate==0 ){ + p->validHMS = 0; + }else{ + return 1; + } + p->validJD = 0; + p->validYMD = 1; + p->Y = neg ? -Y : Y; + p->M = M; + p->D = D; + if( p->validTZ ){ + computeJD(p); + } + return 0; +} + +/* +** Set the time to the current time reported by the VFS. +** +** Return the number of errors. +*/ +static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ + p->iJD = sqlite3StmtCurrentTime(context); + if( p->iJD>0 ){ + p->validJD = 1; + return 0; + }else{ + return 1; + } +} + +/* +** Input "r" is a numeric quantity which might be a julian day number, +** or the number of seconds since 1970. If the value if r is within +** range of a julian day number, install it as such and set validJD. +** If the value is a valid unix timestamp, put it in p->s and set p->rawS. +*/ +static void setRawDateNumber(DateTime *p, double r){ + p->s = r; + p->rawS = 1; + if( r>=0.0 && r<5373484.5 ){ + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; + } +} + +/* +** Attempt to parse the given string into a julian day number. Return +** the number of errors. +** +** The following are acceptable forms for the input string: +** +** YYYY-MM-DD HH:MM:SS.FFF +/-HH:MM +** DDDD.DD +** now +** +** In the first form, the +/-HH:MM is always optional. The fractional +** seconds extension (the ".FFF") is optional. The seconds portion +** (":SS.FFF") is option. The year and date can be omitted as long +** as there is a time string. The time string can be omitted as long +** as there is a year and date. +*/ +static int parseDateOrTime( + sqlite3_context *context, + const char *zDate, + DateTime *p +){ + double r; + if( parseYyyyMmDd(zDate,p)==0 ){ + return 0; + }else if( parseHhMmSs(zDate, p)==0 ){ + return 0; + }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ + return setDateTimeToCurrent(context, p); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + setRawDateNumber(p, r); + return 0; + } + return 1; +} + +/* The julian day number for 9999-12-31 23:59:59.999 is 5373484.4999999. +** Multiplying this by 86400000 gives 464269060799999 as the maximum value +** for DateTime.iJD. +** +** But some older compilers (ex: gcc 4.2.1 on older Macs) cannot deal with +** such a large integer literal, so we have to encode it. +*/ +#define INT_464269060799999 ((((i64)0x1a640)<<32)|0x1072fdff) + +/* +** Return TRUE if the given julian day number is within range. +** +** The input is the JulianDay times 86400000. +*/ +static int validJulianDay(sqlite3_int64 iJD){ + return iJD>=0 && iJD<=INT_464269060799999; +} + +/* +** Compute the Year, Month, and Day from the julian day number. +*/ +static void computeYMD(DateTime *p){ + int Z, A, B, C, D, E, X1; + if( p->validYMD ) return; + if( !p->validJD ){ + p->Y = 2000; + p->M = 1; + p->D = 1; + }else if( !validJulianDay(p->iJD) ){ + datetimeError(p); + return; + }else{ + Z = (int)((p->iJD + 43200000)/86400000); + A = (int)((Z - 1867216.25)/36524.25); + A = Z + 1 + A - (A/4); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); + X1 = (int)(30.6001*E); + p->D = B - D - X1; + p->M = E<14 ? E-1 : E-13; + p->Y = p->M>2 ? C - 4716 : C - 4715; + } + p->validYMD = 1; +} + +/* +** Compute the Hour, Minute, and Seconds from the julian day number. +*/ +static void computeHMS(DateTime *p){ + int s; + if( p->validHMS ) return; + computeJD(p); + s = (int)((p->iJD + 43200000) % 86400000); + p->s = s/1000.0; + s = (int)p->s; + p->s -= s; + p->h = s/3600; + s -= p->h*3600; + p->m = s/60; + p->s += s - p->m*60; + p->rawS = 0; + p->validHMS = 1; +} + +/* +** Compute both YMD and HMS +*/ +static void computeYMD_HMS(DateTime *p){ + computeYMD(p); + computeHMS(p); +} + +/* +** Clear the YMD and HMS and the TZ +*/ +static void clearYMD_HMS_TZ(DateTime *p){ + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; +} + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S \ + && defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#undef HAVE_LOCALTIME_S +#define HAVE_LOCALTIME_S 1 +#endif + +/* +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. +** +** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C +** library function localtime_r() is used to assist in the calculation of +** local time. +*/ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S + struct tm *pX; +#if SQLITE_THREADSAFE>0 + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ + DateTime x, y; + time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + + x = *p; + computeYMD_HMS(&x); + if( x.Y<1971 || x.Y>=2038 ){ + /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only + ** works for years between 1970 and 2037. For dates outside this range, + ** SQLite attempts to map the year into an equivalent year within this + ** range, do the calculation, then map the year back. + */ + x.Y = 2000; + x.M = 1; + x.D = 1; + x.h = 0; + x.m = 0; + x.s = 0.0; + } else { + int s = (int)(x.s + 0.5); + x.s = s; + } + x.tz = 0; + x.validJD = 0; + computeJD(&x); + t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; + } + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; + y.validYMD = 1; + y.validHMS = 1; + y.validJD = 0; + y.rawS = 0; + y.validTZ = 0; + y.isError = 0; + computeJD(&y); + *pRc = SQLITE_OK; + return y.iJD - x.iJD; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + +/* +** The following table defines various date transformations of the form +** +** 'NNN days' +** +** Where NNN is an arbitrary floating-point number and "days" can be one +** of several units of time. +*/ +static const struct { + u8 eType; /* Transformation type code */ + u8 nName; /* Length of th name */ + char *zName; /* Name of the transformation */ + double rLimit; /* Maximum NNN value for this transform */ + double rXform; /* Constant used for this transform */ +} aXformType[] = { + { 0, 6, "second", 464269060800.0, 1000.0 }, + { 0, 6, "minute", 7737817680.0, 60000.0 }, + { 0, 4, "hour", 128963628.0, 3600000.0 }, + { 0, 3, "day", 5373485.0, 86400000.0 }, + { 1, 5, "month", 176546.0, 2592000000.0 }, + { 2, 4, "year", 14713.0, 31536000000.0 }, +}; + +/* +** Process a modifier to a date-time stamp. The modifiers are +** as follows: +** +** NNN days +** NNN hours +** NNN minutes +** NNN.NNNN seconds +** NNN months +** NNN years +** start of month +** start of year +** start of week +** start of day +** weekday N +** unixepoch +** localtime +** utc +** +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. +*/ +static int parseModifier( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* The text of the modifier */ + int n, /* Length of zMod in bytes */ + DateTime *p /* The date/time value to be modified */ +){ + int rc = 1; + double r; + switch(sqlite3UpperToLower[(u8)z[0]] ){ +#ifndef SQLITE_OMIT_LOCALTIME + case 'l': { + /* localtime + ** + ** Assuming the current time value is UTC (a.k.a. GMT), shift it to + ** show local time. + */ + if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ + computeJD(p); + p->iJD += localtimeOffset(p, pCtx, &rc); + clearYMD_HMS_TZ(p); + } + break; + } +#endif + case 'u': { + /* + ** unixepoch + ** + ** Treat the current value of p->s as the number of + ** seconds since 1970. Convert to a real julian day number. + */ + if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){ + r = p->s*1000.0 + 210866760000000.0; + if( r>=0.0 && r<464269060800000.0 ){ + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + rc = 0; + } + } +#ifndef SQLITE_OMIT_LOCALTIME + else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ + if( p->tzSet==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } + p->tzSet = 1; + }else{ + rc = SQLITE_OK; + } + } +#endif + break; + } + case 'w': { + /* + ** weekday N + ** + ** Move the date to the same time on the next occurrence of + ** weekday N where 0==Sunday, 1==Monday, and so forth. If the + ** date is already on the appropriate weekday, this is a no-op. + */ + if( sqlite3_strnicmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && (n=(int)r)==r && n>=0 && r<7 ){ + sqlite3_int64 Z; + computeYMD_HMS(p); + p->validTZ = 0; + p->validJD = 0; + computeJD(p); + Z = ((p->iJD + 129600000)/86400000) % 7; + if( Z>n ) Z -= 7; + p->iJD += (n - Z)*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } + case 's': { + /* + ** start of TTTTT + ** + ** Move the date backwards to the beginning of the current day, + ** or month or year. + */ + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( !p->validJD && !p->validYMD && !p->validHMS ) break; + z += 9; + computeYMD(p); + p->validHMS = 1; + p->h = p->m = 0; + p->s = 0.0; + p->rawS = 0; + p->validTZ = 0; + p->validJD = 0; + if( sqlite3_stricmp(z,"month")==0 ){ + p->D = 1; + rc = 0; + }else if( sqlite3_stricmp(z,"year")==0 ){ + p->M = 1; + p->D = 1; + rc = 0; + }else if( sqlite3_stricmp(z,"day")==0 ){ + rc = 0; + } + break; + } + case '+': + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + double rRounder; + int i; + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + rc = 1; + break; + } + if( z[n]==':' ){ + /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the + ** specified number of hours, minutes, seconds, and fractional seconds + ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be + ** omitted. + */ + const char *z2 = z; + DateTime tx; + sqlite3_int64 day; + if( !sqlite3Isdigit(*z2) ) z2++; + memset(&tx, 0, sizeof(tx)); + if( parseHhMmSs(z2, &tx) ) break; + computeJD(&tx); + tx.iJD -= 43200000; + day = tx.iJD/86400000; + tx.iJD -= day*86400000; + if( z[0]=='-' ) tx.iJD = -tx.iJD; + computeJD(p); + clearYMD_HMS_TZ(p); + p->iJD += tx.iJD; + rc = 0; + break; + } + + /* If control reaches this point, it means the transformation is + ** one of the forms like "+NNN days". */ + z += n; + while( sqlite3Isspace(*z) ) z++; + n = sqlite3Strlen30(z); + if( n>10 || n<3 ) break; + if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; + computeJD(p); + rc = 1; + rRounder = r<0 ? -0.5 : +0.5; + for(i=0; i-aXformType[i].rLimit && rM += (int)r; + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + p->validJD = 0; + r -= (int)r; + break; + } + case 2: { /* Special processing to add years */ + int y = (int)r; + computeYMD_HMS(p); + p->Y += y; + p->validJD = 0; + r -= (int)r; + break; + } + } + computeJD(p); + p->iJD += (sqlite3_int64)(r*aXformType[i].rXform + rRounder); + rc = 0; + break; + } + } + clearYMD_HMS_TZ(p); + break; + } + default: { + break; + } + } + return rc; +} + +/* +** Process time function arguments. argv[0] is a date-time stamp. +** argv[1] and following are modifiers. Parse them all and write +** the resulting time into the DateTime structure p. Return 0 +** on success and 1 if there are any errors. +** +** If there are zero parameters (if even argv[0] is undefined) +** then assume a default value of "now" for argv[0]. +*/ +static int isDate( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + DateTime *p +){ + int i, n; + const unsigned char *z; + int eType; + memset(p, 0, sizeof(*p)); + if( argc==0 ){ + if( !sqlite3NotPureFunc(context) ) return 1; + return setDateTimeToCurrent(context, p); + } + if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT + || eType==SQLITE_INTEGER ){ + setRawDateNumber(p, sqlite3_value_double(argv[0])); + }else{ + z = sqlite3_value_text(argv[0]); + if( !z || parseDateOrTime(context, (char*)z, p) ){ + return 1; + } + } + for(i=1; iisError || !validJulianDay(p->iJD) ) return 1; + return 0; +} + + +/* +** The following routines implement the various date and time functions +** of SQLite. +*/ + +/* +** julianday( TIMESTRING, MOD, MOD, ...) +** +** Return the julian day number of the date specified in the arguments +*/ +static void juliandayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + computeJD(&x); + sqlite3_result_double(context, x.iJD/86400000.0); + } +} + +/* +** datetime( TIMESTRING, MOD, MOD, ...) +** +** Return YYYY-MM-DD HH:MM:SS +*/ +static void datetimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeYMD_HMS(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d %02d:%02d:%02d", + x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** time( TIMESTRING, MOD, MOD, ...) +** +** Return HH:MM:SS +*/ +static void timeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeHMS(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%02d:%02d:%02d", x.h, x.m, (int)x.s); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** date( TIMESTRING, MOD, MOD, ...) +** +** Return YYYY-MM-DD +*/ +static void dateFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeYMD(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d", x.Y, x.M, x.D); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) +** +** Return a string described by FORMAT. Conversions as follows: +** +** %d day of month +** %f ** fractional seconds SS.SSS +** %H hour 00-24 +** %j day of year 000-366 +** %J ** julian day number +** %m month 01-12 +** %M minute 00-59 +** %s seconds since 1970-01-01 +** %S seconds 00-59 +** %w day of week 0-6 sunday==0 +** %W week of year 00-53 +** %Y year 0000-9999 +** %% % +*/ +static void strftimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + u64 n; + size_t i,j; + char *z; + sqlite3 *db; + const char *zFmt; + char zBuf[100]; + if( argc==0 ) return; + zFmt = (const char*)sqlite3_value_text(argv[0]); + if( zFmt==0 || isDate(context, argc-1, argv+1, &x) ) return; + db = sqlite3_context_db_handle(context); + for(i=0, n=1; zFmt[i]; i++, n++){ + if( zFmt[i]=='%' ){ + switch( zFmt[i+1] ){ + case 'd': + case 'H': + case 'm': + case 'M': + case 'S': + case 'W': + n++; + /* fall thru */ + case 'w': + case '%': + break; + case 'f': + n += 8; + break; + case 'j': + n += 3; + break; + case 'Y': + n += 8; + break; + case 's': + case 'J': + n += 50; + break; + default: + return; /* ERROR. return a NULL */ + } + i++; + } + } + testcase( n==sizeof(zBuf)-1 ); + testcase( n==sizeof(zBuf) ); + testcase( n==(u64)db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + testcase( n==(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( n(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + return; + }else{ + z = sqlite3DbMallocRawNN(db, (int)n); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + computeJD(&x); + computeYMD_HMS(&x); + for(i=j=0; zFmt[i]; i++){ + if( zFmt[i]!='%' ){ + z[j++] = zFmt[i]; + }else{ + i++; + switch( zFmt[i] ){ + case 'd': sqlite3_snprintf(3, &z[j],"%02d",x.D); j+=2; break; + case 'f': { + double s = x.s; + if( s>59.999 ) s = 59.999; + sqlite3_snprintf(7, &z[j],"%06.3f", s); + j += sqlite3Strlen30(&z[j]); + break; + } + case 'H': sqlite3_snprintf(3, &z[j],"%02d",x.h); j+=2; break; + case 'W': /* Fall thru */ + case 'j': { + int nDay; /* Number of days since 1st day of year */ + DateTime y = x; + y.validJD = 0; + y.M = 1; + y.D = 1; + computeJD(&y); + nDay = (int)((x.iJD-y.iJD+43200000)/86400000); + if( zFmt[i]=='W' ){ + int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ + wd = (int)(((x.iJD+43200000)/86400000)%7); + sqlite3_snprintf(3, &z[j],"%02d",(nDay+7-wd)/7); + j += 2; + }else{ + sqlite3_snprintf(4, &z[j],"%03d",nDay+1); + j += 3; + } + break; + } + case 'J': { + sqlite3_snprintf(20, &z[j],"%.16g",x.iJD/86400000.0); + j+=sqlite3Strlen30(&z[j]); + break; + } + case 'm': sqlite3_snprintf(3, &z[j],"%02d",x.M); j+=2; break; + case 'M': sqlite3_snprintf(3, &z[j],"%02d",x.m); j+=2; break; + case 's': { + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3Int64ToText(iS, &z[j]); + j += sqlite3Strlen30(&z[j]); + break; + } + case 'S': sqlite3_snprintf(3,&z[j],"%02d",(int)x.s); j+=2; break; + case 'w': { + z[j++] = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + break; + } + case 'Y': { + sqlite3_snprintf(5,&z[j],"%04d",x.Y); j+=sqlite3Strlen30(&z[j]); + break; + } + default: z[j++] = '%'; break; + } + } + } + z[j] = 0; + sqlite3_result_text(context, z, -1, + z==zBuf ? SQLITE_TRANSIENT : SQLITE_DYNAMIC); +} + +/* +** current_time() +** +** This function returns the same value as time('now'). +*/ +static void ctimeFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + timeFunc(context, 0, 0); +} + +/* +** current_date() +** +** This function returns the same value as date('now'). +*/ +static void cdateFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + dateFunc(context, 0, 0); +} + +/* +** current_timestamp() +** +** This function returns the same value as datetime('now'). +*/ +static void ctimestampFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + datetimeFunc(context, 0, 0); +} +#endif /* !defined(SQLITE_OMIT_DATETIME_FUNCS) */ + +#ifdef SQLITE_OMIT_DATETIME_FUNCS +/* +** If the library is compiled to omit the full-scale date and time +** handling (to get a smaller binary), the following minimal version +** of the functions current_time(), current_date() and current_timestamp() +** are included instead. This is to support column declarations that +** include "DEFAULT CURRENT_TIME" etc. +** +** This function uses the C-library functions time(), gmtime() +** and strftime(). The format string to pass to strftime() is supplied +** as the user-data for the function. +*/ +static void currentTimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + time_t t; + char *zFormat = (char *)sqlite3_user_data(context); + sqlite3_int64 iT; + struct tm *pTm; + struct tm sNow; + char zBuf[20]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + iT = sqlite3StmtCurrentTime(context); + if( iT<=0 ) return; + t = iT/1000 - 10000*(sqlite3_int64)21086676; +#if HAVE_GMTIME_R + pTm = gmtime_r(&t, &sNow); +#else + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); + pTm = gmtime(&t); + if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); +#endif + if( pTm ){ + strftime(zBuf, 20, zFormat, &sNow); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} +#endif + +/* +** This function registered all of the above C functions as SQL +** functions. This should be the only routine in this file with +** external linkage. +*/ +void sqlite3RegisterDateTimeFunctions(void){ + static FuncDef aDateTimeFuncs[] = { +#ifndef SQLITE_OMIT_DATETIME_FUNCS + PURE_DATE(julianday, -1, 0, 0, juliandayFunc ), + PURE_DATE(date, -1, 0, 0, dateFunc ), + PURE_DATE(time, -1, 0, 0, timeFunc ), + PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), + PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), +#else + STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), + STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), + STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), +#endif + }; + sqlite3InsertBuiltinFuncs(aDateTimeFuncs, ArraySize(aDateTimeFuncs)); +} diff --git a/third_party/sqlite3/dbpage.c b/third_party/sqlite3/dbpage.c new file mode 100644 index 000000000..d6540a00f --- /dev/null +++ b/third_party/sqlite3/dbpage.c @@ -0,0 +1,418 @@ +/* +** 2017-10-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_dbpage" virtual table. +** +** The sqlite_dbpage virtual table is used to read or write whole raw +** pages of the database file. The pager interface is used so that +** uncommitted changes and changes recorded in the WAL file are correctly +** retrieved. +** +** Usage example: +** +** SELECT data FROM sqlite_dbpage('aux1') WHERE pgno=123; +** +** This is an eponymous virtual table so it does not need to be created before +** use. The optional argument to the sqlite_dbpage() table name is the +** schema for the database file that is to be read. The default schema is +** "main". +** +** The data field of sqlite_dbpage table can be updated. The new +** value must be a BLOB which is the correct page size, otherwise the +** update fails. Rows may not be deleted or inserted. +*/ +#include "third_party/sqlite3/sqliteInt.inc" /* Requires access to internal data inc */ + +/* clang-format off */ + +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) && \ + !defined(SQLITE_OMIT_VIRTUALTABLE) + +typedef struct DbpageTable DbpageTable; +typedef struct DbpageCursor DbpageCursor; + +struct DbpageCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int pgno; /* Current page number */ + int mxPgno; /* Last page to visit on this scan */ + Pager *pPager; /* Pager being read/written */ + DbPage *pPage1; /* Page 1 of the database */ + int iDb; /* Index of database to analyze */ + int szPage; /* Size of each page in bytes */ +}; + +struct DbpageTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database */ +}; + +/* Columns */ +#define DBPAGE_COLUMN_PGNO 0 +#define DBPAGE_COLUMN_DATA 1 +#define DBPAGE_COLUMN_SCHEMA 2 + + + +/* +** Connect to or create a dbpagevfs virtual table. +*/ +static int dbpageConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbpageTable *pTab = 0; + int rc = SQLITE_OK; + + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); + if( rc==SQLITE_OK ){ + pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(DbpageTable)); + pTab->db = db; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbpagevfs virtual table. +*/ +static int dbpageDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 +*/ +static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + int iPlan = 0; + + /* If there is a schema= constraint, it must be honored. Report a + ** ridiculously large estimated cost if the schema= constraint is + ** unavailable + */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->iColumn!=DBPAGE_COLUMN_SCHEMA ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( !p->usable ){ + /* No solution. */ + return SQLITE_CONSTRAINT; + } + iPlan = 2; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + + /* If we reach this point, it means that either there is no schema= + ** constraint (in which case we use the "main" schema) or else the + ** schema constraint was accepted. Lower the estimated cost accordingly + */ + pIdxInfo->estimatedCost = 1.0e6; + + /* Check for constraints against pgno */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[i].argvIndex = iPlan ? 2 : 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + iPlan |= 1; + break; + } + } + pIdxInfo->idxNum = iPlan; + + if( pIdxInfo->nOrderBy>=1 + && pIdxInfo->aOrderBy[0].iColumn<=0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + return SQLITE_OK; +} + +/* +** Open a new dbpagevfs cursor. +*/ +static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbpageCursor *pCsr; + + pCsr = (DbpageCursor *)sqlite3_malloc64(sizeof(DbpageCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memset(pCsr, 0, sizeof(DbpageCursor)); + pCsr->base.pVtab = pVTab; + pCsr->pgno = -1; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a dbpagevfs cursor. +*/ +static int dbpageClose(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a dbpagevfs cursor to the next entry in the file. +*/ +static int dbpageNext(sqlite3_vtab_cursor *pCursor){ + int rc = SQLITE_OK; + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + pCsr->pgno++; + return rc; +} + +static int dbpageEof(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + return pCsr->pgno > pCsr->mxPgno; +} + +/* +** idxNum: +** +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 +** +** idxStr is not used +*/ +static int dbpageFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc; + sqlite3 *db = pTab->db; + Btree *pBt; + + /* Default setting is no rows of result */ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + + if( idxNum & 2 ){ + const char *zSchema; + assert( argc>=1 ); + zSchema = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(db, zSchema); + if( pCsr->iDb<0 ) return SQLITE_OK; + }else{ + pCsr->iDb = 0; + } + pBt = db->aDb[pCsr->iDb].pBt; + if( pBt==0 ) return SQLITE_OK; + pCsr->pPager = sqlite3BtreePager(pBt); + pCsr->szPage = sqlite3BtreeGetPageSize(pBt); + pCsr->mxPgno = sqlite3BtreeLastPage(pBt); + if( idxNum & 1 ){ + assert( argc>(idxNum>>1) ); + pCsr->pgno = sqlite3_value_int(argv[idxNum>>1]); + if( pCsr->pgno<1 || pCsr->pgno>pCsr->mxPgno ){ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + }else{ + pCsr->mxPgno = pCsr->pgno; + } + }else{ + assert( pCsr->pgno==1 ); + } + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); + rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0); + return rc; +} + +static int dbpageColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + int rc = SQLITE_OK; + switch( i ){ + case 0: { /* pgno */ + sqlite3_result_int(ctx, pCsr->pgno); + break; + } + case 1: { /* data */ + DbPage *pDbPage = 0; + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); + break; + } + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + sqlite3_result_text(ctx, db->aDb[pCsr->iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + *pRowid = pCsr->pgno; + return SQLITE_OK; +} + +static int dbpageUpdate( + sqlite3_vtab *pVtab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + DbpageTable *pTab = (DbpageTable *)pVtab; + Pgno pgno; + DbPage *pDbPage = 0; + int rc = SQLITE_OK; + char *zErr = 0; + const char *zSchema; + int iDb; + Btree *pBt; + Pager *pPager; + int szPage; + + if( pTab->db->flags & SQLITE_Defensive ){ + zErr = "read-only"; + goto update_fail; + } + if( argc==1 ){ + zErr = "cannot delete"; + goto update_fail; + } + pgno = sqlite3_value_int(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } + pBt = pTab->db->aDb[iDb].pBt; + if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){ + zErr = "bad page number"; + goto update_fail; + } + szPage = sqlite3BtreeGetPageSize(pBt); + if( sqlite3_value_type(argv[3])!=SQLITE_BLOB + || sqlite3_value_bytes(argv[3])!=szPage + ){ + zErr = "bad page value"; + goto update_fail; + } + pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + memcpy(sqlite3PagerGetData(pDbPage), + sqlite3_value_blob(argv[3]), + szPage); + } + } + sqlite3PagerUnref(pDbPage); + return rc; + +update_fail: + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); + return SQLITE_ERROR; +} + +/* Since we do not know in advance which database files will be +** written by the sqlite_dbpage virtual table, start a write transaction +** on them all. +*/ +static int dbpageBegin(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3 *db = pTab->db; + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); + } + return SQLITE_OK; +} + + +/* +** Invoke this routine to register the "dbpage" virtual table module +*/ +int sqlite3DbpageRegister(sqlite3 *db){ + static sqlite3_module dbpage_module = { + 0, /* iVersion */ + dbpageConnect, /* xCreate */ + dbpageConnect, /* xConnect */ + dbpageBestIndex, /* xBestIndex */ + dbpageDisconnect, /* xDisconnect */ + dbpageDisconnect, /* xDestroy */ + dbpageOpen, /* xOpen - open a cursor */ + dbpageClose, /* xClose - close a cursor */ + dbpageFilter, /* xFilter - configure scan constraints */ + dbpageNext, /* xNext - advance a cursor */ + dbpageEof, /* xEof - check for end of scan */ + dbpageColumn, /* xColumn - read data */ + dbpageRowid, /* xRowid - read data */ + dbpageUpdate, /* xUpdate */ + dbpageBegin, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); +} +#elif defined(SQLITE_ENABLE_DBPAGE_VTAB) +int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ diff --git a/third_party/sqlite3/dbstat.c b/third_party/sqlite3/dbstat.c new file mode 100644 index 000000000..16626f6d7 --- /dev/null +++ b/third_party/sqlite3/dbstat.c @@ -0,0 +1,855 @@ +/* +** 2010 July 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "dbstat" virtual table. +** +** The dbstat virtual table is used to extract low-level storage +** information from an SQLite database in order to implement the +** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script +** for an example implementation. +** +** Additional information is available on the "dbstat.html" page of the +** official SQLite documentation. +*/ +#include "third_party/sqlite3/sqliteInt.inc" /* Requires access to internal data inc */ + +/* clang-format off */ + +#if (defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST)) && \ + !defined(SQLITE_OMIT_VIRTUALTABLE) + +/* +** Page paths: +** +** The value of the 'path' column describes the path taken from the +** root-node of the b-tree structure to each page. The value of the +** root-node path is '/'. +** +** The value of the path for the left-most child page of the root of +** a b-tree is '/000/'. (Btrees store content ordered from left to right +** so the pages to the left have smaller keys than the pages to the right.) +** The next to left-most child of the root page is +** '/001', and so on, each sibling page identified by a 3-digit hex +** value. The children of the 451st left-most sibling have paths such +** as '/1c2/000/, '/1c2/001/' etc. +** +** Overflow pages are specified by appending a '+' character and a +** six-digit hexadecimal value to the path to the cell they are linked +** from. For example, the three overflow pages in a chain linked from +** the left-most cell of the 450th child of the root page are identified +** by the paths: +** +** '/1c2/000+000000' // First page in overflow chain +** '/1c2/000+000001' // Second page in overflow chain +** '/1c2/000+000002' // Third page in overflow chain +** +** If the paths are sorted using the BINARY collation sequence, then +** the overflow pages associated with a cell will appear earlier in the +** sort-order than its child page: +** +** '/1c2/000/' // Left-most child of 451st child of root +*/ +static const char zDbstatSchema[] = + "CREATE TABLE x(" + " name TEXT," /* 0 Name of table or index */ + " path TEXT," /* 1 Path to page from root (NULL for agg) */ + " pageno INTEGER," /* 2 Page number (page count for aggregates) */ + " pagetype TEXT," /* 3 'internal', 'leaf', 'overflow', or NULL */ + " ncell INTEGER," /* 4 Cells on page (0 for overflow) */ + " payload INTEGER," /* 5 Bytes of payload on this page */ + " unused INTEGER," /* 6 Bytes of unused space on this page */ + " mx_payload INTEGER," /* 7 Largest payload size of all cells */ + " pgoffset INTEGER," /* 8 Offset of page in file (NULL for agg) */ + " pgsize INTEGER," /* 9 Size of the page (sum for aggregate) */ + " schema TEXT HIDDEN," /* 10 Database schema being analyzed */ + " aggregate BOOLEAN HIDDEN" /* 11 aggregate info for each table */ + ")" +; + +/* Forward reference to data structured used in this module */ +typedef struct StatTable StatTable; +typedef struct StatCursor StatCursor; +typedef struct StatPage StatPage; +typedef struct StatCell StatCell; + +/* Size information for a single cell within a btree page */ +struct StatCell { + int nLocal; /* Bytes of local payload */ + u32 iChildPg; /* Child node (or 0 if this is a leaf) */ + int nOvfl; /* Entries in aOvfl[] */ + u32 *aOvfl; /* Array of overflow page numbers */ + int nLastOvfl; /* Bytes of payload on final overflow page */ + int iOvfl; /* Iterates through aOvfl[] */ +}; + +/* Size information for a single btree page */ +struct StatPage { + u32 iPgno; /* Page number */ + DbPage *pPg; /* Page content */ + int iCell; /* Current cell */ + + char *zPath; /* Path to this page */ + + /* Variables populated by statDecodePage(): */ + u8 flags; /* Copy of flags byte */ + int nCell; /* Number of cells on page */ + int nUnused; /* Number of unused bytes on page */ + StatCell *aCell; /* Array of parsed cells */ + u32 iRightChildPg; /* Right-child page number (or 0) */ + int nMxPayload; /* Largest payload of any cell on the page */ +}; + +/* The cursor for scanning the dbstat virtual table */ +struct StatCursor { + sqlite3_vtab_cursor base; /* base class. MUST BE FIRST! */ + sqlite3_stmt *pStmt; /* Iterates through set of root pages */ + u8 isEof; /* After pStmt has returned SQLITE_DONE */ + u8 isAgg; /* Aggregate results for each table */ + int iDb; /* Schema used for this query */ + + StatPage aPage[32]; /* Pages in path to current page */ + int iPage; /* Current entry in aPage[] */ + + /* Values to return. */ + u32 iPageno; /* Value of 'pageno' column */ + char *zName; /* Value of 'name' column */ + char *zPath; /* Value of 'path' column */ + char *zPagetype; /* Value of 'pagetype' column */ + int nPage; /* Number of pages in current btree */ + int nCell; /* Value of 'ncell' column */ + int nMxPayload; /* Value of 'mx_payload' column */ + i64 nUnused; /* Value of 'unused' column */ + i64 nPayload; /* Value of 'payload' column */ + i64 iOffset; /* Value of 'pgOffset' column */ + i64 szPage; /* Value of 'pgSize' column */ +}; + +/* An instance of the DBSTAT virtual table */ +struct StatTable { + sqlite3_vtab base; /* base class. MUST BE FIRST! */ + sqlite3 *db; /* Database connection that owns this vtab */ + int iDb; /* Index of database to analyze */ +}; + +#ifndef get2byte +# define get2byte(x) ((x)[0]<<8 | (x)[1]) +#endif + +/* +** Connect to or create a new DBSTAT virtual table. +*/ +static int statConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + StatTable *pTab = 0; + int rc = SQLITE_OK; + int iDb; + + if( argc>=4 ){ + Token nm; + sqlite3TokenInit(&nm, (char*)argv[3]); + iDb = sqlite3FindDb(db, &nm); + if( iDb<0 ){ + *pzErr = sqlite3_mprintf("no such database: %s", argv[3]); + return SQLITE_ERROR; + } + }else{ + iDb = 0; + } + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + rc = sqlite3_declare_vtab(db, zDbstatSchema); + if( rc==SQLITE_OK ){ + pTab = (StatTable *)sqlite3_malloc64(sizeof(StatTable)); + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(StatTable)); + pTab->db = db; + pTab->iDb = iDb; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy the DBSTAT virtual table. +*/ +static int statDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Compute the best query strategy and return the result in idxNum. +** +** idxNum-Bit Meaning +** ---------- ---------------------------------------------- +** 0x01 There is a schema=? term in the WHERE clause +** 0x02 There is a name=? term in the WHERE clause +** 0x04 There is an aggregate=? term in the WHERE clause +** 0x08 Output should be ordered by name and path +*/ +static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + int iSchema = -1; + int iName = -1; + int iAgg = -1; + + /* Look for a valid schema=? constraint. If found, change the idxNum to + ** 1 and request the value of that constraint be sent to xFilter. And + ** lower the cost estimate to encourage the constrained version to be + ** used. + */ + for(i=0; inConstraint; i++){ + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pIdxInfo->aConstraint[i].usable==0 ){ + /* Force DBSTAT table should always be the right-most table in a join */ + return SQLITE_CONSTRAINT; + } + switch( pIdxInfo->aConstraint[i].iColumn ){ + case 0: { /* name */ + iName = i; + break; + } + case 10: { /* schema */ + iSchema = i; + break; + } + case 11: { /* aggregate */ + iAgg = i; + break; + } + } + } + i = 0; + if( iSchema>=0 ){ + pIdxInfo->aConstraintUsage[iSchema].argvIndex = ++i; + pIdxInfo->aConstraintUsage[iSchema].omit = 1; + pIdxInfo->idxNum |= 0x01; + } + if( iName>=0 ){ + pIdxInfo->aConstraintUsage[iName].argvIndex = ++i; + pIdxInfo->idxNum |= 0x02; + } + if( iAgg>=0 ){ + pIdxInfo->aConstraintUsage[iAgg].argvIndex = ++i; + pIdxInfo->idxNum |= 0x04; + } + pIdxInfo->estimatedCost = 1.0; + + /* Records are always returned in ascending order of (name, path). + ** If this will satisfy the client, set the orderByConsumed flag so that + ** SQLite does not do an external sort. + */ + if( ( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==0 + && pIdxInfo->aOrderBy[0].desc==0 + ) || + ( pIdxInfo->nOrderBy==2 + && pIdxInfo->aOrderBy[0].iColumn==0 + && pIdxInfo->aOrderBy[0].desc==0 + && pIdxInfo->aOrderBy[1].iColumn==1 + && pIdxInfo->aOrderBy[1].desc==0 + ) + ){ + pIdxInfo->orderByConsumed = 1; + pIdxInfo->idxNum |= 0x08; + } + + return SQLITE_OK; +} + +/* +** Open a new DBSTAT cursor. +*/ +static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + StatTable *pTab = (StatTable *)pVTab; + StatCursor *pCsr; + + pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memset(pCsr, 0, sizeof(StatCursor)); + pCsr->base.pVtab = pVTab; + pCsr->iDb = pTab->iDb; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +static void statClearCells(StatPage *p){ + int i; + if( p->aCell ){ + for(i=0; inCell; i++){ + sqlite3_free(p->aCell[i].aOvfl); + } + sqlite3_free(p->aCell); + } + p->nCell = 0; + p->aCell = 0; +} + +static void statClearPage(StatPage *p){ + statClearCells(p); + sqlite3PagerUnref(p->pPg); + sqlite3_free(p->zPath); + memset(p, 0, sizeof(StatPage)); +} + +static void statResetCsr(StatCursor *pCsr){ + int i; + sqlite3_reset(pCsr->pStmt); + for(i=0; iaPage); i++){ + statClearPage(&pCsr->aPage[i]); + } + pCsr->iPage = 0; + sqlite3_free(pCsr->zPath); + pCsr->zPath = 0; + pCsr->isEof = 0; +} + +/* Resize the space-used counters inside of the cursor */ +static void statResetCounts(StatCursor *pCsr){ + pCsr->nCell = 0; + pCsr->nMxPayload = 0; + pCsr->nUnused = 0; + pCsr->nPayload = 0; + pCsr->szPage = 0; + pCsr->nPage = 0; +} + +/* +** Close a DBSTAT cursor. +*/ +static int statClose(sqlite3_vtab_cursor *pCursor){ + StatCursor *pCsr = (StatCursor *)pCursor; + statResetCsr(pCsr); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** For a single cell on a btree page, compute the number of bytes of +** content (payload) stored on that page. That is to say, compute the +** number of bytes of content not found on overflow pages. +*/ +static int getLocalPayload( + int nUsable, /* Usable bytes per page */ + u8 flags, /* Page flags */ + int nTotal /* Total record (payload) size */ +){ + int nLocal; + int nMinLocal; + int nMaxLocal; + + if( flags==0x0D ){ /* Table leaf node */ + nMinLocal = (nUsable - 12) * 32 / 255 - 23; + nMaxLocal = nUsable - 35; + }else{ /* Index interior and leaf nodes */ + nMinLocal = (nUsable - 12) * 32 / 255 - 23; + nMaxLocal = (nUsable - 12) * 64 / 255 - 23; + } + + nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4); + if( nLocal>nMaxLocal ) nLocal = nMinLocal; + return nLocal; +} + +/* Populate the StatPage object with information about the all +** cells found on the page currently under analysis. +*/ +static int statDecodePage(Btree *pBt, StatPage *p){ + int nUnused; + int iOff; + int nHdr; + int isLeaf; + int szPage; + + u8 *aData = sqlite3PagerGetData(p->pPg); + u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0]; + + p->flags = aHdr[0]; + if( p->flags==0x0A || p->flags==0x0D ){ + isLeaf = 1; + nHdr = 8; + }else if( p->flags==0x05 || p->flags==0x02 ){ + isLeaf = 0; + nHdr = 12; + }else{ + goto statPageIsCorrupt; + } + if( p->iPgno==1 ) nHdr += 100; + p->nCell = get2byte(&aHdr[3]); + p->nMxPayload = 0; + szPage = sqlite3BtreeGetPageSize(pBt); + + nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell; + nUnused += (int)aHdr[7]; + iOff = get2byte(&aHdr[1]); + while( iOff ){ + int iNext; + if( iOff>=szPage ) goto statPageIsCorrupt; + nUnused += get2byte(&aData[iOff+2]); + iNext = get2byte(&aData[iOff]); + if( iNext0 ) goto statPageIsCorrupt; + iOff = iNext; + } + p->nUnused = nUnused; + p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]); + + if( p->nCell ){ + int i; /* Used to iterate through cells */ + int nUsable; /* Usable bytes per page */ + + sqlite3BtreeEnter(pBt); + nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt); + sqlite3BtreeLeave(pBt); + p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell)); + if( p->aCell==0 ) return SQLITE_NOMEM_BKPT; + memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell)); + + for(i=0; inCell; i++){ + StatCell *pCell = &p->aCell[i]; + + iOff = get2byte(&aData[nHdr+i*2]); + if( iOff=szPage ) goto statPageIsCorrupt; + if( !isLeaf ){ + pCell->iChildPg = sqlite3Get4byte(&aData[iOff]); + iOff += 4; + } + if( p->flags==0x05 ){ + /* A table interior node. nPayload==0. */ + }else{ + u32 nPayload; /* Bytes of payload total (local+overflow) */ + int nLocal; /* Bytes of payload stored locally */ + iOff += getVarint32(&aData[iOff], nPayload); + if( p->flags==0x0D ){ + u64 dummy; + iOff += sqlite3GetVarint(&aData[iOff], &dummy); + } + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; + nLocal = getLocalPayload(nUsable, p->flags, nPayload); + if( nLocal<0 ) goto statPageIsCorrupt; + pCell->nLocal = nLocal; + assert( nPayload>=(u32)nLocal ); + assert( nLocal<=(nUsable-35) ); + if( nPayload>(u32)nLocal ){ + int j; + int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); + if( iOff+nLocal>nUsable || nPayload>0x7fffffff ){ + goto statPageIsCorrupt; + } + pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); + pCell->nOvfl = nOvfl; + pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl); + if( pCell->aOvfl==0 ) return SQLITE_NOMEM_BKPT; + pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]); + for(j=1; jaOvfl[j-1]; + DbPage *pPg = 0; + rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg, 0); + if( rc!=SQLITE_OK ){ + assert( pPg==0 ); + return rc; + } + pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg)); + sqlite3PagerUnref(pPg); + } + } + } + } + } + + return SQLITE_OK; + +statPageIsCorrupt: + p->flags = 0; + statClearCells(p); + return SQLITE_OK; +} + +/* +** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on +** the current value of pCsr->iPageno. +*/ +static void statSizeAndOffset(StatCursor *pCsr){ + StatTable *pTab = (StatTable *)((sqlite3_vtab_cursor *)pCsr)->pVtab; + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3_file *fd; + sqlite3_int64 x[2]; + + /* If connected to a ZIPVFS backend, find the page size and + ** offset from ZIPVFS. + */ + fd = sqlite3PagerFile(pPager); + x[0] = pCsr->iPageno; + if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ + pCsr->iOffset = x[0]; + pCsr->szPage += x[1]; + }else{ + /* Not ZIPVFS: The default page size and offset */ + pCsr->szPage += sqlite3BtreeGetPageSize(pBt); + pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); + } +} + +/* +** Move a DBSTAT cursor to the next entry. Normally, the next +** entry will be the next page, but in aggregated mode (pCsr->isAgg!=0), +** the next entry is the next btree. +*/ +static int statNext(sqlite3_vtab_cursor *pCursor){ + int rc; + int nPayload; + char *z; + StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable *)pCursor->pVtab; + Btree *pBt = pTab->db->aDb[pCsr->iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + + sqlite3_free(pCsr->zPath); + pCsr->zPath = 0; + +statNextRestart: + if( pCsr->aPage[0].pPg==0 ){ + /* Start measuring space on the next btree */ + statResetCounts(pCsr); + rc = sqlite3_step(pCsr->pStmt); + if( rc==SQLITE_ROW ){ + int nPage; + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); + sqlite3PagerPagecount(pPager, &nPage); + if( nPage==0 ){ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } + rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg, 0); + pCsr->aPage[0].iPgno = iRoot; + pCsr->aPage[0].iCell = 0; + if( !pCsr->isAgg ){ + pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } + pCsr->iPage = 0; + pCsr->nPage = 1; + }else{ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } + }else{ + /* Continue analyzing the btree previously started */ + StatPage *p = &pCsr->aPage[pCsr->iPage]; + if( !pCsr->isAgg ) statResetCounts(pCsr); + while( p->iCellnCell ){ + StatCell *pCell = &p->aCell[p->iCell]; + while( pCell->iOvflnOvfl ){ + int nUsable, iOvfl; + sqlite3BtreeEnter(pBt); + nUsable = sqlite3BtreeGetPageSize(pBt) - + sqlite3BtreeGetReserveNoMutex(pBt); + sqlite3BtreeLeave(pBt); + pCsr->nPage++; + statSizeAndOffset(pCsr); + if( pCell->iOvflnOvfl-1 ){ + pCsr->nPayload += nUsable - 4; + }else{ + pCsr->nPayload += pCell->nLastOvfl; + pCsr->nUnused += nUsable - 4 - pCell->nLastOvfl; + } + iOvfl = pCell->iOvfl; + pCell->iOvfl++; + if( !pCsr->isAgg ){ + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); + pCsr->iPageno = pCell->aOvfl[iOvfl]; + pCsr->zPagetype = "overflow"; + pCsr->zPath = z = sqlite3_mprintf( + "%s%.3x+%.6x", p->zPath, p->iCell, iOvfl + ); + return z==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; + } + } + if( p->iRightChildPg ) break; + p->iCell++; + } + + if( !p->iRightChildPg || p->iCell>p->nCell ){ + statClearPage(p); + if( pCsr->iPage>0 ){ + pCsr->iPage--; + }else if( pCsr->isAgg ){ + /* label-statNext-done: When computing aggregate space usage over + ** an entire btree, this is the exit point from this function */ + return SQLITE_OK; + } + goto statNextRestart; /* Tail recursion */ + } + pCsr->iPage++; + if( pCsr->iPage>=ArraySize(pCsr->aPage) ){ + statResetCsr(pCsr); + return SQLITE_CORRUPT_BKPT; + } + assert( p==&pCsr->aPage[pCsr->iPage-1] ); + + if( p->iCell==p->nCell ){ + p[1].iPgno = p->iRightChildPg; + }else{ + p[1].iPgno = p->aCell[p->iCell].iChildPg; + } + rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg, 0); + pCsr->nPage++; + p[1].iCell = 0; + if( !pCsr->isAgg ){ + p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } + p->iCell++; + } + + + /* Populate the StatCursor fields with the values to be returned + ** by the xColumn() and xRowid() methods. + */ + if( rc==SQLITE_OK ){ + int i; + StatPage *p = &pCsr->aPage[pCsr->iPage]; + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); + pCsr->iPageno = p->iPgno; + + rc = statDecodePage(pBt, p); + if( rc==SQLITE_OK ){ + statSizeAndOffset(pCsr); + + switch( p->flags ){ + case 0x05: /* table internal */ + case 0x02: /* index internal */ + pCsr->zPagetype = "internal"; + break; + case 0x0D: /* table leaf */ + case 0x0A: /* index leaf */ + pCsr->zPagetype = "leaf"; + break; + default: + pCsr->zPagetype = "corrupted"; + break; + } + pCsr->nCell += p->nCell; + pCsr->nUnused += p->nUnused; + if( p->nMxPayload>pCsr->nMxPayload ) pCsr->nMxPayload = p->nMxPayload; + if( !pCsr->isAgg ){ + pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } + nPayload = 0; + for(i=0; inCell; i++){ + nPayload += p->aCell[i].nLocal; + } + pCsr->nPayload += nPayload; + + /* If computing aggregate space usage by btree, continue with the + ** next page. The loop will exit via the return at label-statNext-done + */ + if( pCsr->isAgg ) goto statNextRestart; + } + } + + return rc; +} + +static int statEof(sqlite3_vtab_cursor *pCursor){ + StatCursor *pCsr = (StatCursor *)pCursor; + return pCsr->isEof; +} + +/* Initialize a cursor according to the query plan idxNum using the +** arguments in argv[0]. See statBestIndex() for a description of the +** meaning of the bits in idxNum. +*/ +static int statFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable*)(pCursor->pVtab); + sqlite3_str *pSql; /* Query of btrees to analyze */ + char *zSql; /* String value of pSql */ + int iArg = 0; /* Count of argv[] parameters used so far */ + int rc = SQLITE_OK; /* Result of this operation */ + const char *zName = 0; /* Only provide analysis of this table */ + + statResetCsr(pCsr); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( idxNum & 0x01 ){ + /* schema=? constraint is present. Get its value */ + const char *zDbase = (const char*)sqlite3_value_text(argv[iArg++]); + pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); + if( pCsr->iDb<0 ){ + pCsr->iDb = 0; + pCsr->isEof = 1; + return SQLITE_OK; + } + }else{ + pCsr->iDb = pTab->iDb; + } + if( idxNum & 0x02 ){ + /* name=? constraint is present */ + zName = (const char*)sqlite3_value_text(argv[iArg++]); + } + if( idxNum & 0x04 ){ + /* aggregate=? constraint is present */ + pCsr->isAgg = sqlite3_value_double(argv[iArg++])!=0.0; + }else{ + pCsr->isAgg = 0; + } + pSql = sqlite3_str_new(pTab->db); + sqlite3_str_appendf(pSql, + "SELECT * FROM (" + "SELECT 'sqlite_schema' AS name,1 AS rootpage,'table' AS type" + " UNION ALL " + "SELECT name,rootpage,type" + " FROM \"%w\".sqlite_schema WHERE rootpage!=0)", + pTab->db->aDb[pCsr->iDb].zDbSName); + if( zName ){ + sqlite3_str_appendf(pSql, "WHERE name=%Q", zName); + } + if( idxNum & 0x08 ){ + sqlite3_str_appendf(pSql, " ORDER BY name"); + } + zSql = sqlite3_str_finish(pSql); + if( zSql==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + rc = statNext(pCursor); + } + return rc; +} + +static int statColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + StatCursor *pCsr = (StatCursor *)pCursor; + switch( i ){ + case 0: /* name */ + sqlite3_result_text(ctx, pCsr->zName, -1, SQLITE_TRANSIENT); + break; + case 1: /* path */ + if( !pCsr->isAgg ){ + sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT); + } + break; + case 2: /* pageno */ + if( pCsr->isAgg ){ + sqlite3_result_int64(ctx, pCsr->nPage); + }else{ + sqlite3_result_int64(ctx, pCsr->iPageno); + } + break; + case 3: /* pagetype */ + if( !pCsr->isAgg ){ + sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); + } + break; + case 4: /* ncell */ + sqlite3_result_int(ctx, pCsr->nCell); + break; + case 5: /* payload */ + sqlite3_result_int(ctx, pCsr->nPayload); + break; + case 6: /* unused */ + sqlite3_result_int(ctx, pCsr->nUnused); + break; + case 7: /* mx_payload */ + sqlite3_result_int(ctx, pCsr->nMxPayload); + break; + case 8: /* pgoffset */ + if( !pCsr->isAgg ){ + sqlite3_result_int64(ctx, pCsr->iOffset); + } + break; + case 9: /* pgsize */ + sqlite3_result_int(ctx, pCsr->szPage); + break; + case 10: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + int iDb = pCsr->iDb; + sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + default: { /* aggregate */ + sqlite3_result_int(ctx, pCsr->isAgg); + break; + } + } + return SQLITE_OK; +} + +static int statRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + StatCursor *pCsr = (StatCursor *)pCursor; + *pRowid = pCsr->iPageno; + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "dbstat" virtual table module +*/ +int sqlite3DbstatRegister(sqlite3 *db){ + static sqlite3_module dbstat_module = { + 0, /* iVersion */ + statConnect, /* xCreate */ + statConnect, /* xConnect */ + statBestIndex, /* xBestIndex */ + statDisconnect, /* xDisconnect */ + statDisconnect, /* xDestroy */ + statOpen, /* xOpen - open a cursor */ + statClose, /* xClose - close a cursor */ + statFilter, /* xFilter - configure scan constraints */ + statNext, /* xNext - advance a cursor */ + statEof, /* xEof - check for end of scan */ + statColumn, /* xColumn - read data */ + statRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); +} +#elif defined(SQLITE_ENABLE_DBSTAT_VTAB) +int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ diff --git a/third_party/sqlite3/delete.c b/third_party/sqlite3/delete.c new file mode 100644 index 000000000..e0a86533b --- /dev/null +++ b/third_party/sqlite3/delete.c @@ -0,0 +1,968 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** in order to generate code for DELETE FROM statements. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +/* +** While a SrcList can in general represent multiple tables and subqueries +** (as in the FROM clause of a SELECT statement) in this case it contains +** the name of a single table, as one might find in an INSERT, DELETE, +** or UPDATE statement. Look up that table in the symbol table and +** return a pointer. Set an error message and return NULL if the table +** name is not found or if any other error occurs. +** +** The following fields are initialized appropriate in pSrc: +** +** pSrc->a[0].pTab Pointer to the Table object +** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** +*/ +Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ + SrcItem *pItem = pSrc->a; + Table *pTab; + assert( pItem && pSrc->nSrc>=1 ); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); + sqlite3DeleteTable(pParse->db, pItem->pTab); + pItem->pTab = pTab; + if( pTab ){ + pTab->nTabRef++; + if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; + } + } + return pTab; +} + +/* Return true if table pTab is read-only. +** +** A table is read-only if any of the following are true: +** +** 1) It is a virtual table and no implementation of the xUpdate method +** has been provided +** +** 2) It is a system table (i.e. sqlite_schema), this call is not +** part of a nested parse and writable_schema pragma has not +** been specified +** +** 3) The table is a shadow table, the database connection is in +** defensive mode, and the current sqlite3_prepare() +** is for a top-level SQL statement. +*/ +static int tabIsReadOnly(Parse *pParse, Table *pTab){ + sqlite3 *db; + if( IsVirtual(pTab) ){ + return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + } + if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; + db = pParse->db; + if( (pTab->tabFlags & TF_Readonly)!=0 ){ + return sqlite3WritableSchema(db)==0 && pParse->nested==0; + } + assert( pTab->tabFlags & TF_Shadow ); + return sqlite3ReadOnlyShadowTables(db); +} + +/* +** Check to make sure the given table is writable. If it is not +** writable, generate an error message and return 1. If it is +** writable return 0; +*/ +int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ + if( tabIsReadOnly(pParse, pTab) ){ + sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); + return 1; + } +#ifndef SQLITE_OMIT_VIEW + if( !viewOk && pTab->pSelect ){ + sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); + return 1; + } +#endif + return 0; +} + + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +/* +** Evaluate a view and store its result in an ephemeral table. The +** pWhere argument is an optional WHERE clause that restricts the +** set of rows in the view that are to be added to the ephemeral table. +*/ +void sqlite3MaterializeView( + Parse *pParse, /* Parsing context */ + Table *pView, /* View definition */ + Expr *pWhere, /* Optional WHERE clause to be added */ + ExprList *pOrderBy, /* Optional ORDER BY clause */ + Expr *pLimit, /* Optional LIMIT clause */ + int iCur /* Cursor number for ephemeral table */ +){ + SelectDest dest; + Select *pSel; + SrcList *pFrom; + sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); + } + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, + SF_IncludeHidden, pLimit); + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); +} +#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Generate an expression tree to implement the WHERE, ORDER BY, +** and LIMIT/OFFSET portion of DELETE and UPDATE statements. +** +** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; +** \__________________________/ +** pLimitWhere (pInClause) +*/ +Expr *sqlite3LimitWhere( + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ +){ + sqlite3 *db = pParse->db; + Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ + Table *pTab; + + /* Check that there isn't an ORDER BY without a LIMIT clause. + */ + if( pOrderBy && pLimit==0 ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + return 0; + } + + /* We only need to generate a select expression if there + ** is a limit/offset term to enforce. + */ + if( pLimit == 0 ) { + return pWhere; + } + + /* Generate a select expression tree to enforce the limit/offset + ** term for the DELETE or UPDATE statement. For example: + ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** becomes: + ** DELETE FROM table_a WHERE rowid IN ( + ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** ); + */ + + pTab = pSrc->a[0].pTab; + if( HasRowid(pTab) ){ + pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); + pEList = sqlite3ExprListAppend( + pParse, 0, sqlite3PExpr(pParse, TK_ROW, 0, 0) + ); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + if( pPk->nKeyCol==1 ){ + const char *zName = pTab->aCol[pPk->aiColumn[0]].zName; + pLhs = sqlite3Expr(db, TK_ID, zName); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); + }else{ + int i; + for(i=0; inKeyCol; i++){ + Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zName); + pEList = sqlite3ExprListAppend(pParse, pEList, p); + } + pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( pLhs ){ + pLhs->x.pList = sqlite3ExprListDup(db, pEList, 0); + } + } + } + + /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree + ** and the SELECT subtree. */ + pSrc->a[0].pTab = 0; + pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); + pSrc->a[0].pTab = pTab; + if( pSrc->a[0].fg.isIndexedBy ){ + pSrc->a[0].u2.pIBIndex = 0; + pSrc->a[0].fg.isIndexedBy = 0; + sqlite3DbFree(db, pSrc->a[0].u1.zIndexedBy); + }else if( pSrc->a[0].fg.isCte ){ + pSrc->a[0].u2.pCteUse->nUse++; + } + + /* generate the SELECT expression tree. */ + pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, + pOrderBy,0,pLimit + ); + + /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); + sqlite3PExprAddSelect(pParse, pInClause, pSelect); + return pInClause; +} +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ + /* && !defined(SQLITE_OMIT_SUBQUERY) */ + +/* +** Generate code for a DELETE FROM statement. +** +** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; +** \________/ \________________/ +** pTabList pWhere +*/ +void sqlite3DeleteFrom( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table from which we should delete things */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit /* LIMIT clause. May be null */ +){ + Vdbe *v; /* The virtual database engine */ + Table *pTab; /* The table from which records will be deleted */ + int i; /* Loop counter */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Index *pIdx; /* For looping over indices of the table */ + int iTabCur; /* Cursor number for the table */ + int iDataCur = 0; /* VDBE cursor for the canonical data source */ + int iIdxCur = 0; /* Cursor number of the first index */ + int nIdx; /* Number of indices */ + sqlite3 *db; /* Main database structure */ + AuthContext sContext; /* Authorization context */ + NameContext sNC; /* Name context to resolve expressions in */ + int iDb; /* Database number */ + int memCnt = 0; /* Memory cell used for change counting */ + int rcauth; /* Value returned by authorization callback */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ + Index *pPk; /* The PRIMARY KEY index on the table */ + int iPk = 0; /* First of nPk registers holding PRIMARY KEY value */ + i16 nPk = 1; /* Number of columns in the PRIMARY KEY */ + int iKey; /* Memory cell holding key of row to be deleted */ + i16 nKey; /* Number of memory cells in the row key */ + int iEphCur = 0; /* Ephemeral table holding all primary key values */ + int iRowSet = 0; /* Register for rowset of rows to delete */ + int addrBypass = 0; /* Address of jump over the delete logic */ + int addrLoop = 0; /* Top of the delete loop */ + int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + int bComplex; /* True if there are triggers or FKs or + ** subqueries in the WHERE clause */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to delete from a view */ + Trigger *pTrigger; /* List of table triggers, if required */ +#endif + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto delete_from_cleanup; + } + assert( pTabList->nSrc==1 ); + + + /* Locate the table which we want to delete. This table has to be + ** put in an SrcList structure because some of the subroutines we + ** will be calling are designed to work with multiple tables and expect + ** an SrcList* parameter instead of just a Table* parameter. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto delete_from_cleanup; + + /* Figure out if we have any triggers and if the table being + ** deleted from is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define isView 0 +#endif + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, "DELETE" + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + + /* If pTab is really a view, make sure it has been initialized. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto delete_from_cleanup; + } + + if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + goto delete_from_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); + assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); + if( rcauth==SQLITE_DENY ){ + goto delete_from_cleanup; + } + assert(!isView || pTrigger); + + /* Assign cursor numbers to the table and all its indices. + */ + assert( pTabList->nSrc==1 ); + iTabCur = pTabList->a[0].iCursor = pParse->nTab++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + pParse->nTab++; + } + + /* Start the view context + */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* Begin generating code. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto delete_from_cleanup; + } + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, bComplex, iDb); + + /* If we are trying to delete from a view, realize that view into + ** an ephemeral table. + */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, iTabCur + ); + iDataCur = iIdxCur = iTabCur; + pOrderBy = 0; + pLimit = 0; + } +#endif + + /* Resolve the column names in the WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto delete_from_cleanup; + } + + /* Initialize the counter of the number of rows deleted, if + ** we are counting rows. + */ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ + memCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); + } + +#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + /* Special case: A DELETE without a WHERE clause deletes everything. + ** It is easier just to erase the whole table. Prior to version 3.6.5, + ** this optimization caused the row change count (the value returned by + ** API function sqlite3_count_changes) to be set incorrectly. + ** + ** The "rcauth==SQLITE_OK" terms is the + ** IMPLEMENTATION-OF: R-17228-37124 If the action code is SQLITE_DELETE and + ** the callback returns SQLITE_IGNORE then the DELETE operation proceeds but + ** the truncate optimization is disabled and all rows are deleted + ** individually. + */ + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + && db->xPreUpdateCallback==0 +#endif + ){ + assert( !isView ); + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1, + pTab->zName, P4_STATIC); + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + } + }else +#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ + { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); + if( HasRowid(pTab) ){ + /* For a rowid table, initialize the RowSet to an empty set */ + pPk = 0; + nPk = 1; + iRowSet = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + }else{ + /* For a WITHOUT ROWID table, create an ephemeral table used to + ** hold all primary keys for rows to be deleted. */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + iEphCur = pParse->nTab++; + addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + + /* Construct a query to find the rowid or primary key for every row + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. + */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); + if( pWInfo==0 ) goto delete_from_cleanup; + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); + if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); + } + + /* Keep track of the number of rows to be deleted */ + if( memCnt ){ + sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); + } + + /* Extract the rowid or primary key for the current row */ + if( pPk ){ + for(i=0; iaiColumn[i]>=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pPk->aiColumn[i], iPk+i); + } + iKey = iPk; + }else{ + iKey = ++pParse->nMem; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); + } + + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only + ** one, so just keep it in its register(s) and fall through to the + ** delete code. */ + nKey = nPk; /* OP_Found will use an unpacked key */ + aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); + if( aToOpen==0 ){ + sqlite3WhereEnd(pWInfo); + goto delete_from_cleanup; + } + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; + if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrBypass = sqlite3VdbeMakeLabel(pParse); + }else{ + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_DeferredSeek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } + sqlite3WhereEnd(pWInfo); + } + + /* Unless this is a view, open cursors for the table we are + ** deleting from and all its indices. If this is a view, then the + ** only effect this statement has is to fire the INSTEAD OF + ** triggers. + */ + if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + testcase( IsVirtual(pTab) ); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, + iTabCur, aToOpen, &iDataCur, &iIdxCur); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeJumpHereOrPopInst(v, iAddrOnce); + } + } + + /* Set up a loop over the rowids/primary-keys that were found in the + ** where-clause loop above. + */ + if( eOnePass!=ONEPASS_OFF ){ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ + assert( pPk!=0 || pTab->pSelect!=0 ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); + VdbeCoverage(v); + } + }else if( pPk ){ + addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iEphCur, 0, iKey); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + } + assert( nKey==0 ); /* OP_Found will use a composite key */ + }else{ + addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); + VdbeCoverage(v); + assert( nKey==1 ); + } + + /* Delete the row */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); + sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE ){ + sqlite3VdbeAddOp1(v, OP_Close, iTabCur); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + } + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); + }else +#endif + { + int count = (pParse->nested==0); /* True to count changes */ + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); + } + + /* End of the loop over all rowids/primary-keys. */ + if( eOnePass!=ONEPASS_OFF ){ + sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); + }else if( pPk ){ + sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrLoop); + }else{ + sqlite3VdbeGoto(v, addrLoop); + sqlite3VdbeJumpHere(v, addrLoop); + } + } /* End non-truncate path */ + + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* Return the number of rows that were deleted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( memCnt ){ + sqlite3VdbeAddOp2(v, OP_ChngCntRow, memCnt, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); + } + +delete_from_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); +#endif + sqlite3DbFree(db, aToOpen); + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** they may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif + +/* +** This routine generates VDBE code that causes a single row of a +** single table to be deleted. Both the original table entry and +** all indices are removed. +** +** Preconditions: +** +** 1. iDataCur is an open cursor on the btree that is the canonical data +** store for the table. (This will be either the table itself, +** in the case of a rowid table, or the PRIMARY KEY index in the case +** of a WITHOUT ROWID table.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. +** +** 3. The primary key for the row to be deleted must be stored in a +** sequence of nPk memory cells starting at iPk. If nPk==0 that means +** that a search record formed from OP_MakeRecord is contained in the +** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. +*/ +void sqlite3GenerateRowDelete( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the row to be deleted */ + Trigger *pTrigger, /* List of triggers to (potentially) fire */ + int iDataCur, /* Cursor from which column data is extracted */ + int iIdxCur, /* First index cursor */ + int iPk, /* First memory cell containing the PRIMARY KEY */ + i16 nPk, /* Number of PRIMARY KEY memory cells */ + u8 count, /* If non-zero, increment the row change counter */ + u8 onconf, /* Default ON CONFLICT policy for triggers */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ +){ + Vdbe *v = pParse->pVdbe; /* Vdbe */ + int iOld = 0; /* First register in OLD.* array */ + int iLabel; /* Label resolved to end of generated code */ + u8 opSeek; /* Seek opcode */ + + /* Vdbe is guaranteed to have been allocated by this stage. */ + assert( v ); + VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", + iDataCur, iIdxCur, iPk, (int)nPk)); + + /* Seek cursor iCur to the row to delete. If this row no longer exists + ** (this can happen if a trigger program has already deleted it), do + ** not attempt to delete it or fire any DELETE triggers. */ + iLabel = sqlite3VdbeMakeLabel(pParse); + opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + if( eMode==ONEPASS_OFF ){ + sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + VdbeCoverageIf(v, opSeek==OP_NotExists); + VdbeCoverageIf(v, opSeek==OP_NotFound); + } + + /* If there are any triggers to fire, allocate a range of registers to + ** use for the old.* references in the triggers. */ + if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ + u32 mask; /* Mask of OLD.* columns in use */ + int iCol; /* Iterator used while populating OLD.* */ + int addrStart; /* Start of BEFORE trigger programs */ + + /* TODO: Could use temporary registers here. Also could attempt to + ** avoid copying the contents of the rowid register. */ + mask = sqlite3TriggerColmask( + pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + ); + mask |= sqlite3FkOldmask(pParse, pTab); + iOld = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); + + /* Populate the OLD.* pseudo-table register array. These values will be + ** used by any BEFORE and AFTER triggers that exist. */ + sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); + for(iCol=0; iColnCol; iCol++){ + testcase( mask!=0xffffffff && iCol==31 ); + testcase( mask!=0xffffffff && iCol==32 ); + if( mask==0xffffffff || (iCol<=31 && (mask & MASKBIT32(iCol))!=0) ){ + int kk = sqlite3TableColumnToStorage(pTab, iCol); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+kk+1); + } + } + + /* Invoke BEFORE DELETE trigger programs. */ + addrStart = sqlite3VdbeCurrentAddr(v); + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel + ); + + /* If any BEFORE triggers were coded, then seek the cursor to the + ** row to be deleted again. It may be that the BEFORE triggers moved + ** the cursor or already deleted the row that the cursor was + ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. + */ + if( addrStart=0 ); + iIdxNoSeek = -1; + } + + /* Do FK processing. This call checks that any FK constraints that + ** refer to this table (i.e. constraints attached to other tables) + ** are not violated by deleting this row. */ + sqlite3FkCheck(pParse, pTab, iOld, 0, 0, 0); + } + + /* Delete the index and table entries. Skip this step if pTab is really + ** a view (in which case the only effect of the DELETE statement is to + ** fire the INSTEAD OF triggers). + ** + ** If variable 'count' is non-zero, then this OP_Delete instruction should + ** invoke the update-hook. The pre-update-hook, on the other hand should + ** be invoked unless table pTab is a system table. The difference is that + ** the update-hook is not invoked for rows removed by REPLACE, but the + ** pre-update-hook is. + */ + if( pTab->pSelect==0 ){ + u8 p5 = 0; + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); + if( pParse->nested==0 || 0==sqlite3_stricmp(pTab->zName, "sqlite_stat1") ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + } + if( eMode!=ONEPASS_OFF ){ + sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); + } + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; + sqlite3VdbeChangeP5(v, p5); + } + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just deleted. */ + sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); + + /* Invoke AFTER DELETE trigger programs. */ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + + /* Jump here if the row had already been deleted before any BEFORE + ** trigger programs were invoked. Or if a trigger program throws a + ** RAISE(IGNORE) exception. */ + sqlite3VdbeResolveLabel(v, iLabel); + VdbeModuleComment((v, "END: GenRowDel()")); +} + +/* +** This routine generates VDBE code that causes the deletion of all +** index entries associated with a single row of a single table, pTab +** +** Preconditions: +** +** 1. A read/write cursor "iDataCur" must be open on the canonical storage +** btree for the table pTab. (This will be either the table itself +** for rowid tables or to the primary key index for WITHOUT ROWID +** tables.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. (The pTab->pIndex +** index is the 0-th index.) +** +** 3. The "iDataCur" cursor must be already be positioned on the row +** that is to be deleted. +*/ +void sqlite3GenerateRowIndexDelete( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Table containing the row to be deleted */ + int iDataCur, /* Cursor of table holding data. */ + int iIdxCur, /* First index cursor */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ +){ + int i; /* Index loop counter */ + int r1 = -1; /* Register holding an index key */ + int iPartIdxLabel; /* Jump destination for skipping partial index entries */ + Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ + Vdbe *v; /* The prepared statement under construction */ + Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ + + v = pParse->pVdbe; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + assert( iIdxCur+i!=iDataCur || pPk==pIdx ); + if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; + if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; + VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + pPrior = pIdx; + } +} + +/* +** Generate code that will assemble an index key and stores it in register +** regOut. The key with be for index pIdx which is an index on pTab. +** iCur is the index of a cursor open on the pTab table and pointing to +** the entry that needs indexing. If pTab is a WITHOUT ROWID table, then +** iCur must be the cursor of the PRIMARY KEY index. +** +** Return a register number which is the first in a block of +** registers that holds the elements of the index key. The +** block of registers has already been deallocated by the time +** this routine returns. +** +** If *piPartIdxLabel is not NULL, fill it in with a label and jump +** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). +** A partial index should be skipped if its WHERE clause evaluates +** to false or null. If pIdx is not a partial index, *piPartIdxLabel +** will be set to zero which is an empty label that is ignored by +** sqlite3ResolvePartIdxLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. +*/ +int sqlite3GenerateIndexKey( + Parse *pParse, /* Parsing context */ + Index *pIdx, /* The index for which to generate a key */ + int iDataCur, /* Cursor number from which to take column data */ + int regOut, /* Put the new key into this register if not 0 */ + int prefixOnly, /* Compute only a unique prefix of the key */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ +){ + Vdbe *v = pParse->pVdbe; + int j; + int regBase; + int nCol; + + if( piPartIdxLabel ){ + if( pIdx->pPartIdxWhere ){ + *piPartIdxLabel = sqlite3VdbeMakeLabel(pParse); + pParse->iSelfTab = iDataCur + 1; + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; + pPrior = 0; /* Ticket a9efb42811fa41ee 2019-11-02; + ** pPartIdxWhere may have corrupted regPrior registers */ + }else{ + *piPartIdxLabel = 0; + } + } + nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; + for(j=0; jaiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the OP_RealAffinity + ** opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); + } + if( regOut ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + } + sqlite3ReleaseTempRange(pParse, regBase, nCol); + return regBase; +} + +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + } +} diff --git a/third_party/sqlite3/expr.c b/third_party/sqlite3/expr.c new file mode 100644 index 000000000..cfab346ec --- /dev/null +++ b/third_party/sqlite3/expr.c @@ -0,0 +1,6119 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for analyzing expressions and +** for generating VDBE code that evaluates expressions in SQLite. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +char sqlite3TableColumnAffinity(Table *pTab, int iCol){ + assert( iColnCol ); + return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; +} + +/* +** Return the 'affinity' of the expression pExpr if any. +** +** If pExpr is a column, a reference to a column via an 'AS' alias, +** or a sub-select with a column as the return value, then the +** affinity of that column is returned. Otherwise, 0x00 is returned, +** indicating no affinity for the expression. +** +** i.e. the WHERE clause expressions in the following statements all +** have an affinity: +** +** CREATE TABLE t1(a); +** SELECT * FROM t1 WHERE a; +** SELECT a AS b FROM t1 WHERE b; +** SELECT * FROM t1 WHERE (select a from t1); +*/ +char sqlite3ExprAffinity(const Expr *pExpr){ + int op; + while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + assert( pExpr!=0 ); + } + op = pExpr->op; + if( op==TK_SELECT ){ + assert( pExpr->flags&EP_xIsSelect ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } + if( op==TK_REGISTER ) op = pExpr->op2; +#ifndef SQLITE_OMIT_CAST + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } +#endif + if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->y.pTab ){ + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + } + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft->flags&EP_xIsSelect ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + return pExpr->affExpr; +} + +/* +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to a new Expr node that +** implements the COLLATE operator. +** +** If a memory allocation error occurs, that fact is recorded in pParse->db +** and the pExpr parameter is returned unchanged. +*/ +Expr *sqlite3ExprAddCollateToken( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName, /* Name of collating sequence */ + int dequote /* True to dequote pCollName */ +){ + assert( pExpr!=0 || pParse->db->mallocFailed ); + if( pExpr==0 ) return 0; + if( pExpr->op==TK_VECTOR ){ + ExprList *pList = pExpr->x.pList; + if( ALWAYS(pList!=0) ){ + int i; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = sqlite3ExprAddCollateToken(pParse,pList->a[i].pExpr, + pCollName, dequote); + } + } + }else if( pCollName->n>0 ){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); + if( pNew ){ + pNew->pLeft = pExpr; + pNew->flags |= EP_Collate|EP_Skip; + pExpr = pNew; + } + } + return pExpr; +} +Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ + Token s; + assert( zC!=0 ); + sqlite3TokenInit(&s, (char*)zC); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); +} + +/* +** Skip over any TK_COLLATE operators. +*/ +Expr *sqlite3ExprSkipCollate(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** Skip over any TK_COLLATE operators and/or any unlikely() +** or likelihood() or likely() functions at the root of an +** expression. +*/ +Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip|EP_Unlikely) ){ + if( ExprHasProperty(pExpr, EP_Unlikely) ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( pExpr->x.pList->nExpr>0 ); + assert( pExpr->op==TK_FUNCTION ); + pExpr = pExpr->x.pList->a[0].pExpr; + }else{ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + } + return pExpr; +} + +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return NULL. +** +** See also: sqlite3ExprNNCollSeq() +** +** The sqlite3ExprNNCollSeq() works the same exact that it returns the +** default collation if pExpr has no defined collation. +** +** The collating sequence might be determined by a COLLATE operator +** or by the presence of a column with a defined collating sequence. +** COLLATE operators take first precedence. Left operands take +** precedence over right operands. +*/ +CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ + sqlite3 *db = pParse->db; + CollSeq *pColl = 0; + const Expr *p = pExpr; + while( p ){ + int op = p->op; + if( op==TK_REGISTER ) op = p->op2; + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER) + && p->y.pTab!=0 + ){ + /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = p->iColumn; + if( j>=0 ){ + const char *zColl = p->y.pTab->aCol[j].zColl; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + } + break; + } + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + if( op==TK_VECTOR ){ + p = p->x.pList->a[0].pExpr; + continue; + } + if( op==TK_COLLATE ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } + if( p->flags & EP_Collate ){ + if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ + p = p->pLeft; + }else{ + Expr *pNext = p->pRight; + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + if( p->x.pList!=0 + && !db->mallocFailed + && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) + ){ + int i; + for(i=0; ALWAYS(ix.pList->nExpr); i++){ + if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ + pNext = p->x.pList->a[i].pExpr; + break; + } + } + } + p = pNext; + } + }else{ + break; + } + } + if( sqlite3CheckCollSeq(pParse, pColl) ){ + pColl = 0; + } + return pColl; +} + +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return a pointer to the +** defautl collation sequence. +** +** See also: sqlite3ExprCollSeq() +** +** The sqlite3ExprCollSeq() routine works the same except that it +** returns NULL if there is no defined collation. +*/ +CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr){ + CollSeq *p = sqlite3ExprCollSeq(pParse, pExpr); + if( p==0 ) p = pParse->db->pDfltColl; + assert( p!=0 ); + return p; +} + +/* +** Return TRUE if the two expressions have equivalent collating sequences. +*/ +int sqlite3ExprCollSeqMatch(Parse *pParse, const Expr *pE1, const Expr *pE2){ + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pE1); + CollSeq *pColl2 = sqlite3ExprNNCollSeq(pParse, pE2); + return sqlite3StrICmp(pColl1->zName, pColl2->zName)==0; +} + +/* +** pExpr is an operand of a comparison operator. aff2 is the +** type affinity of the other operand. This routine returns the +** type affinity that should be used for the comparison operator. +*/ +char sqlite3CompareAffinity(const Expr *pExpr, char aff2){ + char aff1 = sqlite3ExprAffinity(pExpr); + if( aff1>SQLITE_AFF_NONE && aff2>SQLITE_AFF_NONE ){ + /* Both sides of the comparison are columns. If one has numeric + ** affinity, use that. Otherwise use no affinity. + */ + if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ + return SQLITE_AFF_NUMERIC; + }else{ + return SQLITE_AFF_BLOB; + } + }else{ + /* One side is a column, the other is not. Use the columns affinity. */ + assert( aff1<=SQLITE_AFF_NONE || aff2<=SQLITE_AFF_NONE ); + return (aff1<=SQLITE_AFF_NONE ? aff2 : aff1) | SQLITE_AFF_NONE; + } +} + +/* +** pExpr is a comparison operator. Return the type affinity that should +** be applied to both operands prior to doing the comparison. +*/ +static char comparisonAffinity(const Expr *pExpr){ + char aff; + assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || + pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || + pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); + assert( pExpr->pLeft ); + aff = sqlite3ExprAffinity(pExpr->pLeft); + if( pExpr->pRight ){ + aff = sqlite3CompareAffinity(pExpr->pRight, aff); + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); + }else if( aff==0 ){ + aff = SQLITE_AFF_BLOB; + } + return aff; +} + +/* +** pExpr is a comparison expression, eg. '=', '<', IN(...) etc. +** idx_affinity is the affinity of an indexed column. Return true +** if the index with affinity idx_affinity may be used to implement +** the comparison in pExpr. +*/ +int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity){ + char aff = comparisonAffinity(pExpr); + if( affflags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + }else if( pRight && (pRight->flags & EP_Collate)!=0 ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + }else{ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + if( !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + } + } + return pColl; +} + +/* Expresssion p is a comparison operator. Return a collation sequence +** appropriate for the comparison operator. +** +** This is normally just a wrapper around sqlite3BinaryCompareCollSeq(). +** However, if the OP_Commuted flag is set, then the order of the operands +** is reversed in the sqlite3BinaryCompareCollSeq() call so that the +** correct collating sequence is found. +*/ +CollSeq *sqlite3ExprCompareCollSeq(Parse *pParse, const Expr *p){ + if( ExprHasProperty(p, EP_Commuted) ){ + return sqlite3BinaryCompareCollSeq(pParse, p->pRight, p->pLeft); + }else{ + return sqlite3BinaryCompareCollSeq(pParse, p->pLeft, p->pRight); + } +} + +/* +** Generate code for a comparison operator. +*/ +static int codeCompare( + Parse *pParse, /* The parsing (and code generating) context */ + Expr *pLeft, /* The left operand */ + Expr *pRight, /* The right operand */ + int opcode, /* The comparison opcode */ + int in1, int in2, /* Register holding operands */ + int dest, /* Jump here if true. */ + int jumpIfNull, /* If true, jump if either operand is NULL */ + int isCommuted /* The comparison has been commuted */ +){ + int p5; + int addr; + CollSeq *p4; + + if( pParse->nErr ) return 0; + if( isCommuted ){ + p4 = sqlite3BinaryCompareCollSeq(pParse, pRight, pLeft); + }else{ + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + } + p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); + addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, + (void*)p4, P4_COLLSEQ); + sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + return addr; +} + +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +int sqlite3ExprIsVector(Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +int sqlite3ExprVectorSize(Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} + +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField /* Which column of the vector to return */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( pVector->flags & EP_xIsSelect ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT. Not deleted. + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** iTable: 0 or the number of columns on the LHS of an assignment + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); + if( pRet ){ + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + assert( pRet==0 || pRet->iTable==0 ); + }else{ + if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; + pRet = sqlite3ExprDup(pParse->db, pVector, 0); +#ifndef SQLITE_OMIT_ALTERTABLE + sqlite3RenameTokenRemap(pParse, pRet, pVector); +#endif + } + return pRet; +} + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrDone = sqlite3VdbeMakeLabel(pParse); + int isCommuted = ExprHasProperty(pExpr,EP_Commuted); + + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + if( pParse->nErr ) return; + if( nLeft!=sqlite3ExprVectorSize(pRight) ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return; + } + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + p5 |= SQLITE_STOREP2; + if( opx==TK_LE ) opx = TK_LT; + if( opx==TK_GE ) opx = TK_GT; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL, *pR; + int r1, r2; + assert( i>=0 && i0 +/* +** Check that argument nHeight is less than or equal to the maximum +** expression depth allowed. If it is not, leave an error message in +** pParse. +*/ +int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ + int rc = SQLITE_OK; + int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; + if( nHeight>mxHeight ){ + sqlite3ErrorMsg(pParse, + "Expression tree is too large (maximum depth %d)", mxHeight + ); + rc = SQLITE_ERROR; + } + return rc; +} + +/* The following three functions, heightOfExpr(), heightOfExprList() +** and heightOfSelect(), are used to determine the maximum height +** of any expression tree referenced by the structure passed as the +** first argument. +** +** If this maximum height is greater than the current value pointed +** to by pnHeight, the second parameter, then set *pnHeight to that +** value. +*/ +static void heightOfExpr(Expr *p, int *pnHeight){ + if( p ){ + if( p->nHeight>*pnHeight ){ + *pnHeight = p->nHeight; + } + } +} +static void heightOfExprList(ExprList *p, int *pnHeight){ + if( p ){ + int i; + for(i=0; inExpr; i++){ + heightOfExpr(p->a[i].pExpr, pnHeight); + } + } +} +static void heightOfSelect(Select *pSelect, int *pnHeight){ + Select *p; + for(p=pSelect; p; p=p->pPrior){ + heightOfExpr(p->pWhere, pnHeight); + heightOfExpr(p->pHaving, pnHeight); + heightOfExpr(p->pLimit, pnHeight); + heightOfExprList(p->pEList, pnHeight); + heightOfExprList(p->pGroupBy, pnHeight); + heightOfExprList(p->pOrderBy, pnHeight); + } +} + +/* +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or +** Expr.pSelect member has a height of 1. Any other expression +** has a height equal to the maximum height of any other +** referenced Expr plus one. +** +** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, +** if appropriate. +*/ +static void exprSetHeight(Expr *p){ + int nHeight = 0; + heightOfExpr(p->pLeft, &nHeight); + heightOfExpr(p->pRight, &nHeight); + if( ExprHasProperty(p, EP_xIsSelect) ){ + heightOfSelect(p->x.pSelect, &nHeight); + }else if( p->x.pList ){ + heightOfExprList(p->x.pList, &nHeight); + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } + p->nHeight = nHeight + 1; +} + +/* +** Set the Expr.nHeight variable using the exprSetHeight() function. If +** the height is greater than the maximum allowed expression depth, +** leave an error in pParse. +** +** Also propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; + exprSetHeight(p); + sqlite3ExprCheckHeight(pParse, p->nHeight); +} + +/* +** Return the maximum height of any expression tree referenced +** by the select statement passed as an argument. +*/ +int sqlite3SelectExprHeight(Select *p){ + int nHeight = 0; + heightOfSelect(p, &nHeight); + return nHeight; +} +#else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ +/* +** Propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; + if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } +} +#define exprSetHeight(y) +#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ + +/* +** This routine is the core allocator for Expr nodes. +** +** Construct a new expression node and return a pointer to it. Memory +** for this node and for the pToken argument is a single allocation +** obtained from sqlite3DbMalloc(). The calling function +** is responsible for making sure the node eventually gets freed. +** +** If dequote is true, then the token (if it exists) is dequoted. +** If dequote is false, no dequoting is performed. The deQuote +** parameter is ignored if pToken is NULL or if the token does not +** appear to be quoted. If the quotes were of the form "..." (double-quotes) +** then the EP_DblQuoted flag is set on the expression node. +** +** Special case: If op==TK_INTEGER and pToken points to a string that +** can be translated into a 32-bit integer, then the token is not +** stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage +** is allocated to hold the integer text and the dequote flag is ignored. +*/ +Expr *sqlite3ExprAlloc( + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ + int op, /* Expression opcode */ + const Token *pToken, /* Token argument. Might be NULL */ + int dequote /* True to dequote */ +){ + Expr *pNew; + int nExtra = 0; + int iValue = 0; + + assert( db!=0 ); + if( pToken ){ + if( op!=TK_INTEGER || pToken->z==0 + || sqlite3GetInt32(pToken->z, &iValue)==0 ){ + nExtra = pToken->n+1; + assert( iValue>=0 ); + } + } + pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); + if( pNew ){ + memset(pNew, 0, sizeof(Expr)); + pNew->op = (u8)op; + pNew->iAgg = -1; + if( pToken ){ + if( nExtra==0 ){ + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iValue; + }else{ + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); + } + } + } +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + +/* +** Allocate a new expression node from a zero-terminated token that has +** already been dequoted. +*/ +Expr *sqlite3Expr( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const char *zToken /* Token argument. Might be NULL */ +){ + Token x; + x.z = zToken; + x.n = sqlite3Strlen30(zToken); + return sqlite3ExprAlloc(db, op, &x, 0); +} + +/* +** Attach subtrees pLeft and pRight to the Expr node pRoot. +** +** If pRoot==NULL that means that a memory allocation error has occurred. +** In that case, delete the subtrees pLeft and pRight. +*/ +void sqlite3ExprAttachSubtrees( + sqlite3 *db, + Expr *pRoot, + Expr *pLeft, + Expr *pRight +){ + if( pRoot==0 ){ + assert( db->mallocFailed ); + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + }else{ + if( pRight ){ + pRoot->pRight = pRight; + pRoot->flags |= EP_Propagate & pRight->flags; + } + if( pLeft ){ + pRoot->pLeft = pLeft; + pRoot->flags |= EP_Propagate & pLeft->flags; + } + exprSetHeight(pRoot); + } +} + +/* +** Allocate an Expr node which joins as many as two subtrees. +** +** One or both of the subtrees can be NULL. Return a pointer to the new +** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, +** free the subtrees and return NULL. +*/ +Expr *sqlite3PExpr( + Parse *pParse, /* Parsing context */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight /* Right operand */ +){ + Expr *p; + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & 0xff; + p->iAgg = -1; + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + sqlite3ExprCheckHeight(pParse, p->nHeight); + }else{ + sqlite3ExprDelete(pParse->db, pLeft); + sqlite3ExprDelete(pParse->db, pRight); + } + return p; +} + +/* +** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due +** do a memory allocation failure) then delete the pSelect object. +*/ +void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ + if( pExpr ){ + pExpr->x.pSelect = pSelect; + ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, pExpr); + }else{ + assert( pParse->db->mallocFailed ); + sqlite3SelectDelete(pParse->db, pSelect); + } +} + + +/* +** Join two expressions using an AND operator. If either expression is +** NULL, then just return the other expression. +** +** If one side or the other of the AND is known to be false, then instead +** of returning an AND expression, just return a constant expression with +** a value of false. +*/ +Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ + sqlite3 *db = pParse->db; + if( pLeft==0 ){ + return pRight; + }else if( pRight==0 ){ + return pLeft; + }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } +} + +/* +** Construct a new expression node for a function with multiple +** arguments. +*/ +Expr *sqlite3ExprFunction( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Argument list */ + Token *pToken, /* Name of the function */ + int eDistinct /* SF_Distinct or SF_ALL or 0 */ +){ + Expr *pNew; + sqlite3 *db = pParse->db; + assert( pToken ); + pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ + return 0; + } + if( pList && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); + } + pNew->x.pList = pList; + ExprSetProperty(pNew, EP_HasFunc); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + sqlite3ExprSetHeightAndFlags(pParse, pNew); + if( eDistinct==SF_Distinct ) ExprSetProperty(pNew, EP_Distinct); + return pNew; +} + +/* +** Check to see if a function is usable according to current access +** rules: +** +** SQLITE_FUNC_DIRECT - Only usable from top-level SQL +** +** SQLITE_FUNC_UNSAFE - Usable if TRUSTED_SCHEMA or from +** top-level SQL +** +** If the function is not usable, create an error. +*/ +void sqlite3ExprFunctionUsable( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The function invocation */ + FuncDef *pDef /* The function being invoked */ +){ + assert( !IN_RENAME_OBJECT ); + assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 + || (pParse->db->flags & SQLITE_TrustedSchema)==0 + ){ + /* Functions prohibited in triggers and views if: + ** (1) tagged with SQLITE_DIRECTONLY + ** (2) not tagged with SQLITE_INNOCUOUS (which means it + ** is tagged with SQLITE_FUNC_UNSAFE) and + ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning + ** that the schema is possibly tainted). + */ + sqlite3ErrorMsg(pParse, "unsafe use of %s()", pDef->zName); + } + } +} + +/* +** Assign a variable number to an expression that encodes a wildcard +** in the original SQL statement. +** +** Wildcards consisting of a single "?" are assigned the next sequential +** variable number. +** +** Wildcards of the form "?nnn" are assigned the number "nnn". We make +** sure "nnn" is not too big to avoid a denial of service attack when +** the SQL statement comes from an external source. +** +** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number +** as the previous instance of the same wildcard. Or if this is the first +** instance of the wildcard, the next sequential variable number is +** assigned. +*/ +void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ + sqlite3 *db = pParse->db; + const char *z; + ynVar x; + + if( pExpr==0 ) return; + assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); + z = pExpr->u.zToken; + assert( z!=0 ); + assert( z[0]!=0 ); + assert( n==(u32)sqlite3Strlen30(z) ); + if( z[1]==0 ){ + /* Wildcard of the form "?". Assign the next variable number */ + assert( z[0]=='?' ); + x = (ynVar)(++pParse->nVar); + }else{ + int doAdd = 0; + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + return; + } + x = (ynVar)i; + if( x>pParse->nVar ){ + pParse->nVar = (int)x; + doAdd = 1; + }else if( sqlite3VListNumToName(pParse->pVList, x)==0 ){ + doAdd = 1; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + x = (ynVar)sqlite3VListNameToNum(pParse->pVList, z, n); + if( x==0 ){ + x = (ynVar)(++pParse->nVar); + doAdd = 1; + } + } + if( doAdd ){ + pParse->pVList = sqlite3VListAdd(db, pParse->pVList, z, n, x); + } + } + pExpr->iColumn = x; + if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "too many SQL variables"); + } +} + +/* +** Recursively delete an expression tree. +*/ +static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ + assert( p!=0 ); + /* Sanity check: Assert that the IntValue is non-negative if it exists */ + assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); + + assert( !ExprHasProperty(p, EP_WinFunc) || p->y.pWin!=0 || db->mallocFailed ); + assert( p->op!=TK_FUNCTION || ExprHasProperty(p, EP_TokenOnly|EP_Reduced) + || p->y.pWin==0 || ExprHasProperty(p, EP_WinFunc) ); +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( p->x.pSelect==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); + if( p->pRight ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); + sqlite3ExprDeleteNN(db, p->pRight); + }else if( ExprHasProperty(p, EP_xIsSelect) ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); + sqlite3SelectDelete(db, p->x.pSelect); + }else{ + sqlite3ExprListDelete(db, p->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + sqlite3WindowDelete(db, p->y.pWin); + } +#endif + } + } + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); + if( !ExprHasProperty(p, EP_Static) ){ + sqlite3DbFreeNN(db, p); + } +} +void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p ) sqlite3ExprDeleteNN(db, p); +} + + +/* +** Arrange to cause pExpr to be deleted when the pParse is deleted. +** This is similar to sqlite3ExprDelete() except that the delete is +** deferred untilthe pParse is deleted. +** +** The pExpr might be deleted immediately on an OOM error. +** +** The deferred delete is (currently) implemented by adding the +** pExpr to the pParse->pConstExpr list with a register number of 0. +*/ +void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + pParse->pConstExpr = + sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); +} + +/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the +** expression. +*/ +void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ + if( p ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, p); + } + sqlite3ExprDeleteNN(pParse->db, p); + } +} + +/* +** Return the number of bytes allocated for the expression structure +** passed as the first argument. This is always one of EXPR_FULLSIZE, +** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. +*/ +static int exprStructSize(Expr *p){ + if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; + if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; + return EXPR_FULLSIZE; +} + +/* +** The dupedExpr*Size() routines each return the number of bytes required +** to store a copy of an expression or expression tree. They differ in +** how much of the tree is measured. +** +** dupedExprStructSize() Size of only the Expr structure +** dupedExprNodeSize() Size of Expr + space for token +** dupedExprSize() Expr + token + subtree components +** +*************************************************************************** +** +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and +** (2) the EP_xxx flags that indicate what the structure size should be. +** The return values is always one of: +** +** EXPR_FULLSIZE +** EXPR_REDUCEDSIZE | EP_Reduced +** EXPR_TOKENONLYSIZE | EP_TokenOnly +** +** The size of the structure can be found by masking the return value +** of this routine with 0xfff. The flags can be found by masking the +** return value with EP_Reduced|EP_TokenOnly. +** +** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size +** (unreduced) Expr objects as they or originally constructed by the parser. +** During expression analysis, extra information is computed and moved into +** later parts of the Expr object and that extra information might get chopped +** off if the expression is reduced. Note also that it does not work to +** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** to reduce a pristine expression tree from the parser. The implementation +** of dupedExprStructSize() contain multiple assert() statements that attempt +** to enforce this constraint. +*/ +static int dupedExprStructSize(Expr *p, int flags){ + int nSize; + assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ + assert( EXPR_FULLSIZE<=0xfff ); + assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); + if( 0==flags || p->op==TK_SELECT_COLUMN +#ifndef SQLITE_OMIT_WINDOWFUNC + || ExprHasProperty(p, EP_WinFunc) +#endif + ){ + nSize = EXPR_FULLSIZE; + }else{ + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasProperty(p, EP_FromJoin) ); + assert( !ExprHasProperty(p, EP_MemToken) ); + assert( !ExprHasVVAProperty(p, EP_NoReduce) ); + if( p->pLeft || p->x.pList ){ + nSize = EXPR_REDUCEDSIZE | EP_Reduced; + }else{ + assert( p->pRight==0 ); + nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; + } + } + return nSize; +} + +/* +** This function returns the space in bytes required to store the copy +** of the Expr structure and a copy of the Expr.u.zToken string (if that +** string is defined.) +*/ +static int dupedExprNodeSize(Expr *p, int flags){ + int nByte = dupedExprStructSize(p, flags) & 0xfff; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nByte += sqlite3Strlen30NN(p->u.zToken)+1; + } + return ROUND8(nByte); +} + +/* +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. The second argument is a +** mask containing EXPRDUP_XXX flags. +** +** The value returned includes space to create a copy of the Expr struct +** itself and the buffer referred to by Expr.u.zToken, if any. +** +** If the EXPRDUP_REDUCE flag is set, then the return value includes +** space to duplicate all Expr nodes in the tree formed by Expr.pLeft +** and Expr.pRight variables (but not for any structures pointed to or +** descended from the Expr.x.pList or Expr.x.pSelect variables). +*/ +static int dupedExprSize(Expr *p, int flags){ + int nByte = 0; + if( p ){ + nByte = dupedExprNodeSize(p, flags); + if( flags&EXPRDUP_REDUCE ){ + nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); + } + } + return nByte; +} + +/* +** This function is similar to sqlite3ExprDup(), except that if pzBuffer +** is not NULL then *pzBuffer is assumed to point to a buffer large enough +** to store the copy of expression p, the copies of p->u.zToken +** (if applicable), and the copies of the p->pLeft and p->pRight expressions, +** if any. Before returning, *pzBuffer is set to the first byte past the +** portion of the buffer copied into by this function. +*/ +static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ + Expr *pNew; /* Value to return */ + u8 *zAlloc; /* Memory space from which to build Expr object */ + u32 staticFlag; /* EP_Static if space not obtained from malloc */ + + assert( db!=0 ); + assert( p ); + assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + staticFlag = 0; + } + pNew = (Expr *)zAlloc; + + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, dupFlags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } + if( dupFlags ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + u32 nSize = (u32)exprStructSize(p); + memcpy(zAlloc, p, nSize); + if( nSizeflags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; + ExprClearVVAProperties(pNew); + if( dupFlags ){ + ExprSetVVAProperty(pNew, EP_Immutable); + } + + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } + + if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); + }else{ + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + } + } + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ + zAlloc += dupedExprNodeSize(p, dupFlags); + if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); + assert( ExprHasProperty(pNew, EP_WinFunc) ); + } +#endif /* SQLITE_OMIT_WINDOWFUNC */ + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + if( pNew->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->iColumn==0 || p->pRight==0 ); + assert( p->pRight==0 || p->pRight==p->pLeft ); + }else{ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + } + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } + } + } + return pNew; +} + +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +#ifndef SQLITE_OMIT_CTE +static With *withDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} +#else +# define withDup(x,y) 0 +#endif + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_WinFunc) ){ + Select *pSelect = pWalker->u.pSelect; + Window *pWin = pExpr->y.pWin; + assert( pWin ); + assert( IsWindowFunc(pExpr) ); + assert( pWin->ppThis==0 ); + sqlite3WindowLink(pSelect, pWin); + } + return WRC_Continue; +} +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; + w.pParse = 0; + w.u.pSelect = p; + sqlite3WalkSelect(&w, p); +} +#endif + + +/* +** The following group of routines make deep copies of expressions, +** expression lists, ID lists, and select statements. The copies can +** be deleted (by being passed to their respective ...Delete() routines) +** without effecting the originals. +** +** The expression list, ID, and source lists return by sqlite3ExprListDup(), +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** by subsequent calls to sqlite*ListAppend() routines. +** +** Any tables that the SrcList might point to are not duplicated. +** +** The flags parameter contains a combination of the EXPRDUP_XXX flags. +** If the EXPRDUP_REDUCE flag is set, then the structure returned is a +** truncated version of the usual Expr structure that will be stored as +** part of the in-memory representation of the database schema. +*/ +Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ + assert( flags==0 || flags==EXPRDUP_REDUCE ); + return p ? exprDup(db, p, flags, 0) : 0; +} +ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ + ExprList *pNew; + struct ExprList_item *pItem, *pOldItem; + int i; + Expr *pPriorSelectCol = 0; + assert( db!=0 ); + if( p==0 ) return 0; + pNew = sqlite3DbMallocRawNN(db, sqlite3DbMallocSize(db, p)); + if( pNew==0 ) return 0; + pNew->nExpr = p->nExpr; + pItem = pNew->a; + pOldItem = p->a; + for(i=0; inExpr; i++, pItem++, pOldItem++){ + Expr *pOldExpr = pOldItem->pExpr; + Expr *pNewExpr; + pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); + if( pOldExpr + && pOldExpr->op==TK_SELECT_COLUMN + && (pNewExpr = pItem->pExpr)!=0 + ){ + assert( pNewExpr->iColumn==0 || i>0 ); + if( pNewExpr->iColumn==0 ){ + assert( pOldExpr->pLeft==pOldExpr->pRight ); + pPriorSelectCol = pNewExpr->pLeft = pNewExpr->pRight; + }else{ + assert( i>0 ); + assert( pItem[-1].pExpr!=0 ); + assert( pNewExpr->iColumn==pItem[-1].pExpr->iColumn+1 ); + assert( pPriorSelectCol==pItem[-1].pExpr->pLeft ); + pNewExpr->pLeft = pPriorSelectCol; + } + } + pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); + pItem->sortFlags = pOldItem->sortFlags; + pItem->eEName = pOldItem->eEName; + pItem->done = 0; + pItem->bNulls = pOldItem->bNulls; + pItem->bSorterRef = pOldItem->bSorterRef; + pItem->u = pOldItem->u; + } + return pNew; +} + +/* +** If cursors, triggers, views and subqueries are all omitted from +** the build, then none of the following routines, except for +** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes +** called with a NULL argument. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ + || !defined(SQLITE_OMIT_SUBQUERY) +SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ + SrcList *pNew; + int i; + int nByte; + assert( db!=0 ); + if( p==0 ) return 0; + nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); + pNew = sqlite3DbMallocRawNN(db, nByte ); + if( pNew==0 ) return 0; + pNew->nSrc = pNew->nAlloc = p->nSrc; + for(i=0; inSrc; i++){ + SrcItem *pNewItem = &pNew->a[i]; + SrcItem *pOldItem = &p->a[i]; + Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; + pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); + pNewItem->fg = pOldItem->fg; + pNewItem->iCursor = pOldItem->iCursor; + pNewItem->addrFillSub = pOldItem->addrFillSub; + pNewItem->regReturn = pOldItem->regReturn; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->u2 = pOldItem->u2; + if( pNewItem->fg.isCte ){ + pNewItem->u2.pCteUse->nUse++; + } + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } + pTab = pNewItem->pTab = pOldItem->pTab; + if( pTab ){ + pTab->nTabRef++; + } + pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); + pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); + pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + pNewItem->colUsed = pOldItem->colUsed; + } + return pNew; +} +IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ + IdList *pNew; + int i; + assert( db!=0 ); + if( p==0 ) return 0; + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nId = p->nId; + pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); + if( pNew->a==0 ){ + sqlite3DbFreeNN(db, pNew); + return 0; + } + /* Note that because the size of the allocation for p->a[] is not + ** necessarily a power of two, sqlite3IdListAppend() may not be called + ** on the duplicate created by this function. */ + for(i=0; inId; i++){ + struct IdList_item *pNewItem = &pNew->a[i]; + struct IdList_item *pOldItem = &p->a[i]; + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->idx = pOldItem->idx; + } + return pNew; +} +Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + Select *p; + + assert( db!=0 ); + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = withDup(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin && db->mallocFailed==0 ) gatherSelectWindows(pNew); +#endif + pNew->selId = p->selId; + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + + return pRet; +} +#else +Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + assert( p==0 ); + return 0; +} +#endif + + +/* +** Add a new element to the end of an expression list. If pList is +** initially NULL, then create a new expression list. +** +** The pList argument must be either NULL or a pointer to an ExprList +** obtained from a prior call to sqlite3ExprListAppend(). This routine +** may not be used with an ExprList obtained from sqlite3ExprListDup(). +** Reason: This routine assumes that the number of slots in pList->a[] +** is a power of two. That is true for sqlite3ExprListAppend() returns +** but is not necessarily true from the return value of sqlite3ExprListDup(). +** +** If a memory allocation error occurs, the entire list is freed and +** NULL is returned. If non-NULL is returned, then it is guaranteed +** that the new entry was successfully appended. +*/ +ExprList *sqlite3ExprListAppend( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + sqlite3 *db = pParse->db; + assert( db!=0 ); + if( pList==0 ){ + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); + if( pList==0 ){ + goto no_mem; + } + pList->nExpr = 0; + }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ + ExprList *pNew; + pNew = sqlite3DbRealloc(db, pList, + sizeof(*pList)+(2*(sqlite3_int64)pList->nExpr-1)*sizeof(pList->a[0])); + if( pNew==0 ){ + goto no_mem; + } + pList = pNew; + } + pItem = &pList->a[pList->nExpr++]; + assert( offsetof(struct ExprList_item,zEName)==sizeof(pItem->pExpr) ); + assert( offsetof(struct ExprList_item,pExpr)==0 ); + memset(&pItem->zEName,0,sizeof(*pItem)-offsetof(struct ExprList_item,zEName)); + pItem->pExpr = pExpr; + return pList; + +no_mem: + /* Avoid leaking memory if malloc has failed. */ + sqlite3ExprDelete(db, pExpr); + sqlite3ExprListDelete(db, pList); + return 0; +} + +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the RHS, append +** TK_SELECT_COLUMN expressions. +*/ +ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + + /* If the RHS is a vector, then we can immediately check to see that + ** the size of the RHS and LHS match. But if the RHS is a SELECT, + ** wildcards ("*") in the result set of the SELECT must be expanded before + ** we can do the size check, so defer the size check until code generation. + */ + if( pExpr->op!=TK_SELECT && pColumns->nId!=(n=sqlite3ExprVectorSize(pExpr)) ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + + for(i=0; inId; i++){ + Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i); + assert( pSubExpr!=0 || db->mallocFailed ); + assert( pSubExpr==0 || pSubExpr->iTable==0 ); + if( pSubExpr==0 ) continue; + pSubExpr->iTable = pColumns->nId; + pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); + if( pList ){ + assert( pList->nExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zEName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + + if( !db->mallocFailed && pExpr->op==TK_SELECT && ALWAYS(pList!=0) ){ + Expr *pFirst = pList->a[iFirst].pExpr; + assert( pFirst!=0 ); + assert( pFirst->op==TK_SELECT_COLUMN ); + + /* Store the SELECT statement in pRight so it will be deleted when + ** sqlite3ExprListDelete() is called */ + pFirst->pRight = pExpr; + pExpr = 0; + + /* Remember the size of the LHS in iTable so that we can check that + ** the RHS and LHS sizes match during code generation. */ + pFirst->iTable = pColumns->nId; + } + +vector_append_error: + sqlite3ExprUnmapAndDelete(pParse, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; +} + +/* +** Set the sort order for the last element on the given ExprList. +*/ +void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ + struct ExprList_item *pItem; + if( p==0 ) return; + assert( p->nExpr>0 ); + + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC + ); + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC + ); + + pItem = &p->a[p->nExpr-1]; + assert( pItem->bNulls==0 ); + if( iSortOrder==SQLITE_SO_UNDEFINED ){ + iSortOrder = SQLITE_SO_ASC; + } + pItem->sortFlags = (u8)iSortOrder; + + if( eNulls!=SQLITE_SO_UNDEFINED ){ + pItem->bNulls = 1; + if( iSortOrder!=eNulls ){ + pItem->sortFlags |= KEYINFO_ORDER_BIGNULL; + } + } +} + +/* +** Set the ExprList.a[].zEName element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pName should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +void sqlite3ExprListSetName( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + Token *pName, /* Name to be added */ + int dequote /* True to cause the name to be dequoted */ +){ + assert( pList!=0 || pParse->db->mallocFailed!=0 ); + assert( pParse->eParseMode!=PARSE_MODE_UNMAP || dequote==0 ); + if( pList ){ + struct ExprList_item *pItem; + assert( pList->nExpr>0 ); + pItem = &pList->a[pList->nExpr-1]; + assert( pItem->zEName==0 ); + assert( pItem->eEName==ENAME_NAME ); + pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote ){ + /* If dequote==0, then pName->z does not point to part of a DDL + ** statement handled by the parser. And so no token need be added + ** to the token-map. */ + sqlite3Dequote(pItem->zEName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)pItem->zEName, pName); + } + } + } +} + +/* +** Set the ExprList.a[].zSpan element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pSpan should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +void sqlite3ExprListSetSpan( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + const char *zStart, /* Start of the span */ + const char *zEnd /* End of the span */ +){ + sqlite3 *db = pParse->db; + assert( pList!=0 || db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; + assert( pList->nExpr>0 ); + if( pItem->zEName==0 ){ + pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd); + pItem->eEName = ENAME_SPAN; + } + } +} + +/* +** If the expression list pEList contains more than iLimit elements, +** leave an error message in pParse. +*/ +void sqlite3ExprListCheckLength( + Parse *pParse, + ExprList *pEList, + const char *zObject +){ + int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + testcase( pEList && pEList->nExpr==mx ); + testcase( pEList && pEList->nExpr==mx+1 ); + if( pEList && pEList->nExpr>mx ){ + sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); + } +} + +/* +** Delete an entire expression list. +*/ +static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ + int i = pList->nExpr; + struct ExprList_item *pItem = pList->a; + assert( pList->nExpr>0 ); + do{ + sqlite3ExprDelete(db, pItem->pExpr); + sqlite3DbFree(db, pItem->zEName); + pItem++; + }while( --i>0 ); + sqlite3DbFreeNN(db, pList); +} +void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + if( pList ) exprListDeleteNN(db, pList); +} + +/* +** Return the bitwise-OR of all Expr.flags fields in the given +** ExprList. +*/ +u32 sqlite3ExprListFlags(const ExprList *pList){ + int i; + u32 m = 0; + assert( pList!=0 ); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + m |= pExpr->flags; + } + return m; +} + +/* +** This is a SELECT-node callback for the expression walker that +** always "fails". By "fail" in this case, we mean set +** pWalker->eCode to zero and abort. +** +** This callback is used by multiple expression walkers. +*/ +int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->eCode = 0; + return WRC_Abort; +} + +/* +** Check the input string to see if it is "true" or "false" (in any case). +** +** If the string is.... Return +** "true" EP_IsTrue +** "false" EP_IsFalse +** anything else 0 +*/ +u32 sqlite3IsTrueOrFalse(const char *zIn){ + if( sqlite3StrICmp(zIn, "true")==0 ) return EP_IsTrue; + if( sqlite3StrICmp(zIn, "false")==0 ) return EP_IsFalse; + return 0; +} + + +/* +** If the input expression is an ID with the name "true" or "false" +** then convert it into an TK_TRUEFALSE term. Return non-zero if +** the conversion happened, and zero if the expression is unaltered. +*/ +int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + u32 v; + assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); + if( !ExprHasProperty(pExpr, EP_Quoted) + && (v = sqlite3IsTrueOrFalse(pExpr->u.zToken))!=0 + ){ + pExpr->op = TK_TRUEFALSE; + ExprSetProperty(pExpr, v); + return 1; + } + return 0; +} + +/* +** The argument must be a TK_TRUEFALSE Expr node. Return 1 if it is TRUE +** and 0 if it is FALSE. +*/ +int sqlite3ExprTruthValue(const Expr *pExpr){ + pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); + assert( pExpr->op==TK_TRUEFALSE ); + assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 + || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); + return pExpr->u.zToken[4]==0; +} + +/* +** If pExpr is an AND or OR expression, try to simplify it by eliminating +** terms that are always true or false. Return the simplified expression. +** Or return the original expression if no simplification is possible. +** +** Examples: +** +** (x<10) AND true => (x<10) +** (x<10) AND false => false +** (x<10) AND (y=22 OR false) => (x<10) AND (y=22) +** (x<10) AND (y=22 OR true) => (x<10) +** (y=22) OR true => true +*/ +Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ + assert( pExpr!=0 ); + if( pExpr->op==TK_AND || pExpr->op==TK_OR ){ + Expr *pRight = sqlite3ExprSimplifiedAndOr(pExpr->pRight); + Expr *pLeft = sqlite3ExprSimplifiedAndOr(pExpr->pLeft); + if( ExprAlwaysTrue(pLeft) || ExprAlwaysFalse(pRight) ){ + pExpr = pExpr->op==TK_AND ? pRight : pLeft; + }else if( ExprAlwaysTrue(pRight) || ExprAlwaysFalse(pLeft) ){ + pExpr = pExpr->op==TK_AND ? pLeft : pRight; + } + } + return pExpr; +} + + +/* +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. +** +** These callback routines are used to implement the following: +** +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprIsTableConstant() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. +** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating DEFAULT +** expressions in a CREATE TABLE statement. The Walker.eCode value is 5 +** when parsing an existing schema out of the sqlite_schema table and 4 +** when processing a new CREATE TABLE statement. A bound parameter raises +** an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_schema tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. +*/ +static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression + ** from being considered constant. */ + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + switch( pExpr->op ){ + /* Consider functions to be constant if all their arguments are constant + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ + case TK_FUNCTION: + if( (pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc)) + && !ExprHasProperty(pExpr, EP_WinFunc) + ){ + if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } + case TK_ID: + /* Convert "true" or "false" in a DEFAULT clause into the + ** appropriate TK_TRUEFALSE operator */ + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through + case TK_COLUMN: + case TK_AGG_FUNCTION: + case TK_AGG_COLUMN: + testcase( pExpr->op==TK_ID ); + testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + testcase( pExpr->op==TK_AGG_COLUMN ); + if( ExprHasProperty(pExpr, EP_FixedCol) && pWalker->eCode!=2 ){ + return WRC_Continue; + } + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + } + /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: + case TK_REGISTER: + case TK_DOT: + testcase( pExpr->op==TK_REGISTER ); + testcase( pExpr->op==TK_IF_NULL_ROW ); + testcase( pExpr->op==TK_DOT ); + pWalker->eCode = 0; + return WRC_Abort; + case TK_VARIABLE: + if( pWalker->eCode==5 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_schema table */ + pExpr->op = TK_NULL; + }else if( pWalker->eCode==4 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->eCode = 0; + return WRC_Abort; + } + /* no break */ deliberate_fall_through + default: + testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */ + testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */ + return WRC_Continue; + } +} +static int exprIsConst(Expr *p, int initFlag, int iCur){ + Walker w; + w.eCode = initFlag; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant +** and 0 if it involves variables or function calls. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +int sqlite3ExprIsConstant(Expr *p){ + return exprIsConst(p, 1, 0); +} + +/* +** Walk an expression tree. Return non-zero if +** +** (1) the expression is constant, and +** (2) the expression does originate in the ON or USING clause +** of a LEFT JOIN, and +** (3) the expression does not contain any EP_FixedCol TK_COLUMN +** operands created by the constant propagation optimization. +** +** When this routine returns true, it indicates that the expression +** can be added to the pParse->pConstExpr list and evaluated once when +** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). +*/ +int sqlite3ExprIsConstantNotJoin(Expr *p){ + return exprIsConst(p, 2, 0); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + + +/* +** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy(). +*/ +static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ + ExprList *pGroupBy = pWalker->u.pGroupBy; + int i; + + /* Check if pExpr is identical to any GROUP BY term. If so, consider + ** it constant. */ + for(i=0; inExpr; i++){ + Expr *p = pGroupBy->a[i].pExpr; + if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){ + CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p); + if( sqlite3IsBinary(pColl) ){ + return WRC_Prune; + } + } + } + + /* Check if pExpr is a sub-select. If so, consider it variable. */ + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + return exprNodeIsConstant(pWalker, pExpr); +} + +/* +** Walk the expression tree passed as the first argument. Return non-zero +** if the expression consists entirely of constants or copies of terms +** in pGroupBy that sort with the BINARY collation sequence. +** +** This routine is used to determine if a term of the HAVING clause can +** be promoted into the WHERE clause. In order for such a promotion to work, +** the value of the HAVING clause term must be the same for all members of +** a "group". The requirement that the GROUP BY term must be BINARY +** assumes that no other collating sequence will have a finer-grained +** grouping than binary. In other words (A=B COLLATE binary) implies +** A=B in every other collating sequence. The requirement that the +** GROUP BY be BINARY is stricter than necessary. It would also work +** to promote HAVING clauses that use the same alternative collating +** sequence as the GROUP BY term, but that is much harder to check, +** alternative collating sequences are uncommon, and this is only an +** optimization, so we take the easy way out and simply require the +** GROUP BY to use the BINARY collating sequence. +*/ +int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ + Walker w; + w.eCode = 1; + w.xExprCallback = exprNodeIsConstantOrGroupBy; + w.xSelectCallback = 0; + w.u.pGroupBy = pGroupBy; + w.pParse = pParse; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Walk an expression tree for the DEFAULT field of a column definition +** in a CREATE TABLE statement. Return non-zero if the expression is +** acceptable for use as a DEFAULT. That is to say, return non-zero if +** the expression is constant or a function call with constant arguments. +** Return and 0 if there are any variables. +** +** isInit is true when parsing from sqlite_schema. isInit is false when +** processing a new CREATE TABLE statement. When isInit is true, parameters +** (such as ? or $abc) in the expression are converted into NULL. When +** isInit is false, parameters raise an error. Parameters should not be +** allowed in a CREATE TABLE statement, but some legacy versions of SQLite +** allowed it, so we need to support it when reading sqlite_schema for +** backwards compatibility. +** +** If isInit is true, set EP_FromDDL on every TK_FUNCTION node. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 4+isInit, 0); +} + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Walk an expression tree. Return 1 if the expression contains a +** subquery of some kind. Return 0 if there are no subqueries. +*/ +int sqlite3ExprContainsSubquery(Expr *p){ + Walker w; + w.eCode = 1; + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + sqlite3WalkExpr(&w, p); + return w.eCode==0; +} +#endif + +/* +** If the expression p codes a constant integer that is small enough +** to fit in a 32-bit integer, return 1 and put the value of the integer +** in *pValue. If the expression is not an integer or if it is too big +** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +*/ +int sqlite3ExprIsInteger(Expr *p, int *pValue){ + int rc = 0; + if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ + + /* If an expression is an integer literal that fits in a signed 32-bit + ** integer, then the EP_IntValue flag will have already been set */ + assert( p->op!=TK_INTEGER || (p->flags & EP_IntValue)!=0 + || sqlite3GetInt32(p->u.zToken, &rc)==0 ); + + if( p->flags & EP_IntValue ){ + *pValue = p->u.iValue; + return 1; + } + switch( p->op ){ + case TK_UPLUS: { + rc = sqlite3ExprIsInteger(p->pLeft, pValue); + break; + } + case TK_UMINUS: { + int v; + if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + assert( v!=(-2147483647-1) ); + *pValue = -v; + rc = 1; + } + break; + } + default: break; + } + return rc; +} + +/* +** Return FALSE if there is no chance that the expression can be NULL. +** +** If the expression might be NULL or if the expression is too complex +** to tell return TRUE. +** +** This routine is used as an optimization, to skip OP_IsNull opcodes +** when we know that a value cannot be NULL. Hence, a false positive +** (returning TRUE when in fact the expression can never be NULL) might +** be a small performance hit but is otherwise harmless. On the other +** hand, a false negative (returning FALSE when the result could be NULL) +** will likely result in an incorrect answer. So when in doubt, return +** TRUE. +*/ +int sqlite3ExprCanBeNull(const Expr *p){ + u8 op; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + return 0; + case TK_COLUMN: + return ExprHasProperty(p, EP_CanBeNull) || + p->y.pTab==0 || /* Reference to column of index on expression */ + (p->iColumn>=0 + && ALWAYS(p->y.pTab->aCol!=0) /* Defense against OOM problems */ + && p->y.pTab->aCol[p->iColumn].notNull==0); + default: + return 1; + } +} + +/* +** Return TRUE if the given expression is a constant which would be +** unchanged by OP_Affinity with the affinity given in the second +** argument. +** +** This routine is used to determine if the OP_Affinity operation +** can be omitted. When in doubt return FALSE. A false negative +** is harmless. A false positive, however, can result in the wrong +** answer. +*/ +int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ + u8 op; + int unaryMinus = 0; + if( aff==SQLITE_AFF_BLOB ) return 1; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + if( p->op==TK_UMINUS ) unaryMinus = 1; + p = p->pLeft; + } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: { + return aff>=SQLITE_AFF_NUMERIC; + } + case TK_FLOAT: { + return aff>=SQLITE_AFF_NUMERIC; + } + case TK_STRING: { + return !unaryMinus && aff==SQLITE_AFF_TEXT; + } + case TK_BLOB: { + return !unaryMinus; + } + case TK_COLUMN: { + assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ + return aff>=SQLITE_AFF_NUMERIC && p->iColumn<0; + } + default: { + return 0; + } + } +} + +/* +** Return TRUE if the given string is a row-id column name. +*/ +int sqlite3IsRowid(const char *z){ + if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; + if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; + if( sqlite3StrICmp(z, "OID")==0 ) return 1; + return 0; +} + +/* +** pX is the RHS of an IN operator. If pX is a SELECT statement +** that can be simplified to a direct table access, then return +** a pointer to the SELECT statement. If pX is not a SELECT statement, +** or if the SELECT statement needs to be manifested into a transient +** table, then return NULL. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +static Select *isCandidateForInOpt(Expr *pX){ + Select *p; + SrcList *pSrc; + ExprList *pEList; + Table *pTab; + int i; + if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ + p = pX->x.pSelect; + if( p->pPrior ) return 0; /* Not a compound SELECT */ + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + return 0; /* No DISTINCT keyword and no aggregate functions */ + } + assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ + if( p->pLimit ) return 0; /* Has no LIMIT clause */ + if( p->pWhere ) return 0; /* Has no WHERE clause */ + pSrc = p->pSrc; + assert( pSrc!=0 ); + if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ + if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ + pTab = pSrc->a[0].pTab; + assert( pTab!=0 ); + assert( pTab->pSelect==0 ); /* FROM clause is not a view */ + if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ + pEList = p->pEList; + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } + return p; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code that checks the left-most column of index table iCur to see if +** it contains any NULL entries. Cause the register at regHasNull to be set +** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull +** to be set to NULL if iCur contains one or more NULL values. +*/ +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ + int addr1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "first_entry_in(%d)", iCur)); + sqlite3VdbeJumpHere(v, addr1); +} +#endif + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** The argument is an IN operator with a list (not a subquery) on the +** right-hand side. Return TRUE if that list is constant. +*/ +static int sqlite3InRhsIsConstant(Expr *pIn){ + Expr *pLHS; + int res; + assert( !ExprHasProperty(pIn, EP_xIsSelect) ); + pLHS = pIn->pLeft; + pIn->pLeft = 0; + res = sqlite3ExprIsConstant(pIn); + pIn->pLeft = pLHS; + return res; +} +#endif + +/* +** This function is used by the implementation of the IN (...) operator. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that is the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. +** +** The returned value of this function indicates the b-tree type, as follows: +** +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be +** implemented as a sequence of comparisons. +** +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: +** +** SELECT , ... FROM +** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephemeral table instead of an +** existing table. +** +** The inFlags parameter must contain, at a minimum, one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a fast +** membership test. When the IN_INDEX_LOOP bit is set, the IN index will +** be used to loop over all values of the RHS of the IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table will be created unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. +** +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. +** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** +** When the b-tree is being used for membership tests, the calling function +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prRhsHasNull is not a NULL pointer and +** if there is any chance that the (...) might contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. +** +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The IN expression */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap, /* Mapping from Index fields to RHS fields */ + int *piTab /* OUT: index to use */ +){ + Select *p; /* SELECT to the right of IN operator */ + int eType = 0; /* Type of RHS table. IN_INDEX_* */ + int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int mustBeUnique; /* True if RHS must be unique */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + + assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } + + /* Check to see if an existing table or index can be used to + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. */ + if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table
. */ + int iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; + + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + + /* Code an OP_Transaction and OP_TableLock for
. */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbtnum, 0, pTab->zName); + + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); + VdbeCoverage(v); + + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; + ExplainQueryPlan((pParse, 0, + "USING ROWID SEARCH ON TABLE %s FOR IN-OPERATOR",pTab->zName)); + sqlite3VdbeJumpHere(v, iAddr); + }else{ + Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; + + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); + } + } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnpPartIdxWhere!=0 ) continue; + /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute + ** BITMASK(nExpr) without overflowing */ + testcase( pIdx->nColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + ExplainQueryPlan((pParse, 0, + "USING INDEX %s FOR IN-OPERATOR",pIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ + + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not constant or has two or fewer terms, + ** then it is not worth creating an ephemeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + if( eType==0 ){ + /* Could not find an existing table or index to use as the RHS b-tree. + ** We will have to generate an ephemeral table to do the job. + */ + u32 savedNQueryLoop = pParse->nQueryLoop; + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( inFlags & IN_INDEX_LOOP ){ + pParse->nQueryLoop = 0; + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; + } + assert( pX->op==TK_IN ); + sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( rMayHaveNull ){ + sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); + } + pParse->nQueryLoop = savedNQueryLoop; + } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + if( pParse->nErr==0 ){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); + } +} +#endif + +/* +** Expression pExpr is a vector that has been used in a context where +** it is not permitted. If pExpr is a sub-select vector, this routine +** loads the Parse object with a message of the form: +** +** "sub-select returns N columns - expected 1" +** +** Or, if it is a regular scalar vector: +** +** "row value misused" +*/ +void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->flags & EP_xIsSelect ){ + sqlite3SubselectError(pParse, pExpr->x.pSelect->pEList->nExpr, 1); + }else +#endif + { + sqlite3ErrorMsg(pParse, "row value misused"); + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code that will construct an ephemeral table containing all terms +** in the RHS of an IN operator. The IN operator can be in either of two +** forms: +** +** x IN (4,5,11) -- IN operator with list on right-hand side +** x IN (SELECT a FROM b) -- IN operator with subquery on the right +** +** The pExpr parameter is the IN operator. The cursor number for the +** constructed ephermeral table is returned. The first time the ephemeral +** table is computed, the cursor number is also stored in pExpr->iTable, +** however the cursor number returned might not be the same, as it might +** have been duplicated using OP_OpenDup. +** +** If the LHS expression ("x" in the examples) is a column value, or +** the SELECT statement returns a column value, then the affinity of that +** column is used to build the index keys. If both 'x' and the +** SELECT... statement are columns, then numeric affinity is used +** if either column has NUMERIC or INTEGER affinity. If neither +** 'x' nor the SELECT... statement are columns, then numeric affinity +** is used. +*/ +void sqlite3CodeRhsOfIN( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The IN operator */ + int iTab /* Use this cursor number */ +){ + int addrOnce = 0; /* Address of the OP_Once instruction at top */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft; /* the LHS of the IN operator */ + KeyInfo *pKeyInfo = 0; /* Key information */ + int nVal; /* Size of vector pLeft */ + Vdbe *v; /* The prepared statement under construction */ + + v = pParse->pVdbe; + assert( v!=0 ); + + /* The evaluation of the IN must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can compute the RHS just once + ** and reuse it many names. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ + /* Reuse of the RHS is allowed */ + /* If this routine has already been coded, but the previous code + ** might not have been invoked yet, so invoke it now as a subroutine. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", + pExpr->x.pSelect->selId)); + } + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); + sqlite3VdbeJumpHere(v, addrOnce); + return; + } + + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* Check to see if this is a vector IN operator */ + pLeft = pExpr->pLeft; + nVal = sqlite3ExprVectorSize(pLeft); + + /* Construct the ephemeral table that will contain the content of + ** RHS of the IN operator. + */ + pExpr->iTable = iTab; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, nVal); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + VdbeComment((v, "Result of SELECT %u", pExpr->x.pSelect->selId)); + }else{ + VdbeComment((v, "RHS of IN operator")); + } +#endif + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + Select *pSelect = pExpr->x.pSelect; + ExprList *pEList = pSelect->pEList; + + ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSelect->selId + )); + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + SelectDest dest; + int i; + sqlite3SelectDestInit(&dest, SRT_Set, iTab); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3DbFree(pParse->db, dest.zAffSdst); + sqlite3KeyInfoUnref(pKeyInfo); + return; + } + sqlite3DbFree(pParse->db, dest.zAffSdst); + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } + } + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + char affinity; /* Affinity of the LHS of the IN */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2; + affinity = sqlite3ExprAffinity(pLeft); + if( affinity<=SQLITE_AFF_NONE ){ + affinity = SQLITE_AFF_BLOB; + }else if( affinity==SQLITE_AFF_REAL ){ + affinity = SQLITE_AFF_NUMERIC; + } + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. + */ + if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, addrOnce); + ExprClearProperty(pExpr, EP_Subrtn); + addrOnce = 0; + } + + /* Evaluate the expression and insert it into the temp table */ + sqlite3ExprCode(pParse, pE2, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, r1, 1, r2, &affinity, 1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r1, 1); + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); + } + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); + } +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +/* +** Generate code for scalar subqueries used as a subquery expression +** or EXISTS operator: +** +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery +** +** The pExpr parameter is the SELECT or EXISTS operator to be coded. +** +** Return the register that holds the result. For a multi-column SELECT, +** the result is stored in a contiguous array of registers and the +** return value is the register of the left-most result column. +** Return 0 if an error occurs. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ + int addrOnce = 0; /* Address of OP_Once at top of subroutine */ + int rReg = 0; /* Register storing resulting */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ + Expr *pLimit; /* New limit expression */ + + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; + + /* The evaluation of the EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can run this code just once + ** save the results, and reuse the same result on subsequent invocations. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) ){ + /* If this routine has already been coded, then invoke it as a + ** subroutine. */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + ExplainQueryPlan((pParse, 0, "REUSE SUBQUERY %d", pSel->selId)); + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + return pExpr->iTable; + } + + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. + */ + ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSel->selId)); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); + VdbeComment((v, "Init EXISTS result")); + } + if( pSel->pLimit ){ + /* The subquery already has a limit. If the pre-existing limit is X + ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + } + sqlite3ExprDelete(db, pSel->pLimit->pLeft); + pSel->pLimit->pLeft = pLimit; + }else{ + /* If there is no pre-existing limit add a limit of 1 */ + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); + } + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + return 0; + } + pExpr->iTable = rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); + + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); + } + + return rReg; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( (pIn->flags & EP_xIsSelect) ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + sqlite3VectorErrorMsg(pParse, pIn->pLeft); + return 1; + } + return 0; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for an IN expression. +** +** x IN (SELECT ...) +** x IN (value, value, ...) +** +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. +** +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. +** +** This routine generates code that jumps to destIfFalse if the LHS is not +** contained within the RHS. If due to NULLs we cannot determine if the LHS +** is contained in the RHS then jump to destIfNull. If the LHS is contained +** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. +*/ +static void sqlite3ExprCodeIN( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The IN expression */ + int destIfFalse, /* Jump here if LHS is not contained in the RHS */ + int destIfNull /* Jump here if the results are unknown due to NULLs */ +){ + int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + int eType; /* Type of the RHS */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ + Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ + int iTab = 0; /* Index to use */ + u8 okConstFactor = pParse->okConstFactor; + + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero( + pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 + ); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened with cursor iTab + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ + v = pParse->pVdbe; + assert( v!=0 ); /* OOM detected prior to this routine */ + VdbeNoopComment((v, "begin IN expr")); + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, + destIfFalse==destIfNull ? 0 : &rRhsHasNull, + aiMap, &iTab); + + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. + ** + ** Avoid factoring the LHS of the IN(...) expression out of the loop, + ** even if it is constant, as OP_Affinity may be used on the register + ** by code generated below. */ + assert( pParse->okConstFactor==okConstFactor ); + pParse->okConstFactor = 0; + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + pParse->okConstFactor = okConstFactor; + for(i=0; ix.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(pParse); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + sqlite3ReleaseTempReg(pParse, regToFree); + if( iinExpr-1 || destIfNull!=destIfFalse ){ + int op = rLhs!=r2 ? OP_Eq : OP_NotNull; + sqlite3VdbeAddOp4(v, op, rLhs, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, iinExpr-1 && op==OP_NotNull); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_NotNull); + sqlite3VdbeChangeP5(v, zAff[0]); + }else{ + int op = rLhs!=r2 ? OP_Ne : OP_IsNull; + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, op, rLhs, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, op==OP_Ne); + VdbeCoverageIf(v, op==OP_IsNull); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); + } + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeGoto(v, destIfFalse); + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; + }else{ + destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse); + } + if( pParse->nErr ) goto sqlite3ExprCodeIN_finished; + for(i=0; ipLeft, i); + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); + VdbeCoverage(v); + } + } + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, iTab, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(pParse); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; i1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); + VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Generate an instruction that will put the floating point +** value described by z[0..n-1] into register iMem. +** +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. +*/ +static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ + if( ALWAYS(z!=0) ){ + double value; + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ + if( negateFlag ) value = -value; + sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); + } +} +#endif + + +/* +** Generate an instruction that will put the integer describe by +** text z[0..n-1] into register iMem. +** +** Expr.u.zToken is always UTF8 and zero-terminated. +*/ +static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ + Vdbe *v = pParse->pVdbe; + if( pExpr->flags & EP_IntValue ){ + int i = pExpr->u.iValue; + assert( i>=0 ); + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else{ + int c; + i64 value; + const char *z = pExpr->u.zToken; + assert( z!=0 ); + c = sqlite3DecOrHexToI64(z, &value); + if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){ +#ifdef SQLITE_OMIT_FLOATING_POINT + sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); +#else +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s%s", negFlag?"-":"",z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } +#endif + }else{ + if( negFlag ){ value = c==3 ? SMALLEST_INT64 : -value; } + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); + } + } +} + + +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur + 1; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + pParse->iSelfTab = 0; + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** Generate code that will compute the value of generated column pCol +** and store the result in register regOut +*/ +void sqlite3ExprCodeGeneratedColumn( + Parse *pParse, + Column *pCol, + int regOut +){ + int iAddr; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pParse->iSelfTab!=0 ); + if( pParse->iSelfTab>0 ){ + iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); + }else{ + iAddr = 0; + } + sqlite3ExprCodeCopy(pParse, pCol->pDflt, regOut); + if( pCol->affinity>=SQLITE_AFF_TEXT ){ + sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); + } + if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + +/* +** Generate code to extract the value of the iCol-th column of a table. +*/ +void sqlite3ExprCodeGetColumnOfTable( + Vdbe *v, /* Parsing context */ + Table *pTab, /* The table containing the value */ + int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ + int iCol, /* Index of the column to extract */ + int regOut /* Extract the value into this register */ +){ + Column *pCol; + assert( v!=0 ); + if( pTab==0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); + return; + } + if( iCol<0 || iCol==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + }else{ + int op; + int x; + if( IsVirtual(pTab) ){ + op = OP_VColumn; + x = iCol; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ + Parse *pParse = sqlite3VdbeParser(v); + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pCol->zName); + }else{ + int savedSelfTab = pParse->iSelfTab; + pCol->colFlags |= COLFLAG_BUSY; + pParse->iSelfTab = iTabCur+1; + sqlite3ExprCodeGeneratedColumn(pParse, pCol, regOut); + pParse->iSelfTab = savedSelfTab; + pCol->colFlags &= ~COLFLAG_BUSY; + } + return; +#endif + }else if( !HasRowid(pTab) ){ + testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); + x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + op = OP_Column; + }else{ + x = sqlite3TableColumnToStorage(pTab,iCol); + testcase( x!=iCol ); + op = OP_Column; + } + sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); + sqlite3ColumnDefault(v, pTab, iCol, regOut); + } +} + +/* +** Generate code that will extract the iColumn-th column from +** table pTab and store the column value in register iReg. +** +** There must be an open cursor to pTab in iTable when this routine +** is called. If iColumn<0 then code is generated that extracts the rowid. +*/ +int sqlite3ExprCodeGetColumn( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column + FLAGS */ +){ + assert( pParse->pVdbe!=0 ); + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); + if( p5 ){ + VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + if( pOp->opcode==OP_Column ) pOp->p5 = p5; + } + return iReg; +} + +/* +** Generate code to move content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. +*/ +void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); +} + +/* +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. +*/ +static void exprToRegister(Expr *pExpr, int iReg){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); + if( NEVER(p==0) ) return; + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); +} + +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in continguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ +#if SQLITE_OMIT_SUBQUERY + iResult = 0; +#else + iResult = sqlite3CodeSubselect(pParse, p); +#endif + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + +/* +** If the last opcode is a OP_Copy, then set the do-not-merge flag (p5) +** so that a subsequent copy will not be merged into this one. +*/ +static void setDoNotMergeFlagOnCopy(Vdbe *v){ + if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + } +} + +/* +** Generate code to implement special SQL functions that are implemented +** in-line rather than by using the usual callbacks. +*/ +static int exprCodeInlineFunction( + Parse *pParse, /* Parsing context */ + ExprList *pFarg, /* List of function arguments */ + int iFuncId, /* Function ID. One of the INTFUNC_... values */ + int target /* Store function result in this register */ +){ + int nFarg; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pFarg!=0 ); + nFarg = pFarg->nExpr; + assert( nFarg>0 ); /* All in-line functions have at least one argument */ + switch( iFuncId ){ + case INLINEFUNC_coalesce: { + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evaluation of + ** arguments past the first non-NULL argument. + */ + int endCoalesce = sqlite3VdbeMakeLabel(pParse); + int i; + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + } + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + case INLINEFUNC_iif: { + Expr caseExpr; + memset(&caseExpr, 0, sizeof(caseExpr)); + caseExpr.op = TK_CASE; + caseExpr.x.pList = pFarg; + return sqlite3ExprCodeTarget(pParse, &caseExpr, target); + } + + default: { + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + assert( nFarg==1 || nFarg==2 ); + target = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + break; + } + + /*********************************************************************** + ** Test-only SQL functions that are only usable if enabled + ** via SQLITE_TESTCTRL_INTERNAL_FUNCTIONS + */ + case INLINEFUNC_expr_compare: { + /* Compare two expressions using sqlite3ExprCompare() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_expr_implies_expr: { + /* Compare two expressions using sqlite3ExprImpliesExpr() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_implies_nonnull_row: { + /* REsult of sqlite3ExprImpliesNonNullRow() */ + Expr *pA1; + assert( nFarg==2 ); + pA1 = pFarg->a[1].pExpr; + if( pA1->op==TK_COLUMN ){ + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } + +#ifdef SQLITE_DEBUG + case INLINEFUNC_affinity: { + /* The AFFINITY() function evaluates to a string that describes + ** the type affinity of the argument. This is used for testing of + ** the SQLite type logic. + */ + const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + char aff; + assert( nFarg==1 ); + aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + sqlite3VdbeLoadString(v, target, + (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); + break; + } +#endif + } + return target; +} + + +/* +** Generate code into the current Vdbe to evaluate the given +** expression. Attempt to store the results in register "target". +** Return the register where results are stored. +** +** With this routine, there is no guarantee that results will +** be stored in target. The result might be stored in some other +** register if it is convenient to do so. The calling function +** must check the return code and move the results to the desired +** register. +*/ +int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; /* The VM under construction */ + int op; /* The opcode being coded */ + int inReg = target; /* Results stored in register inReg */ + int regFree1 = 0; /* If non-zero free this temporary register */ + int regFree2 = 0; /* If non-zero free this temporary register */ + int r1, r2; /* Various register numbers */ + Expr tempX; /* Temporary expression node */ + int p5 = 0; + + assert( target>0 && target<=pParse->nMem ); + assert( v!=0 ); + +expr_code_doover: + if( pExpr==0 ){ + op = TK_NULL; + }else{ + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + op = pExpr->op; + } + switch( op ){ + case TK_AGG_COLUMN: { + AggInfo *pAggInfo = pExpr->pAggInfo; + struct AggInfo_col *pCol; + assert( pAggInfo!=0 ); + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + if( !pAggInfo->directMode ){ + assert( pCol->iMem>0 ); + return pCol->iMem; + }else if( pAggInfo->useSortingIdx ){ + Table *pTab = pCol->pTab; + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pCol->iSorterColumn, target); + if( pCol->iColumn<0 ){ + VdbeComment((v,"%s.rowid",pTab->zName)); + }else{ + VdbeComment((v,"%s.%s",pTab->zName,pTab->aCol[pCol->iColumn].zName)); + if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } + } + return target; + } + /* Otherwise, fall thru into the TK_COLUMN case */ + /* no break */ deliberate_fall_through + } + case TK_COLUMN: { + int iTab = pExpr->iTable; + int iReg; + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + /* This COLUMN expression is really a constant due to WHERE clause + ** constraints, and that constant is coded by the pExpr->pLeft + ** expresssion. However, make sure the constant has the correct + ** datatype by applying the Affinity of the table column to the + ** constant. + */ + int aff; + iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); + if( pExpr->y.pTab ){ + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + }else{ + aff = pExpr->affExpr; + } + if( aff>SQLITE_AFF_BLOB ){ + static const char zAff[] = "B\000C\000D\000E"; + assert( SQLITE_AFF_BLOB=='A' ); + assert( SQLITE_AFF_TEXT=='B' ); + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, + &zAff[(aff-'B')*2], P4_STATIC); + } + return iReg; + } + if( iTab<0 ){ + if( pParse->iSelfTab<0 ){ + /* Other columns in the same row for CHECK constraints or + ** generated columns or for inserting into partial index. + ** The row is unpacked into registers beginning at + ** 0-(pParse->iSelfTab). The rowid (if any) is in a register + ** immediately prior to the first column. + */ + Column *pCol; + Table *pTab = pExpr->y.pTab; + int iSrc; + int iCol = pExpr->iColumn; + assert( pTab!=0 ); + assert( iCol>=XN_ROWID ); + assert( iColnCol ); + if( iCol<0 ){ + return -1-pParse->iSelfTab; + } + pCol = pTab->aCol + iCol; + testcase( iCol!=sqlite3TableColumnToStorage(pTab,iCol) ); + iSrc = sqlite3TableColumnToStorage(pTab, iCol) - pParse->iSelfTab; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zName); + return 0; + } + pCol->colFlags |= COLFLAG_BUSY; + if( pCol->colFlags & COLFLAG_NOTAVAIL ){ + sqlite3ExprCodeGeneratedColumn(pParse, pCol, iSrc); + } + pCol->colFlags &= ~(COLFLAG_BUSY|COLFLAG_NOTAVAIL); + return iSrc; + }else +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + if( pCol->affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iSrc, target); + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + return target; + }else{ + return iSrc; + } + }else{ + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab - 1; + } + } + iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, + pExpr->iColumn, iTab, target, + pExpr->op2); + if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } + return iReg; + } + case TK_INTEGER: { + codeInteger(pParse, pExpr, 0, target); + return target; + } + case TK_TRUEFALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target); + return target; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pExpr->u.zToken, 0, target); + return target; + } +#endif + case TK_STRING: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); + return target; + } + default: { + /* Make NULL the default case so that if a bug causes an illegal + ** Expr node to be passed into this function, it will be handled + ** sanely and not crash. But keep the assert() to bring the problem + ** to the attention of the developers. */ + assert( op==TK_NULL ); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + return target; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + int n; + const char *z; + char *zBlob; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + z = &pExpr->u.zToken[2]; + n = sqlite3Strlen30(z) - 1; + assert( z[n]=='\'' ); + zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); + sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); + return target; + } +#endif + case TK_VARIABLE: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pExpr->u.zToken[0]!=0 ); + sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); + if( pExpr->u.zToken[1]!=0 ){ + const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); + assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); + pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ + sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); + } + return target; + } + case TK_REGISTER: { + return pExpr->iTable; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + if( inReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); + return inReg; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | p5, + ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } + break; + } + case TK_AND: + case TK_OR: + case TK_PLUS: + case TK_STAR: + case TK_MINUS: + case TK_REM: + case TK_BITAND: + case TK_BITOR: + case TK_SLASH: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: { + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); + assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); + assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); + assert( TK_BITAND==OP_BitAnd ); testcase( op==TK_BITAND ); + assert( TK_BITOR==OP_BitOr ); testcase( op==TK_BITOR ); + assert( TK_SLASH==OP_Divide ); testcase( op==TK_SLASH ); + assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); + assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); + assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_UMINUS: { + Expr *pLeft = pExpr->pLeft; + assert( pLeft ); + if( pLeft->op==TK_INTEGER ){ + codeInteger(pParse, pLeft, 1, target); + return target; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( pLeft->op==TK_FLOAT ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pLeft->u.zToken, 1, target); + return target; +#endif + }else{ + tempX.op = TK_INTEGER; + tempX.flags = EP_IntValue|EP_TokenOnly; + tempX.u.iValue = 0; + ExprClearVVAProperties(&tempX); + r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); + sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); + testcase( regFree2==0 ); + } + break; + } + case TK_BITNOT: + case TK_NOT: { + assert( TK_BITNOT==OP_BitNot ); testcase( op==TK_BITNOT ); + assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + sqlite3VdbeAddOp2(v, op, r1, inReg); + break; + } + case TK_TRUTH: { + int isTrue; /* IS TRUE or IS NOT TRUE */ + int bNormal; /* IS TRUE or IS FALSE */ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + bNormal = pExpr->op2==TK_IS; + testcase( isTrue && bNormal); + testcase( !isTrue && bNormal); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + int addr; + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + addr = sqlite3VdbeAddOp1(v, op, r1); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + sqlite3VdbeJumpHere(v, addr); + break; + } + case TK_AGG_FUNCTION: { + AggInfo *pInfo = pExpr->pAggInfo; + if( pInfo==0 + || NEVER(pExpr->iAgg<0) + || NEVER(pExpr->iAgg>=pInfo->nFunc) + ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + }else{ + return pInfo->aFunc[pExpr->iAgg].iMem; + } + break; + } + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + int nFarg; /* Number of function arguments */ + FuncDef *pDef; /* The function definition object */ + const char *zId; /* The function name */ + u32 constMask = 0; /* Mask of function arguments that are constant */ + int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ + u8 enc = ENC(db); /* The text encoding used by this database */ + CollSeq *pColl = 0; /* A collating sequence */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + return pExpr->y.pWin->regResult; + } +#endif + + if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + /* SQL functions can be expensive. So try to avoid running them + ** multiple times if we know they always give the same result */ + return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( !ExprHasProperty(pExpr, EP_TokenOnly) ); + pFarg = pExpr->x.pList; + nFarg = pFarg ? pFarg->nExpr : 0; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + zId = pExpr->u.zToken; + pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif + if( pDef==0 || pDef->xFinalize!=0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); + break; + } + if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); + assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); + return exprCodeInlineFunction(pParse, pFarg, + SQLITE_PTR_TO_INT(pDef->pUserData), target); + }else if( pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE) ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); + } + + for(i=0; ia[i].pExpr) ){ + testcase( i==31 ); + constMask |= MASKBIT32(i); + } + if( (pDef->funcFlags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + } + } + if( pFarg ){ + if( constMask ){ + r1 = pParse->nMem+1; + pParse->nMem += nFarg; + }else{ + r1 = sqlite3GetTempRange(pParse, nFarg); + } + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); + pFarg->a[0].pExpr->op2 = + pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + } + } + + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, + SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + }else{ + r1 = 0; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Possibly overload the function if the first argument is + ** a virtual table column. + ** + ** For infix functions (LIKE, GLOB, REGEXP, and MATCH) use the + ** second argument, not the first, as the argument to test to + ** see if it is a column in a virtual table. This is done because + ** the left operand of infix functions (the operand we want to + ** control overloading) ends up as the second argument to the + ** function. The expression "A glob B" is equivalent to + ** "glob(B,A). We want to use the A in "A glob B" to test + ** for function overloading. But we use the B term in "glob(B,A)". + */ + if( nFarg>=2 && ExprHasProperty(pExpr, EP_InfixFunc) ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); + }else if( nFarg>0 ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); + } +#endif + if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + if( !pColl ) pColl = db->pDfltColl; + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + } +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + if( pDef->funcFlags & SQLITE_FUNC_OFFSET ){ + Expr *pArg = pFarg->a[0].pExpr; + if( pArg->op==TK_COLUMN ){ + sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + }else +#endif + { + sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, + pDef, pExpr->op2); + } + if( nFarg ){ + if( constMask==0 ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + }else{ + sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask, 1); + } + } + return target; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: + case TK_SELECT: { + int nCol; + testcase( op==TK_EXISTS ); + testcase( op==TK_SELECT ); + if( pParse->db->mallocFailed ){ + return 0; + }else if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr); + } + break; + } + case TK_SELECT_COLUMN: { + int n; + if( pExpr->pLeft->iTable==0 ){ + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft); + } + assert( pExpr->iTable==0 || pExpr->pLeft->op==TK_SELECT ); + if( pExpr->iTable!=0 + && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) + ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pExpr->iTable, n); + } + return pExpr->pLeft->iTable + pExpr->iColumn; + } + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + sqlite3VdbeResolveLabel(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); + sqlite3VdbeResolveLabel(v, destIfNull); + return target; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. OSSFuzz. */ + } + + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + ** + ** The expression is implemented using an OP_Param opcode. The p1 + ** parameter is set to 0 for an old.rowid reference, or to (i+1) + ** to reference another column of the old.* pseudo-table, where + ** i is the index of the column. For a new.rowid reference, p1 is + ** set to (n+1), where n is the number of columns in each pseudo-table. + ** For a reference to any other column in the new.* pseudo-table, p1 + ** is set to (n+2+i), where n and i are as defined previously. For + ** example, if the table on which triggers are being fired is + ** declared as: + ** + ** CREATE TABLE t1(a, b); + ** + ** Then p1 is interpreted as follows: + ** + ** p1==0 -> old.rowid p1==3 -> new.rowid + ** p1==1 -> old.a p1==4 -> new.a + ** p1==2 -> old.b p1==5 -> new.b + */ + Table *pTab = pExpr->y.pTab; + int iCol = pExpr->iColumn; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + + sqlite3TableColumnToStorage(pTab, iCol); + + assert( pExpr->iTable==0 || pExpr->iTable==1 ); + assert( iCol>=-1 && iColnCol ); + assert( pTab->iPKey<0 || iCol!=pTab->iPKey ); + assert( p1>=0 && p1<(pTab->nCol*2+2) ); + + sqlite3VdbeAddOp2(v, OP_Param, p1, target); + VdbeComment((v, "r[%d]=%s.%s", target, + (pExpr->iTable ? "new" : "old"), + (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[iCol].zName) + )); + +#ifndef SQLITE_OMIT_FLOATING_POINT + /* If the column has REAL affinity, it may currently be stored as an + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ + if( iCol>=0 && pTab->aCol[iCol].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } +#endif + break; + } + + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } + + /* TK_IF_NULL_ROW Expr nodes are inserted ahead of expressions + ** that derive from the right-hand table of a LEFT JOIN. The + ** Expr.iTable value is the table number for the right-hand table. + ** The expression is only evaluated if that table is not currently + ** on a LEFT JOIN NULL row. + */ + case TK_IF_NULL_ROW: { + int addrINR; + u8 okConstFactor = pParse->okConstFactor; + addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); + /* Temporarily disable factoring of constant expressions, since + ** even though expressions may appear to be constant, they are not + ** really constant because they originate from the right-hand side + ** of a LEFT JOIN. */ + pParse->okConstFactor = 0; + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + pParse->okConstFactor = okConstFactor; + sqlite3VdbeJumpHere(v, addrINR); + sqlite3VdbeChangeP3(v, addrINR, inReg); + break; + } + + /* + ** Form A: + ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form B: + ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form A is can be transformed into the equivalent form B as follows: + ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... + ** WHEN x=eN THEN rN ELSE y END + ** + ** X (if it exists) is in pExpr->pLeft. + ** Y is in the last element of pExpr->x.pList if pExpr->x.pList->nExpr is + ** odd. The Y is also optional. If the number of elements in x.pList + ** is even, then Y is omitted and the "otherwise" result is NULL. + ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + ** + ** The result of the expression is the Ri for the first matching Ei, + ** or if there is no matching Ei, the ELSE term Y, or if there is + ** no ELSE term, NULL. + */ + case TK_CASE: { + int endLabel; /* GOTO label for end of CASE stmt */ + int nextCase; /* GOTO label for next WHEN clause */ + int nExpr; /* 2x number of WHEN terms */ + int i; /* Loop counter */ + ExprList *pEList; /* List of WHEN terms */ + struct ExprList_item *aListelem; /* Array of WHEN terms */ + Expr opCompare; /* The X==Ei expression */ + Expr *pX; /* The X expression */ + Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); + assert(pExpr->x.pList->nExpr > 0); + pEList = pExpr->x.pList; + aListelem = pEList->a; + nExpr = pEList->nExpr; + endLabel = sqlite3VdbeMakeLabel(pParse); + if( (pX = pExpr->pLeft)!=0 ){ + pDel = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDel); + break; + } + testcase( pX->op==TK_COLUMN ); + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); + opCompare.op = TK_EQ; + opCompare.pLeft = pDel; + pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; + } + for(i=0; iop==TK_COLUMN ); + sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); + testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); + sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); + sqlite3VdbeGoto(v, endLabel); + sqlite3VdbeResolveLabel(v, nextCase); + } + if( (nExpr&1)!=0 ){ + sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + sqlite3ExprDelete(db, pDel); + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endLabel); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + assert( pExpr->affExpr==OE_Rollback + || pExpr->affExpr==OE_Abort + || pExpr->affExpr==OE_Fail + || pExpr->affExpr==OE_Ignore + ); + if( !pParse->pTriggerTab && !pParse->nested ){ + sqlite3ErrorMsg(pParse, + "RAISE() may only be used within a trigger-program"); + return 0; + } + if( pExpr->affExpr==OE_Abort ){ + sqlite3MayAbort(pParse); + } + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( pExpr->affExpr==OE_Ignore ){ + sqlite3VdbeAddOp4( + v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + VdbeCoverage(v); + }else{ + sqlite3HaltConstraint(pParse, + pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, + pExpr->affExpr, pExpr->u.zToken, 0, 0); + } + + break; + } +#endif + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + return inReg; +} + +/* +** Generate code that will evaluate expression pExpr just one time +** per prepared statement execution. +** +** If the expression uses functions (that might throw an exception) then +** guard them with an OP_Once opcode to ensure that the code is only executed +** once. If no functions are involved, then factor the code out and put it at +** the end of the prepared statement in the initialization section. +** +** If regDest>=0 then the result is always stored in that register and the +** result is not reusable. If regDest<0 then this routine is free to +** store the value whereever it wants. The register where the expression +** is stored is returned. When regDest<0, two identical expressions might +** code to the same register, if they do not contain function calls and hence +** are factored out into the initialization section at the end of the +** prepared statement. +*/ +int sqlite3ExprCodeRunJustOnce( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The expression to code when the VDBE initializes */ + int regDest /* Store the value in this register */ +){ + ExprList *p; + assert( ConstFactorOk(pParse) ); + p = pParse->pConstExpr; + if( regDest<0 && p ){ + struct ExprList_item *pItem; + int i; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->reusable && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 ){ + return pItem->u.iConstExprReg; + } + } + } + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pExpr!=0 && ExprHasProperty(pExpr, EP_HasFunc) ){ + Vdbe *v = pParse->pVdbe; + int addr; + assert( v ); + addr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + pParse->okConstFactor = 0; + if( !pParse->db->mallocFailed ){ + if( regDest<0 ) regDest = ++pParse->nMem; + sqlite3ExprCode(pParse, pExpr, regDest); + } + pParse->okConstFactor = 1; + sqlite3ExprDelete(pParse->db, pExpr); + sqlite3VdbeJumpHere(v, addr); + }else{ + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->reusable = regDest<0; + if( regDest<0 ) regDest = ++pParse->nMem; + pItem->u.iConstExprReg = regDest; + } + pParse->pConstExpr = p; + } + return regDest; +} + +/* +** Generate code to evaluate an expression and store the results +** into a register. Return the register number where the results +** are stored. +** +** If the register is a temporary register that can be deallocated, +** then write its number into *pReg. If the result register is not +** a temporary, then set *pReg to zero. +** +** If pExpr is a constant, then this routine might generate this +** code to fill the register in the initialization section of the +** VDBE program, in order to factor it out of the evaluation loop. +*/ +int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ + int r2; + pExpr = sqlite3ExprSkipCollateAndLikely(pExpr); + if( ConstFactorOk(pParse) + && ALWAYS(pExpr!=0) + && pExpr->op!=TK_REGISTER + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + *pReg = 0; + r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); + }else{ + int r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } + } + return r2; +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. +*/ +void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ + int inReg; + + assert( pExpr==0 || !ExprHasVVAProperty(pExpr,EP_Immutable) ); + assert( target>0 && target<=pParse->nMem ); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + if( pParse->pVdbe==0 ) return; + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + if( inReg!=target ){ + u8 op; + if( ExprHasProperty(pExpr,EP_Subquery) ){ + op = OP_Copy; + }else{ + op = OP_SCopy; + } + sqlite3VdbeAddOp2(pParse->pVdbe, op, inReg, target); + } +} + +/* +** Make a transient copy of expression pExpr and then code it using +** sqlite3ExprCode(). This routine works just like sqlite3ExprCode() +** except that the input expression is guaranteed to be unchanged. +*/ +void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ + sqlite3 *db = pParse->db; + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ) sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprDelete(db, pExpr); +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. If the expression is constant, then this routine +** might choose to code the expression at initialization time. +*/ +void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); + }else{ + sqlite3ExprCodeCopy(pParse, pExpr, target); + } +} + +/* +** Generate code that pushes the value of every element of the given +** expression list into a sequence of registers beginning at target. +** +** Return the number of elements evaluated. The number returned will +** usually be pList->nExpr but might be reduced if SQLITE_ECEL_OMITREF +** is defined. +** +** The SQLITE_ECEL_DUP flag prevents the arguments from being +** filled using OP_SCopy. OP_Copy must be used instead. +** +** The SQLITE_ECEL_FACTOR argument allows constant arguments to be +** factored out into initialization code. +** +** The SQLITE_ECEL_REF flag means that expressions in the list with +** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored +** in registers at srcReg, and so the value can be copied from there. +** If SQLITE_ECEL_OMITREF is also set, then the values with u.x.iOrderByCol>0 +** are simply omitted rather than being copied from srcReg. +*/ +int sqlite3ExprCodeExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The expression list to be coded */ + int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ + u8 flags /* SQLITE_ECEL_* flags */ +){ + struct ExprList_item *pItem; + int i, j, n; + u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; + assert( pList!=0 ); + assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ + n = pList->nExpr; + if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; + for(pItem=pList->a, i=0; ipExpr; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pItem->bSorterRef ){ + i--; + n--; + }else +#endif + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ + if( flags & SQLITE_ECEL_OMITREF ){ + i--; + n--; + }else{ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + } + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); + }else{ + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + VdbeOp *pOp; + if( copyOp==OP_Copy + && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && pOp->p1+pOp->p3+1==inReg + && pOp->p2+pOp->p3+1==target+i + && pOp->p5==0 /* The do-not-merge flag must be clear */ + ){ + pOp->p3++; + }else{ + sqlite3VdbeAddOp2(v, copyOp, inReg, target+i); + } + } + } + } + return n; +} + +/* +** Generate code for a BETWEEN operator. +** +** x BETWEEN y AND z +** +** The above is equivalent to +** +** x>=y AND x<=z +** +** Code it as such, taking care to do the common subexpression +** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. +*/ +static void exprCodeBetween( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The BETWEEN expression */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ + int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +){ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr compLeft; /* The x>=y term */ + Expr compRight; /* The x<=z term */ + int regFree1 = 0; /* Temporary use register */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; + + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pDel = sqlite3ExprDup(db, pExpr->pLeft, 0); + if( db->mallocFailed==0 ){ + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = pDel; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = pDel; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); + }else{ + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_FromJoin bit. Bummer. */ + pDel->flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + } + sqlite3ReleaseTempReg(pParse, regFree1); + } + sqlite3ExprDelete(db, pDel); + + /* Ensure adequate test coverage */ + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is true but execution +** continues straight thru if the expression is false. +** +** If the expression evaluates to NULL (neither true nor false), then +** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. +** +** This code depends on the fact that certain token values (ex: TK_EQ) +** are the same as opcode values (ex: OP_Eq) that implement the corresponding +** operation. Special comments in vdbe.c and the mkopcodeh.awk script in +** the make process cause these values to align. Assert()s in the code +** below verify that the numbers are aligned correctly. +*/ +void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( NEVER(pExpr==0) ) return; /* No way this can happen */ + assert( !ExprHasVVAProperty(pExpr, EP_Immutable) ); + op = pExpr->op; + switch( op ){ + case TK_AND: + case TK_OR: { + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); + }else if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + } + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + }else{ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + } + break; + } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* no break */ deliberate_fall_through + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = jumpIfNull ? dest : destIfFalse; + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeGoto(v, dest); + sqlite3VdbeResolveLabel(v, destIfFalse); + break; + } +#endif + default: { + default_expr: + if( ExprAlwaysTrue(pExpr) ){ + sqlite3VdbeGoto(v, dest); + }else if( ExprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is false but execution +** continues straight thru if the expression is true. +** +** If the expression evaluates to NULL (neither true nor false) then +** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull +** is 0. +*/ +void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( pExpr==0 ) return; + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + + /* The value of pExpr->op and op are related as follows: + ** + ** pExpr->op op + ** --------- ---------- + ** TK_ISNULL OP_NotNull + ** TK_NOTNULL OP_IsNull + ** TK_NE OP_Eq + ** TK_EQ OP_Ne + ** TK_GT OP_Le + ** TK_LE OP_Gt + ** TK_GE OP_Lt + ** TK_LT OP_Ge + ** + ** For other values of pExpr->op, op is undefined and unused. + ** The value of TK_ and OP_ constants are arranged such that we + ** can compute the mapping above using the following expression. + ** Assert()s verify that the computation is correct. + */ + op = ((pExpr->op+(TK_ISNULL&1))^1)-(TK_ISNULL&1); + + /* Verify correct alignment of TK_ and OP_ constants + */ + assert( pExpr->op!=TK_ISNULL || op==OP_NotNull ); + assert( pExpr->op!=TK_NOTNULL || op==OP_IsNull ); + assert( pExpr->op!=TK_NE || op==OP_Eq ); + assert( pExpr->op!=TK_EQ || op==OP_Ne ); + assert( pExpr->op!=TK_LT || op==OP_Ge ); + assert( pExpr->op!=TK_LE || op==OP_Gt ); + assert( pExpr->op!=TK_GT || op==OP_Le ); + assert( pExpr->op!=TK_GE || op==OP_Lt ); + + switch( pExpr->op ){ + case TK_AND: + case TK_OR: { + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); + }else if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + /* IS TRUE and IS NOT FALSE */ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + + }else{ + /* IS FALSE and IS NOT TRUE */ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + } + break; + } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* no break */ deliberate_fall_through + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); + testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + if( jumpIfNull ){ + sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + }else{ + int destIfNull = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); + sqlite3VdbeResolveLabel(v, destIfNull); + } + break; + } +#endif + default: { + default_expr: + if( ExprAlwaysFalse(pExpr) ){ + sqlite3VdbeGoto(v, dest); + }else if( ExprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before +** code generation, and that copy is deleted after code generation. This +** ensures that the original pExpr is unchanged. +*/ +void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ + sqlite3 *db = pParse->db; + Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed==0 ){ + sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); + } + sqlite3ExprDelete(db, pCopy); +} + +/* +** Expression pVar is guaranteed to be an SQL variable. pExpr may be any +** type of expression. +** +** If pExpr is a simple SQL value - an integer, real, string, blob +** or NULL value - then the VDBE currently being prepared is configured +** to re-prepare each time a new value is bound to variable pVar. +** +** Additionally, if pExpr is a simple SQL value and the value is the +** same as that currently bound to variable pVar, non-zero is returned. +** Otherwise, if the values are not the same or if pExpr is not a simple +** SQL value, zero is returned. +*/ +static int exprCompareVariable(Parse *pParse, Expr *pVar, Expr *pExpr){ + int res = 0; + int iVar; + sqlite3_value *pL, *pR = 0; + + sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); + if( pR ){ + iVar = pVar->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + pL = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, SQLITE_AFF_BLOB); + if( pL ){ + if( sqlite3_value_type(pL)==SQLITE_TEXT ){ + sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */ + } + res = 0==sqlite3MemCompare(pL, pR, 0); + } + sqlite3ValueFree(pR); + sqlite3ValueFree(pL); + } + + return res; +} + +/* +** Do a deep comparison of two expression trees. Return 0 if the two +** expressions are completely identical. Return 1 if they differ only +** by a COLLATE operator at the top level. Return 2 if there are differences +** other than the top-level COLLATE operator. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** The pA side might be using TK_REGISTER. If that is the case and pB is +** not using TK_REGISTER but is otherwise equivalent, then still return 0. +** +** Sometimes this routine will return 2 even if the two expressions +** really are equivalent. If we cannot prove that the expressions are +** identical, we return 2 just to be safe. So if this routine +** returns 2, then you do not really know for certain if the two +** expressions are the same. But if you get a 0 or 1 return, then you +** can be sure the expressions are the same. In the places where +** this routine is used, it does not hurt to get an extra 2 - that +** just might result in some slightly slower code. But returning +** an incorrect 0 or 1 could lead to a malfunction. +** +** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in +** pParse->pReprepare can be matched against literals in pB. The +** pParse->pVdbe->expmask bitmask is updated for each variable referenced. +** If pParse is NULL (the normal case) then any TK_VARIABLE term in +** Argument pParse should normally be NULL. If it is not NULL and pA or +** pB causes a return value of 2. +*/ +int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTab){ + u32 combinedFlags; + if( pA==0 || pB==0 ){ + return pB==pA ? 0 : 2; + } + if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){ + return 0; + } + combinedFlags = pA->flags | pB->flags; + if( combinedFlags & EP_IntValue ){ + if( (pA->flags&pB->flags&EP_IntValue)!=0 && pA->u.iValue==pB->u.iValue ){ + return 0; + } + return 2; + } + if( pA->op!=pB->op || pA->op==TK_RAISE ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA->pLeft,pB,iTab)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ + return 1; + } + return 2; + } + if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ + if( pA->op==TK_FUNCTION || pA->op==TK_AGG_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( pA->op==pB->op ); + if( ExprHasProperty(pA,EP_WinFunc)!=ExprHasProperty(pB,EP_WinFunc) ){ + return 2; + } + if( ExprHasProperty(pA,EP_WinFunc) ){ + if( sqlite3WindowCompare(pParse, pA->y.pWin, pB->y.pWin, 1)!=0 ){ + return 2; + } + } +#endif + }else if( pA->op==TK_NULL ){ + return 0; + }else if( pA->op==TK_COLLATE ){ + if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( ALWAYS(pB->u.zToken!=0) && strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + return 2; + } + } + if( (pA->flags & (EP_Distinct|EP_Commuted)) + != (pB->flags & (EP_Distinct|EP_Commuted)) ) return 2; + if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ + if( combinedFlags & EP_xIsSelect ) return 2; + if( (combinedFlags & EP_FixedCol)==0 + && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; + if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2; + if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; + if( pA->op!=TK_STRING + && pA->op!=TK_TRUEFALSE + && ALWAYS((combinedFlags & EP_Reduced)==0) + ){ + if( pA->iColumn!=pB->iColumn ) return 2; + if( pA->op2!=pB->op2 && pA->op==TK_TRUTH ) return 2; + if( pA->op!=TK_IN && pA->iTable!=pB->iTable && pA->iTable!=iTab ){ + return 2; + } + } + } + return 0; +} + +/* +** Compare two ExprList objects. Return 0 if they are identical, 1 +** if they are certainly different, or 2 if it is not possible to +** determine if they are identical or not. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** This routine might return non-zero for equivalent ExprLists. The +** only consequence will be disabled optimizations. But this routine +** must never return 0 if the two ExprList objects are different, or +** a malfunction will result. +** +** Two NULL pointers are considered to be the same. But a NULL pointer +** always differs from a non-NULL pointer. +*/ +int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ + int i; + if( pA==0 && pB==0 ) return 0; + if( pA==0 || pB==0 ) return 1; + if( pA->nExpr!=pB->nExpr ) return 1; + for(i=0; inExpr; i++){ + int res; + Expr *pExprA = pA->a[i].pExpr; + Expr *pExprB = pB->a[i].pExpr; + if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1; + if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res; + } + return 0; +} + +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ + return sqlite3ExprCompare(0, + sqlite3ExprSkipCollateAndLikely(pA), + sqlite3ExprSkipCollateAndLikely(pB), + iTab); +} + +/* +** Return non-zero if Expr p can only be true if pNN is not NULL. +** +** Or if seenNot is true, return non-zero if Expr p can only be +** non-NULL if pNN is not NULL +*/ +static int exprImpliesNotNull( + Parse *pParse, /* Parsing context */ + Expr *p, /* The expression to be checked */ + Expr *pNN, /* The expression that is NOT NULL */ + int iTab, /* Table being evaluated */ + int seenNot /* Return true only if p can be any non-NULL value */ +){ + assert( p ); + assert( pNN ); + if( sqlite3ExprCompare(pParse, p, pNN, iTab)==0 ){ + return pNN->op!=TK_NULL; + } + switch( p->op ){ + case TK_IN: { + if( seenNot && ExprHasProperty(p, EP_xIsSelect) ) return 0; + assert( ExprHasProperty(p,EP_xIsSelect) + || (p->x.pList!=0 && p->x.pList->nExpr>0) ); + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BETWEEN: { + ExprList *pList = p->x.pList; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + if( seenNot ) return 0; + if( exprImpliesNotNull(pParse, pList->a[0].pExpr, pNN, iTab, 1) + || exprImpliesNotNull(pParse, pList->a[1].pExpr, pNN, iTab, 1) + ){ + return 1; + } + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_PLUS: + case TK_MINUS: + case TK_BITOR: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: + seenNot = 1; + /* no break */ deliberate_fall_through + case TK_STAR: + case TK_REM: + case TK_BITAND: + case TK_SLASH: { + if( exprImpliesNotNull(pParse, p->pRight, pNN, iTab, seenNot) ) return 1; + /* no break */ deliberate_fall_through + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: + case TK_UMINUS: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, seenNot); + } + case TK_TRUTH: { + if( seenNot ) return 0; + if( p->op2!=TK_IS ) return 0; + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BITNOT: + case TK_NOT: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + } + return 0; +} + +/* +** Return true if we can prove the pE2 will always be true if pE1 is +** true. Return false if we cannot complete the proof or if pE2 might +** be false. Examples: +** +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** +** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has +** Expr.iTable<0 then assume a table number given by iTab. +** +** If pParse is not NULL, then the values of bound variables in pE1 are +** compared against literal values in pE2 and pParse->pVdbe->expmask is +** modified to record which bound variables are referenced. If pParse +** is NULL, then false will be returned if pE1 contains any bound variables. +** +** When in doubt, return false. Returning true might give a performance +** improvement. Returning false might cause a performance reduction, but +** it will always give the correct answer and is hence always safe. +*/ +int sqlite3ExprImpliesExpr(Parse *pParse, Expr *pE1, Expr *pE2, int iTab){ + if( sqlite3ExprCompare(pParse, pE1, pE2, iTab)==0 ){ + return 1; + } + if( pE2->op==TK_OR + && (sqlite3ExprImpliesExpr(pParse, pE1, pE2->pLeft, iTab) + || sqlite3ExprImpliesExpr(pParse, pE1, pE2->pRight, iTab) ) + ){ + return 1; + } + if( pE2->op==TK_NOTNULL + && exprImpliesNotNull(pParse, pE1, pE2->pLeft, iTab, 0) + ){ + return 1; + } + return 0; +} + +/* +** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). +** If the expression node requires that the table at pWalker->iCur +** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. +** +** This routine controls an optimization. False positives (setting +** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives +** (never setting pWalker->eCode) is a harmless missed optimization. +*/ +static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; + switch( pExpr->op ){ + case TK_ISNOT: + case TK_ISNULL: + case TK_NOTNULL: + case TK_IS: + case TK_OR: + case TK_VECTOR: + case TK_CASE: + case TK_IN: + case TK_FUNCTION: + case TK_TRUTH: + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_ISNULL ); + testcase( pExpr->op==TK_NOTNULL ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_VECTOR ); + testcase( pExpr->op==TK_CASE ); + testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_FUNCTION ); + testcase( pExpr->op==TK_TRUTH ); + return WRC_Prune; + case TK_COLUMN: + if( pWalker->u.iCur==pExpr->iTable ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Prune; + + case TK_AND: + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pExpr->pRight); + } + } + return WRC_Prune; + + case TK_BETWEEN: + if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ + assert( pWalker->eCode ); + return WRC_Abort; + } + return WRC_Prune; + + /* Virtual tables are allowed to use constraints like x=NULL. So + ** a term of the form x=y does not prove that y is not null if x + ** is the column of a virtual table */ + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + /* The y.pTab=0 assignment in wherecode.c always happens after the + ** impliesNotNullRow() test */ + if( (pLeft->op==TK_COLUMN && ALWAYS(pLeft->y.pTab!=0) + && IsVirtual(pLeft->y.pTab)) + || (pRight->op==TK_COLUMN && ALWAYS(pRight->y.pTab!=0) + && IsVirtual(pRight->y.pTab)) + ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through + } + default: + return WRC_Continue; + } +} + +/* +** Return true (non-zero) if expression p can only be true if at least +** one column of table iTab is non-null. In other words, return true +** if expression p will always be NULL or false if every column of iTab +** is NULL. +** +** False negatives are acceptable. In other words, it is ok to return +** zero even if expression p will never be true of every column of iTab +** is NULL. A false negative is merely a missed optimization opportunity. +** +** False positives are not allowed, however. A false positive may result +** in an incorrect answer. +** +** Terms of p that are marked with EP_FromJoin (and hence that come from +** the ON or USING clauses of LEFT JOINS) are excluded from the analysis. +** +** This routine is used to check if a LEFT JOIN can be converted into +** an ordinary JOIN. The p argument is the WHERE clause. If the WHERE +** clause requires that some column of the right table of the LEFT JOIN +** be non-NULL, then the LEFT JOIN can be safely converted into an +** ordinary join. +*/ +int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ + Walker w; + p = sqlite3ExprSkipCollateAndLikely(p); + if( p==0 ) return 0; + if( p->op==TK_NOTNULL ){ + p = p->pLeft; + }else{ + while( p->op==TK_AND ){ + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + p = p->pRight; + } + } + w.xExprCallback = impliesNotNullRow; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + w.eCode = 0; + w.u.iCur = iTab; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3TableColumnToIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + +/* +** An instance of the following structure is used by the tree walker +** to count references to table columns in the arguments of an +** aggregate function, in order to implement the +** sqlite3FunctionThisSrc() routine. +*/ +struct SrcCount { + SrcList *pSrc; /* One particular FROM clause in a nested query */ + int iSrcInner; /* Smallest cursor number in this context */ + int nThis; /* Number of references to columns in pSrcList */ + int nOther; /* Number of references to columns in other FROM clauses */ +}; + +/* +** xSelect callback for sqlite3FunctionUsesThisSrc(). If this is the first +** SELECT with a FROM clause encountered during this iteration, set +** SrcCount.iSrcInner to the cursor number of the leftmost object in +** the FROM cause. +*/ +static int selectSrcCount(Walker *pWalker, Select *pSel){ + struct SrcCount *p = pWalker->u.pSrcCount; + if( p->iSrcInner==0x7FFFFFFF && ALWAYS(pSel->pSrc) && pSel->pSrc->nSrc ){ + pWalker->u.pSrcCount->iSrcInner = pSel->pSrc->a[0].iCursor; + } + return WRC_Continue; +} + +/* +** Count the number of references to columns. +*/ +static int exprSrcCount(Walker *pWalker, Expr *pExpr){ + /* There was once a NEVER() on the second term on the grounds that + ** sqlite3FunctionUsesThisSrc() was always called before + ** sqlite3ExprAnalyzeAggregates() and so the TK_COLUMNs have not yet + ** been converted into TK_AGG_COLUMN. But this is no longer true due + ** to window functions - sqlite3WindowRewrite() may now indirectly call + ** FunctionUsesThisSrc() when creating a new sub-select. */ + if( pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN ){ + int i; + struct SrcCount *p = pWalker->u.pSrcCount; + SrcList *pSrc = p->pSrc; + int nSrc = pSrc ? pSrc->nSrc : 0; + for(i=0; iiTable==pSrc->a[i].iCursor ) break; + } + if( inThis++; + }else if( pExpr->iTableiSrcInner ){ + /* In a well-formed parse tree (no name resolution errors), + ** TK_COLUMN nodes with smaller Expr.iTable values are in an + ** outer context. Those are the only ones to count as "other" */ + p->nOther++; + } + } + return WRC_Continue; +} + +/* +** Determine if any of the arguments to the pExpr Function reference +** pSrcList. Return true if they do. Also return true if the function +** has no arguments or has only constant arguments. Return false if pExpr +** references columns but not columns of tables found in pSrcList. +*/ +int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + struct SrcCount cnt; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprSrcCount; + w.xSelectCallback = selectSrcCount; + w.u.pSrcCount = &cnt; + cnt.pSrc = pSrcList; + cnt.iSrcInner = (pSrcList&&pSrcList->nSrc)?pSrcList->a[0].iCursor:0x7FFFFFFF; + cnt.nThis = 0; + cnt.nOther = 0; + sqlite3WalkExprList(&w, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); + } +#endif + return cnt.nThis>0 || cnt.nOther==0; +} + +/* +** This is a Walker expression node callback. +** +** For Expr nodes that contain pAggInfo pointers, make sure the AggInfo +** object that is referenced does not refer directly to the Expr. If +** it does, make a copy. This is done because the pExpr argument is +** subject to change. +** +** The copy is stored on pParse->pConstExpr with a register number of 0. +** This will cause the expression to be deleted automatically when the +** Parse object is destroyed, but the zero register number means that it +** will not generate any code in the preamble. +*/ +static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ + if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) + && pExpr->pAggInfo!=0 + ){ + AggInfo *pAggInfo = pExpr->pAggInfo; + int iAgg = pExpr->iAgg; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); + if( pExpr->op==TK_AGG_COLUMN ){ + assert( iAgg>=0 && iAggnColumn ); + if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aCol[iAgg].pCExpr = pExpr; + sqlite3ExprDeferredDelete(pParse, pExpr); + } + } + }else{ + assert( iAgg>=0 && iAggnFunc ); + if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aFunc[iAgg].pFExpr = pExpr; + sqlite3ExprDeferredDelete(pParse, pExpr); + } + } + } + } + return WRC_Continue; +} + +/* +** Initialize a Walker object so that will persist AggInfo entries referenced +** by the tree that is walked. +*/ +void sqlite3AggInfoPersistWalkerInit(Walker *pWalker, Parse *pParse){ + memset(pWalker, 0, sizeof(*pWalker)); + pWalker->pParse = pParse; + pWalker->xExprCallback = agginfoPersistExprCb; + pWalker->xSelectCallback = sqlite3SelectWalkNoop; +} + +/* +** Add a new element to the pAggInfo->aCol[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aCol = sqlite3ArrayAllocate( + db, + pInfo->aCol, + sizeof(pInfo->aCol[0]), + &pInfo->nColumn, + &i + ); + return i; +} + +/* +** Add a new element to the pAggInfo->aFunc[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aFunc = sqlite3ArrayAllocate( + db, + pInfo->aFunc, + sizeof(pInfo->aFunc[0]), + &pInfo->nFunc, + &i + ); + return i; +} + +/* +** This is the xExprCallback for a tree walker. It is used to +** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates +** for additional information. +*/ +static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ + int i; + NameContext *pNC = pWalker->u.pNC; + Parse *pParse = pNC->pParse; + SrcList *pSrcList = pNC->pSrcList; + AggInfo *pAggInfo = pNC->uNC.pAggInfo; + + assert( pNC->ncFlags & NC_UAggInfo ); + switch( pExpr->op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + /* Check to see if the column is in one of the tables in the FROM + ** clause of the aggregate query */ + if( ALWAYS(pSrcList!=0) ){ + SrcItem *pItem = pSrcList->a; + for(i=0; inSrc; i++, pItem++){ + struct AggInfo_col *pCol; + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + if( pExpr->iTable==pItem->iCursor ){ + /* If we reach this point, it means that pExpr refers to a table + ** that is in the FROM clause of the aggregate query. + ** + ** Make an entry for the column in pAggInfo->aCol[] if there + ** is not an entry there already. + */ + int k; + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable && + pCol->iColumn==pExpr->iColumn ){ + break; + } + } + if( (k>=pAggInfo->nColumn) + && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 + ){ + pCol = &pAggInfo->aCol[k]; + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iMem = ++pParse->nMem; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && + pE->iColumn==pExpr->iColumn ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } + } + /* There is now an entry for pExpr in pAggInfo->aCol[] (either + ** because it was there before or because we just created it). + ** Convert the pExpr to be a TK_AGG_COLUMN referring to that + ** pAggInfo->aCol[] entry. + */ + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + pExpr->op = TK_AGG_COLUMN; + pExpr->iAgg = (i16)k; + break; + } /* endif pExpr->iTable==pItem->iCursor */ + } /* end loop over pSrcList */ + } + return WRC_Prune; + } + case TK_AGG_FUNCTION: { + if( (pNC->ncFlags & NC_InAggFunc)==0 + && pWalker->walkerDepth==pExpr->op2 + ){ + /* Check to see if pExpr is a duplicate of another aggregate + ** function that is already in the pAggInfo structure + */ + struct AggInfo_func *pItem = pAggInfo->aFunc; + for(i=0; inFunc; i++, pItem++){ + if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ + break; + } + } + if( i>=pAggInfo->nFunc ){ + /* pExpr is original. Make a new entry in pAggInfo->aFunc[] + */ + u8 enc = ENC(pParse->db); + i = addAggInfoFunc(pParse->db, pAggInfo); + if( i>=0 ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pItem = &pAggInfo->aFunc[i]; + pItem->pFExpr = pExpr; + pItem->iMem = ++pParse->nMem; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pItem->pFunc = sqlite3FindFunction(pParse->db, + pExpr->u.zToken, + pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); + if( pExpr->flags & EP_Distinct ){ + pItem->iDistinct = pParse->nTab++; + }else{ + pItem->iDistinct = -1; + } + } + } + /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry + */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->iAgg = (i16)i; + pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; + } + } + } + return WRC_Continue; +} + +/* +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. +** +** This routine should only be called after the expression has been +** analyzed by sqlite3ResolveExprNames(). +*/ +void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ + Walker w; + w.xExprCallback = analyzeAggregate; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + w.walkerDepth = 0; + w.u.pNC = pNC; + w.pParse = 0; + assert( pNC->pSrcList!=0 ); + sqlite3WalkExpr(&w, pExpr); +} + +/* +** Call sqlite3ExprAnalyzeAggregates() for every expression in an +** expression list. Return the number of errors. +** +** If an error is found, the analysis is cut short. +*/ +void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ + struct ExprList_item *pItem; + int i; + if( pList ){ + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); + } + } +} + +/* +** Allocate a single new register for use to hold some intermediate result. +*/ +int sqlite3GetTempReg(Parse *pParse){ + if( pParse->nTempReg==0 ){ + return ++pParse->nMem; + } + return pParse->aTempReg[--pParse->nTempReg]; +} + +/* +** Deallocate a register, making available for reuse for some other +** purpose. +*/ +void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ + if( iReg ){ + sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0); + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = iReg; + } + } +} + +/* +** Allocate or deallocate a block of nReg consecutive registers. +*/ +int sqlite3GetTempRange(Parse *pParse, int nReg){ + int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); + i = pParse->iRangeReg; + n = pParse->nRangeReg; + if( nReg<=n ){ + pParse->iRangeReg += nReg; + pParse->nRangeReg -= nReg; + }else{ + i = pParse->nMem+1; + pParse->nMem += nReg; + } + return i; +} +void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } + sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0, 0); + if( nReg>pParse->nRangeReg ){ + pParse->nRangeReg = nReg; + pParse->iRangeReg = iReg; + } +} + +/* +** Mark all temporary registers as being unavailable for reuse. +** +** Always invoke this procedure after coding a subroutine or co-routine +** that might be invoked from other parts of the code, to ensure that +** the sub/co-routine does not use registers in common with the code that +** invokes the sub/co-routine. +*/ +void sqlite3ClearTempRegCache(Parse *pParse){ + pParse->nTempReg = 0; + pParse->nRangeReg = 0; +} + +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeReg > iFirst + && pParse->iRangeReg <= iLast + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ diff --git a/third_party/sqlite3/fault.c b/third_party/sqlite3/fault.c new file mode 100644 index 000000000..599e70025 --- /dev/null +++ b/third_party/sqlite3/fault.c @@ -0,0 +1,88 @@ +/* +** 2008 Jan 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to support the concept of "benign" +** malloc failures (when the xMalloc() or xRealloc() method of the +** sqlite3_mem_methods structure fails to allocate a block of memory +** and returns 0). +** +** Most malloc failures are non-benign. After they occur, SQLite +** abandons the current operation and returns an error code (usually +** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily +** fatal. For example, if a malloc fails while resizing a hash table, this +** is completely recoverable simply by not carrying out the resize. The +** hash table will continue to function normally. So a malloc failure +** during a hash table resize is a benign fault. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +#ifndef SQLITE_UNTESTABLE + +/* +** Global variables. +*/ +typedef struct BenignMallocHooks BenignMallocHooks; +static SQLITE_WSD struct BenignMallocHooks { + void (*xBenignBegin)(void); + void (*xBenignEnd)(void); +} sqlite3Hooks = { 0, 0 }; + +/* The "wsdHooks" macro will resolve to the appropriate BenignMallocHooks +** structure. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdHooks can refer directly +** to the "sqlite3Hooks" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdHooksInit \ + BenignMallocHooks *x = &GLOBAL(BenignMallocHooks,sqlite3Hooks) +# define wsdHooks x[0] +#else +# define wsdHooksInit +# define wsdHooks sqlite3Hooks +#endif + + +/* +** Register hooks to call when sqlite3BeginBenignMalloc() and +** sqlite3EndBenignMalloc() are called, respectively. +*/ +void sqlite3BenignMallocHooks( + void (*xBenignBegin)(void), + void (*xBenignEnd)(void) +){ + wsdHooksInit; + wsdHooks.xBenignBegin = xBenignBegin; + wsdHooks.xBenignEnd = xBenignEnd; +} + +/* +** This (sqlite3EndBenignMalloc()) is called by SQLite code to indicate that +** subsequent malloc failures are benign. A call to sqlite3EndBenignMalloc() +** indicates that subsequent malloc failures are non-benign. +*/ +void sqlite3BeginBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignBegin ){ + wsdHooks.xBenignBegin(); + } +} +void sqlite3EndBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignEnd ){ + wsdHooks.xBenignEnd(); + } +} + +#endif /* #ifndef SQLITE_UNTESTABLE */ diff --git a/third_party/sqlite3/fkey.c b/third_party/sqlite3/fkey.c new file mode 100644 index 000000000..bb304bcef --- /dev/null +++ b/third_party/sqlite3/fkey.c @@ -0,0 +1,1454 @@ +/* +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used by the compiler to add foreign key +** support to compiled SQL statements. +*/ +#include "third_party/sqlite3/sqliteInt.inc" + +/* clang-format off */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + +/* +** Deferred and Immediate FKs +** -------------------------- +** +** Foreign keys in SQLite come in two flavours: deferred and immediate. +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a +** deferred foreign key constraint is violated, no action is taken +** immediately. However if the application attempts to commit the +** transaction before fixing the constraint violation, the attempt fails. +** +** Deferred constraints are implemented using a simple counter associated +** with the database handle. The counter is set to zero each time a +** database transaction is opened. Each time a statement is executed +** that causes a foreign key violation, the counter is incremented. Each +** time a statement is executed that removes an existing violation from +** the database, the counter is decremented. When the transaction is +** committed, the commit fails if the current value of the counter is +** greater than zero. This scheme has two big drawbacks: +** +** * When a commit fails due to a deferred foreign key constraint, +** there is no way to tell which foreign constraint is not satisfied, +** or which row it is not satisfied for. +** +** * If the database contains foreign key violations when the +** transaction is opened, this may cause the mechanism to malfunction. +** +** Despite these problems, this approach is adopted as it seems simpler +** than the alternatives. +** +** INSERT operations: +** +** I.1) For each FK for which the table is the child table, search +** the parent table for a match. If none is found increment the +** constraint counter. +** +** I.2) For each FK for which the table is the parent table, +** search the child table for rows that correspond to the new +** row in the parent table. Decrement the counter for each row +** found (as the constraint is now satisfied). +** +** DELETE operations: +** +** D.1) For each FK for which the table is the child table, +** search the parent table for a row that corresponds to the +** deleted row in the child table. If such a row is not found, +** decrement the counter. +** +** D.2) For each FK for which the table is the parent table, search +** the child table for rows that correspond to the deleted row +** in the parent table. For each found increment the counter. +** +** UPDATE operations: +** +** An UPDATE command requires that all 4 steps above are taken, but only +** for FK constraints for which the affected columns are actually +** modified (values must be compared at runtime). +** +** Note that I.1 and D.1 are very similar operations, as are I.2 and D.2. +** This simplifies the implementation a bit. +** +** For the purposes of immediate FK constraints, the OR REPLACE conflict +** resolution is considered to delete rows before the new row is inserted. +** If a delete caused by OR REPLACE violates an FK constraint, an exception +** is thrown, even if the FK constraint would be satisfied after the new +** row is inserted. +** +** Immediate constraints are usually handled similarly. The only difference +** is that the counter used is stored as part of each individual statement +** object (struct Vdbe). If, after the statement has run, its immediate +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY +** and the statement transaction is rolled back. An exception is an INSERT +** statement that inserts a single row only (no triggers). In this case, +** instead of using a counter, an exception is thrown immediately if the +** INSERT violates a foreign key constraint. This is necessary as such +** an INSERT does not open a statement transaction. +** +** TODO: How should dropping a table be handled? How should renaming a +** table be handled? +** +** +** Query API Notes +** --------------- +** +** Before coding an UPDATE or DELETE row operation, the code-generator +** for those two operations needs to know whether or not the operation +** requires any FK processing and, if so, which columns of the original +** row are required by the FK processing VDBE code (i.e. if FKs were +** implemented using triggers, which of the old.* columns would be +** accessed). No information is required by the code-generator before +** coding an INSERT operation. The functions used by the UPDATE/DELETE +** generation code to query for this information are: +** +** sqlite3FkRequired() - Test to see if FK processing is required. +** sqlite3FkOldmask() - Query for the set of required old.* columns. +** +** +** Externally accessible module functions +** -------------------------------------- +** +** sqlite3FkCheck() - Check for foreign key violations. +** sqlite3FkActions() - Code triggers for ON UPDATE/ON DELETE actions. +** sqlite3FkDelete() - Delete an FKey structure. +*/ + +/* +** VDBE Calling Convention +** ----------------------- +** +** Example: +** +** For the following INSERT statement: +** +** CREATE TABLE t1(a, b INTEGER PRIMARY KEY, c); +** INSERT INTO t1 VALUES(1, 2, 3.1); +** +** Register (x): 2 (type integer) +** Register (x+1): 1 (type integer) +** Register (x+2): NULL (type NULL) +** Register (x+3): 3.1 (type real) +*/ + +/* +** A foreign key constraint requires that the key columns in the parent +** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. +** Given that pParent is the parent table for foreign key constraint pFKey, +** search the schema for a unique index on the parent key columns. +** +** If successful, zero is returned. If the parent key is an INTEGER PRIMARY +** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx +** is set to point to the unique index. +** +** If the parent key consists of a single column (the foreign key constraint +** is not a composite foreign key), output variable *paiCol is set to NULL. +** Otherwise, it is set to point to an allocated array of size N, where +** N is the number of columns in the parent key. The first element of the +** array is the index of the child table column that is mapped by the FK +** constraint to the parent table column stored in the left-most column +** of index *ppIdx. The second element of the array is the index of the +** child table column that corresponds to the second left-most column of +** *ppIdx, and so on. +** +** If the required index cannot be found, either because: +** +** 1) The named parent key columns do not exist, or +** +** 2) The named parent key columns do exist, but are not subject to a +** UNIQUE or PRIMARY KEY constraint, or +** +** 3) No parent key columns were provided explicitly as part of the +** foreign key definition, and the parent table does not have a +** PRIMARY KEY, or +** +** 4) No parent key columns were provided explicitly as part of the +** foreign key definition, and the PRIMARY KEY of the parent table +** consists of a different number of columns to the child key in +** the child table. +** +** then non-zero is returned, and a "foreign key mismatch" error loaded +** into pParse. If an OOM error occurs, non-zero is returned and the +** pParse->db->mallocFailed flag is set. +*/ +int sqlite3FkLocateIndex( + Parse *pParse, /* Parse context to store any error in */ + Table *pParent, /* Parent table of FK constraint pFKey */ + FKey *pFKey, /* Foreign key to find index for */ + Index **ppIdx, /* OUT: Unique index on parent table */ + int **paiCol /* OUT: Map of index columns in pFKey */ +){ + Index *pIdx = 0; /* Value to return via *ppIdx */ + int *aiCol = 0; /* Value to return via *paiCol */ + int nCol = pFKey->nCol; /* Number of columns in parent key */ + char *zKey = pFKey->aCol[0].zCol; /* Name of left-most parent key column */ + + /* The caller is responsible for zeroing output parameters. */ + assert( ppIdx && *ppIdx==0 ); + assert( !paiCol || *paiCol==0 ); + assert( pParse ); + + /* If this is a non-composite (single column) foreign key, check if it + ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx + ** and *paiCol set to zero and return early. + ** + ** Otherwise, for a composite foreign key (more than one column), allocate + ** space for the aiCol array (returned via output parameter *paiCol). + ** Non-composite foreign keys do not require the aiCol array. + */ + if( nCol==1 ){ + /* The FK maps to the IPK if any of the following are true: + ** + ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly + ** mapped to the primary key of table pParent, or + ** 2) The FK is explicitly mapped to a column declared as INTEGER + ** PRIMARY KEY. + */ + if( pParent->iPKey>=0 ){ + if( !zKey ) return 0; + if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zName, zKey) ) return 0; + } + }else if( paiCol ){ + assert( nCol>1 ); + aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); + if( !aiCol ) return 1; + *paiCol = aiCol; + } + + for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) && pIdx->pPartIdxWhere==0 ){ + /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number + ** of columns. If each indexed column corresponds to a foreign key + ** column of pFKey, then this index is a winner. */ + + if( zKey==0 ){ + /* If zKey is NULL, then this foreign key is implicitly mapped to + ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be + ** identified by the test. */ + if( IsPrimaryKeyIndex(pIdx) ){ + if( aiCol ){ + int i; + for(i=0; iaCol[i].iFrom; + } + break; + } + }else{ + /* If zKey is non-NULL, then this foreign key was declared to + ** map to an explicit list of columns in table pParent. Check if this + ** index matches those columns. Also, check that the index uses + ** the default collation sequences for each column. */ + int i, j; + for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ + const char *zDfltColl; /* Def. collation for column */ + char *zIdxCol; /* Name of indexed column */ + + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + + /* If the index uses a collation sequence that is different from + ** the default collation sequence for the column, this index is + ** unusable. Bail out early in this case. */ + zDfltColl = pParent->aCol[iCol].zColl; + if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; + if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; + + zIdxCol = pParent->aCol[iCol].zName; + for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; + break; + } + } + if( j==nCol ) break; + } + if( i==nCol ) break; /* pIdx is usable */ + } + } + } + + if( !pIdx ){ + if( !pParse->disableTriggers ){ + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); + } + sqlite3DbFree(pParse->db, aiCol); + return 1; + } + + *ppIdx = pIdx; + return 0; +} + +/* +** This function is called when a row is inserted into or deleted from the +** child table of foreign key constraint pFKey. If an SQL UPDATE is executed +** on the child table of pFKey, this function is invoked twice for each row +** affected - once to "delete" the old row, and then again to "insert" the +** new row. +** +** Each time it is called, this function generates VDBE code to locate the +** row in the parent table that corresponds to the row being inserted into +** or deleted from the child table. If the parent row can be found, no +** special action is taken. Otherwise, if the parent row can *not* be +** found in the parent table: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** INSERT immediate Increment the "immediate constraint counter". +** +** DELETE immediate Decrement the "immediate constraint counter". +** +** INSERT deferred Increment the "deferred constraint counter". +** +** DELETE deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.1" and "D.1". +*/ +static void fkLookupParent( + Parse *pParse, /* Parse context */ + int iDb, /* Index of database housing pTab */ + Table *pTab, /* Parent table of FK pFKey */ + Index *pIdx, /* Unique index on parent key columns in pTab */ + FKey *pFKey, /* Foreign key constraint */ + int *aiCol, /* Map from parent key columns to child table columns */ + int regData, /* Address of array containing child table row */ + int nIncr, /* Increment constraint counter by this */ + int isIgnore /* If true, pretend pTab contains all NULL values */ +){ + int i; /* Iterator variable */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ + int iCur = pParse->nTab - 1; /* Cursor number to use */ + int iOk = sqlite3VdbeMakeLabel(pParse); /* jump here if parent key found */ + + sqlite3VdbeVerifyAbortable(v, + (!pFKey->isDeferred + && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite) ? OE_Abort : OE_Ignore); + + /* If nIncr is less than zero, then check at runtime if there are any + ** outstanding constraints to resolve. If there are not, there is no need + ** to check if deleting this row resolves any outstanding violations. + ** + ** Check if any of the key columns in the child table row are NULL. If + ** any are, then the constraint is considered satisfied. No need to + ** search for a matching row in the parent table. */ + if( nIncr<0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + VdbeCoverage(v); + } + for(i=0; inCol; i++){ + int iReg = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + regData + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); + } + + if( isIgnore==0 ){ + if( pIdx==0 ){ + /* If pIdx is NULL, then the parent key is the INTEGER PRIMARY KEY + ** column of the parent table (table pTab). */ + int iMustBeInt; /* Address of MustBeInt instruction */ + int regTemp = sqlite3GetTempReg(pParse); + + /* Invoke MustBeInt to coerce the child key value to an integer (i.e. + ** apply the affinity of the parent key). If this fails, then there + ** is no matching parent key. Before using MustBeInt, make a copy of + ** the value. Otherwise, the value inserted into the child key column + ** will have INTEGER affinity applied to it, which may not be correct. */ + sqlite3VdbeAddOp2(v, OP_SCopy, + sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[0])+1+regData, regTemp); + iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + VdbeCoverage(v); + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + } + + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); + sqlite3VdbeGoto(v, iOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + sqlite3VdbeJumpHere(v, iMustBeInt); + sqlite3ReleaseTempReg(pParse, regTemp); + }else{ + int nCol = pFKey->nCol; + int regTemp = sqlite3GetTempRange(pParse, nCol); + int regRec = sqlite3GetTempReg(pParse); + + sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + for(i=0; ipFrom, aiCol[i])+1+regData, + regTemp+i); + } + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; + for(i=0; ipFrom,aiCol[i]) + +1+regData; + int iParent = 1+regData; + iParent += sqlite3TableColumnToStorage(pIdx->pTable, + pIdx->aiColumn[i]); + assert( pIdx->aiColumn[i]>=0 ); + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } + sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + } + sqlite3VdbeGoto(v, iOk); + } + + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); + + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempRange(pParse, regTemp, nCol); + } + } + + if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite + ){ + /* Special case: If this is an INSERT statement that will insert exactly + ** one row into the table, raise a constraint immediately instead of + ** incrementing a counter. This is necessary as the VM code is being + ** generated for will not open a statement transaction. */ + assert( nIncr==1 ); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + }else{ + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + } + + sqlite3VdbeResolveLabel(v, iOk); + sqlite3VdbeAddOp1(v, OP_Close, iCur); +} + + +/* +** Return an Expr object that refers to a memory register corresponding +** to column iCol of table pTab. +** +** regBase is the first of an array of register that contains the data +** for pTab. regBase itself holds the rowid. regBase+1 holds the first +** column. regBase+2 holds the second column, and so forth. +*/ +static Expr *exprTableRegister( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* The table whose content is at r[regBase]... */ + int regBase, /* Contents of table pTab */ + i16 iCol /* Which column of pTab is desired */ +){ + Expr *pExpr; + Column *pCol; + const char *zColl; + sqlite3 *db = pParse->db; + + pExpr = sqlite3Expr(db, TK_REGISTER, 0); + if( pExpr ){ + if( iCol>=0 && iCol!=pTab->iPKey ){ + pCol = &pTab->aCol[iCol]; + pExpr->iTable = regBase + sqlite3TableColumnToStorage(pTab,iCol) + 1; + pExpr->affExpr = pCol->affinity; + zColl = pCol->zColl; + if( zColl==0 ) zColl = db->pDfltColl->zName; + pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); + }else{ + pExpr->iTable = regBase; + pExpr->affExpr = SQLITE_AFF_INTEGER; + } + } + return pExpr; +} + +/* +** Return an Expr object that refers to column iCol of table pTab which +** has cursor iCur. +*/ +static Expr *exprTableColumn( + sqlite3 *db, /* The database connection */ + Table *pTab, /* The table whose column is desired */ + int iCursor, /* The open cursor on the table */ + i16 iCol /* The column that is wanted */ +){ + Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); + if( pExpr ){ + pExpr->y.pTab = pTab; + pExpr->iTable = iCursor; + pExpr->iColumn = iCol; + } + return pExpr; +} + +/* +** This function is called to generate code executed when a row is deleted +** from the parent table of foreign key constraint pFKey and, if pFKey is +** deferred, when a row is inserted into the same table. When generating +** code for an SQL UPDATE operation, this function may be called twice - +** once to "delete" the old row and once to "insert" the new row. +** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** +** The code generated by this function scans through the rows in the child +** table that correspond to the parent table row being deleted or inserted. +** For each child row found, one of the following actions is taken: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** DELETE immediate Increment the "immediate constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT immediate Decrement the "immediate constraint counter". +** +** DELETE deferred Increment the "deferred constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.2" and "D.2". +*/ +static void fkScanChildren( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* The child table to be scanned */ + Table *pTab, /* The parent table */ + Index *pIdx, /* Index on parent covering the foreign key */ + FKey *pFKey, /* The foreign key linking pSrc to pTab */ + int *aiCol, /* Map from pIdx cols to child table cols */ + int regData, /* Parent row data starts here */ + int nIncr /* Amount to increment deferred counter by */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int i; /* Iterator variable */ + Expr *pWhere = 0; /* WHERE clause to scan with */ + NameContext sNameContext; /* Context used to resolve WHERE clause */ + WhereInfo *pWInfo; /* Context used by sqlite3WhereXXX() */ + int iFkIfZero = 0; /* Address of OP_FkIfZero */ + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( pIdx==0 || pIdx->pTable==pTab ); + assert( pIdx==0 || pIdx->nKeyCol==pFKey->nCol ); + assert( pIdx!=0 || pFKey->nCol==1 ); + assert( pIdx!=0 || HasRowid(pTab) ); + + if( nIncr<0 ){ + iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); + VdbeCoverage(v); + } + + /* Create an Expr object representing an SQL expression like: + ** + ** = AND = ... + ** + ** The collation sequence used for the comparison should be that of + ** the parent key columns. The affinity of the parent key column should + ** be applied to each child key value before the comparison takes place. + */ + for(i=0; inCol; i++){ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + Expr *pEq; /* Expression (pLeft = pRight) */ + i16 iCol; /* Index of column in child table */ + const char *zCol; /* Name of column in child table */ + + iCol = pIdx ? pIdx->aiColumn[i] : -1; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iCol>=0 ); + zCol = pFKey->pFrom->aCol[iCol].zName; + pRight = sqlite3Expr(db, TK_ID, zCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); + } + + /* If the child table is the same as the parent table, then add terms + ** to the WHERE clause that prevent this entry from being scanned. + ** The added WHERE clause terms are like this: + ** + ** $current_rowid!=rowid + ** NOT( $current_a==a AND $current_b==b AND ... ) + ** + ** The first form is used for rowid tables. The second form is used + ** for WITHOUT ROWID tables. In the second form, the *parent* key is + ** (a,b,...). Either the parent or primary key could be used to + ** uniquely identify the current row, but the parent key is more convenient + ** as the required values have already been loaded into registers + ** by the caller. + */ + if( pTab==pFKey->pFrom && nIncr>0 ){ + Expr *pNe; /* Expression (pLeft != pRight) */ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + if( HasRowid(pTab) ){ + pLeft = exprTableRegister(pParse, pTab, regData, -1); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight); + }else{ + Expr *pEq, *pAll = 0; + assert( pIdx!=0 ); + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + pRight = sqlite3Expr(db, TK_ID, pTab->aCol[iCol].zName); + pEq = sqlite3PExpr(pParse, TK_IS, pLeft, pRight); + pAll = sqlite3ExprAnd(pParse, pAll, pEq); + } + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); + } + pWhere = sqlite3ExprAnd(pParse, pWhere, pNe); + } + + /* Resolve the references in the WHERE clause. */ + memset(&sNameContext, 0, sizeof(NameContext)); + sNameContext.pSrcList = pSrc; + sNameContext.pParse = pParse; + sqlite3ResolveExprNames(&sNameContext, pWhere); + + /* Create VDBE to loop through the entries in pSrc that match the WHERE + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ + if( pParse->nErr==0 ){ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + if( pWInfo ){ + sqlite3WhereEnd(pWInfo); + } + } + + /* Clean up the WHERE clause constructed above. */ + sqlite3ExprDelete(db, pWhere); + if( iFkIfZero ){ + sqlite3VdbeJumpHereOrPopInst(v, iFkIfZero); + } +} + +/* +** This function returns a linked list of FKey objects (connected by +** FKey.pNextTo) holding all children of table pTab. For example, +** given the following schema: +** +** CREATE TABLE t1(a PRIMARY KEY); +** CREATE TABLE t2(b REFERENCES t1(a); +** +** Calling this function with table "t1" as an argument returns a pointer +** to the FKey structure representing the foreign key constraint on table +** "t2". Calling this function with "t2" as the argument would return a +** NULL pointer (as there are no FK constraints for which t2 is the parent +** table). +*/ +FKey *sqlite3FkReferences(Table *pTab){ + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); +} + +/* +** The second argument is a Trigger structure allocated by the +** fkActionTrigger() routine. This function deletes the Trigger structure +** and all of its sub-components. +** +** The Trigger structure or any of its sub-components may be allocated from +** the lookaside buffer belonging to database handle dbMem. +*/ +static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ + if( p ){ + TriggerStep *pStep = p->step_list; + sqlite3ExprDelete(dbMem, pStep->pWhere); + sqlite3ExprListDelete(dbMem, pStep->pExprList); + sqlite3SelectDelete(dbMem, pStep->pSelect); + sqlite3ExprDelete(dbMem, p->pWhen); + sqlite3DbFree(dbMem, p); + } +} + +/* +** This function is called to generate code that runs when table pTab is +** being dropped from the database. The SrcList passed as the second argument +** to this function contains a single entry guaranteed to resolve to +** table pTab. +** +** Normally, no code is required. However, if either +** +** (a) The table is the parent table of a FK constraint, or +** (b) The table is the child table of a deferred FK constraint and it is +** determined at runtime that there are outstanding deferred FK +** constraint violations in the database, +** +** then the equivalent of "DELETE FROM " is executed before dropping +** the table from the database. Triggers are disabled while running this +** DELETE, but foreign key actions are not. +*/ +void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ + sqlite3 *db = pParse->db; + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) ){ + int iSkip = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( v ); /* VDBE has already been allocated */ + assert( pTab->pSelect==0 ); /* Not a view */ + if( sqlite3FkReferences(pTab)==0 ){ + /* Search for a deferred foreign key constraint for which this table + ** is the child table. If one cannot be found, return without + ** generating any VDBE code. If one can be found, then jump over + ** the entire DELETE if there are no outstanding deferred constraints + ** when this statement is run. */ + FKey *p; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; + } + if( !p ) return; + iSkip = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); + } + + pParse->disableTriggers = 1; + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0); + pParse->disableTriggers = 0; + + /* If the DELETE has generated immediate foreign key constraint + ** violations, halt the VDBE and return an error at this point, before + ** any modifications to the schema are made. This is because statement + ** transactions are not able to rollback schema changes. + ** + ** If the SQLITE_DeferFKs flag is set, then this is not required, as + ** the statement transaction will not be rolled back even if FK + ** constraints are violated. + */ + if( (db->flags & SQLITE_DeferFKs)==0 ){ + sqlite3VdbeVerifyAbortable(v, OE_Abort); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + } + + if( iSkip ){ + sqlite3VdbeResolveLabel(v, iSkip); + } + } +} + + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the child table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** child key for FK constraint *p are modified. +*/ +static int fkChildIsModified( + Table *pTab, /* Table being updated */ + FKey *p, /* Foreign key for which pTab is the child */ + int *aChange, /* Array indicating modified columns */ + int bChngRowid /* True if rowid is modified by this update */ +){ + int i; + for(i=0; inCol; i++){ + int iChildKey = p->aCol[i].iFrom; + if( aChange[iChildKey]>=0 ) return 1; + if( iChildKey==pTab->iPKey && bChngRowid ) return 1; + } + return 0; +} + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the parent table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** parent key for FK constraint *p are modified. +*/ +static int fkParentIsModified( + Table *pTab, + FKey *p, + int *aChange, + int bChngRowid +){ + int i; + for(i=0; inCol; i++){ + char *zKey = p->aCol[i].zCol; + int iKey; + for(iKey=0; iKeynCol; iKey++){ + if( aChange[iKey]>=0 || (iKey==pTab->iPKey && bChngRowid) ){ + Column *pCol = &pTab->aCol[iKey]; + if( zKey ){ + if( 0==sqlite3StrICmp(pCol->zName, zKey) ) return 1; + }else if( pCol->colFlags & COLFLAG_PRIMKEY ){ + return 1; + } + } + } + } + return 0; +} + +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + +/* +** This function is called when inserting, deleting or updating a row of +** table pTab to generate VDBE code to perform foreign key constraint +** processing for the operation. +** +** For a DELETE operation, parameter regOld is passed the index of the +** first register in an array of (pTab->nCol+1) registers containing the +** rowid of the row being deleted, followed by each of the column values +** of the row being deleted, from left to right. Parameter regNew is passed +** zero in this case. +** +** For an INSERT operation, regOld is passed zero and regNew is passed the +** first register of an array of (pTab->nCol+1) registers containing the new +** row data. +** +** For an UPDATE operation, this function is called twice. Once before +** the original record is deleted from the table using the calling convention +** described for DELETE. Then again after the original record is deleted +** but before the new record is inserted using the INSERT convention. +*/ +void sqlite3FkCheck( + Parse *pParse, /* Parse context */ + Table *pTab, /* Row is being deleted from this table */ + int regOld, /* Previous row data is stored here */ + int regNew, /* New row data is stored here */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + FKey *pFKey; /* Used to iterate through FKs */ + int iDb; /* Index of database containing pTab */ + const char *zDb; /* Name of database containing pTab */ + int isIgnoreErrors = pParse->disableTriggers; + + /* Exactly one of regOld and regNew should be non-zero. */ + assert( (regOld==0)!=(regNew==0) ); + + /* If foreign-keys are disabled, this function is a no-op. */ + if( (db->flags&SQLITE_ForeignKeys)==0 ) return; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + + /* Loop through all the foreign key constraints for which pTab is the + ** child table (the table that the foreign key definition is part of). */ + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + Table *pTo; /* Parent table of foreign key pFKey */ + Index *pIdx = 0; /* Index on key columns in pTo */ + int *aiFree = 0; + int *aiCol; + int iCol; + int i; + int bIgnore = 0; + + if( aChange + && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 + && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 + ){ + continue; + } + + /* Find the parent table of this foreign key. Also find a unique index + ** on the parent key columns in the parent table. If either of these + ** schema items cannot be located, set an error in pParse and return + ** early. */ + if( pParse->disableTriggers ){ + pTo = sqlite3FindTable(db, pFKey->zTo, zDb); + }else{ + pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); + } + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); + if( !isIgnoreErrors || db->mallocFailed ) return; + if( pTo==0 ){ + /* If isIgnoreErrors is true, then a table is being dropped. In this + ** case SQLite runs a "DELETE FROM xxx" on the table being dropped + ** before actually dropping it in order to check FK constraints. + ** If the parent table of an FK constraint on the current table is + ** missing, behave as if it is empty. i.e. decrement the relevant + ** FK counter for each row of the current table with non-NULL keys. + */ + Vdbe *v = sqlite3GetVdbe(pParse); + int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; + for(i=0; inCol; i++){ + int iFromCol, iReg; + iFromCol = pFKey->aCol[i].iFrom; + iReg = sqlite3TableColumnToStorage(pFKey->pFrom,iFromCol) + regOld+1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); + } + continue; + } + assert( pFKey->nCol==1 || (aiFree && pIdx) ); + + if( aiFree ){ + aiCol = aiFree; + }else{ + iCol = pFKey->aCol[0].iFrom; + aiCol = &iCol; + } + for(i=0; inCol; i++){ + if( aiCol[i]==pTab->iPKey ){ + aiCol[i] = -1; + } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Request permission to read the parent key columns. If the + ** authorization callback returns SQLITE_IGNORE, behave as if any + ** values read from the parent table are NULL. */ + if( db->xAuth ){ + int rcauth; + char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; + rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); + bIgnore = (rcauth==SQLITE_IGNORE); + } +#endif + } + + /* Take a shared-cache advisory read-lock on the parent table. Allocate + ** a cursor to use to search the unique index on the parent key columns + ** in the parent table. */ + sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); + pParse->nTab++; + + if( regOld!=0 ){ + /* A row is being removed from the child table. Search for the parent. + ** If the parent does not exist, removing the child row resolves an + ** outstanding foreign key constraint violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); + } + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ + /* A row is being added to the child table. If a parent row cannot + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As all child key + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); + } + + sqlite3DbFree(db, aiFree); + } + + /* Loop through all the foreign key constraints that refer to this table. + ** (the "child" constraints) */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + Index *pIdx = 0; /* Foreign key index for pFKey */ + SrcList *pSrc; + int *aiCol = 0; + + if( aChange && fkParentIsModified(pTab, pFKey, aChange, bChngRowid)==0 ){ + continue; + } + + if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel && !pParse->isMultiWrite + ){ + assert( regOld==0 && regNew!=0 ); + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ + continue; + } + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( !isIgnoreErrors || db->mallocFailed ) return; + continue; + } + assert( aiCol || pFKey->nCol==1 ); + + /* Create a SrcList structure containing the child table. We need the + ** child table as a SrcList for sqlite3WhereBegin() */ + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + SrcItem *pItem = pSrc->a; + pItem->pTab = pFKey->pFrom; + pItem->zName = pFKey->pFrom->zName; + pItem->pTab->nTabRef++; + pItem->iCursor = pParse->nTab++; + + if( regNew!=0 ){ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); + } + if( regOld!=0 ){ + int eAction = pFKey->aAction[aChange!=0]; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } + } + pItem->zName = 0; + sqlite3SrcListDelete(db, pSrc); + } + sqlite3DbFree(db, aiCol); + } +} + +#define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. +*/ +u32 sqlite3FkOldmask( + Parse *pParse, /* Parse context */ + Table *pTab /* Table being modified */ +){ + u32 mask = 0; + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *p; + int i; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); + } + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Index *pIdx = 0; + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); + if( pIdx ){ + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } + } + } + } + return mask; +} + + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. If the operation is a DELETE, then +** parameter aChange is passed a NULL value. For an UPDATE, aChange points +** to an array of size N, where N is the number of columns in table pTab. +** If the i'th column is not modified by the UPDATE, then the corresponding +** entry in the aChange[] array is set to -1. If the column is modified, +** the value is 0 or greater. Parameter chngRowid is set to true if the +** UPDATE statement modifies the rowid fields of the table. +** +** If any foreign key processing will be required, this function returns +** non-zero. If there is no foreign key related processing, this function +** returns zero. +** +** For an UPDATE, this function returns 2 if: +** +** * There are any FKs for which pTab is the child and the parent table +** and any FK processing at all is required (even of a different FK), or +** +** * the UPDATE modifies one or more parent keys for which the action is +** not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL). +** +** Or, assuming some other foreign key processing is required, 1. +*/ +int sqlite3FkRequired( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being modified */ + int *aChange, /* Non-NULL for UPDATE operations */ + int chngRowid /* True for UPDATE that affects rowid */ +){ + int eRet = 1; /* Value to return if bHaveFK is true */ + int bHaveFK = 0; /* If FK processing is required */ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + if( !aChange ){ + /* A DELETE operation. Foreign key processing is required if the + ** table in question is either the child or parent table for any + ** foreign key constraint. */ + bHaveFK = (sqlite3FkReferences(pTab) || pTab->pFKey); + }else{ + /* This is an UPDATE. Foreign key processing is only required if the + ** operation modifies one or more child or parent key columns. */ + FKey *p; + + /* Check if any child key columns are being modified. */ + for(p=pTab->pFKey; p; p=p->pNextFrom){ + if( fkChildIsModified(pTab, p, aChange, chngRowid) ){ + if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) eRet = 2; + bHaveFK = 1; + } + } + + /* Check if any parent key columns are being modified. */ + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ + if( p->aAction[1]!=OE_None ) return 2; + bHaveFK = 1; + } + } + } + } + return bHaveFK ? eRet : 0; +} + +/* +** This function is called when an UPDATE or DELETE operation is being +** compiled on table pTab, which is the parent table of foreign-key pFKey. +** If the current operation is an UPDATE, then the pChanges parameter is +** passed a pointer to the list of columns being modified. If it is a +** DELETE, pChanges is passed a NULL pointer. +** +** It returns a pointer to a Trigger structure containing a trigger +** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. +** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is +** returned (these actions require no special handling by the triggers +** sub-system, code for them is created by fkScanChildren()). +** +** For example, if pFKey is the foreign key and pTab is table "p" in +** the following schema: +** +** CREATE TABLE p(pk PRIMARY KEY); +** CREATE TABLE c(ck REFERENCES p ON DELETE CASCADE); +** +** then the returned trigger structure is equivalent to: +** +** CREATE TRIGGER ... DELETE ON p BEGIN +** DELETE FROM c WHERE ck = old.pk; +** END; +** +** The returned pointer is cached as part of the foreign key object. It +** is eventually freed along with the rest of the foreign key object by +** sqlite3FkDelete(). +*/ +static Trigger *fkActionTrigger( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + FKey *pFKey, /* Foreign key to get action for */ + ExprList *pChanges /* Change-list for UPDATE, NULL for DELETE */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int action; /* One of OE_None, OE_Cascade etc. */ + Trigger *pTrigger; /* Trigger definition to return */ + int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ + + action = pFKey->aAction[iAction]; + if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ + return 0; + } + pTrigger = pFKey->apTrigger[iAction]; + + if( action!=OE_None && !pTrigger ){ + char const *zFrom; /* Name of child table */ + int nFrom; /* Length in bytes of zFrom */ + Index *pIdx = 0; /* Parent key index for this FK */ + int *aiCol = 0; /* child table cols -> parent key cols */ + TriggerStep *pStep = 0; /* First (only) step of trigger program */ + Expr *pWhere = 0; /* WHERE clause of trigger step */ + ExprList *pList = 0; /* Changes list if ON UPDATE CASCADE */ + Select *pSelect = 0; /* If RESTRICT, "SELECT RAISE(...)" */ + int i; /* Iterator variable */ + Expr *pWhen = 0; /* WHEN clause for the trigger */ + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + assert( aiCol || pFKey->nCol==1 ); + + for(i=0; inCol; i++){ + Token tOld = { "old", 3 }; /* Literal "old" token */ + Token tNew = { "new", 3 }; /* Literal "new" token */ + Token tFromCol; /* Name of column in child table */ + Token tToCol; /* Name of column in parent table */ + int iFromCol; /* Idx of column in child table */ + Expr *pEq; /* tFromCol = OLD.tToCol */ + + iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iFromCol>=0 ); + assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); + sqlite3TokenInit(&tToCol, + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); + + /* Create the expression "OLD.zToCol = zFromCol". It is important + ** that the "OLD.zToCol" term is on the LHS of the = operator, so + ** that the affinity and collation sequence associated with the + ** parent table are used for the comparison. */ + pEq = sqlite3PExpr(pParse, TK_EQ, + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), + sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) + ); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); + + /* For ON UPDATE, construct the next term of the WHEN clause. + ** The final WHEN clause will be like this: + ** + ** WHEN NOT(old.col1 IS new.col1 AND ... AND old.colN IS new.colN) + */ + if( pChanges ){ + pEq = sqlite3PExpr(pParse, TK_IS, + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) + ); + pWhen = sqlite3ExprAnd(pParse, pWhen, pEq); + } + + if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ + Expr *pNew; + if( action==OE_Cascade ){ + pNew = sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)); + }else if( action==OE_SetDflt ){ + Column *pCol = pFKey->pFrom->aCol + iFromCol; + Expr *pDflt; + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + pDflt = 0; + }else{ + pDflt = pCol->pDflt; + } + if( pDflt ){ + pNew = sqlite3ExprDup(db, pDflt, 0); + }else{ + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + }else{ + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + pList = sqlite3ExprListAppend(pParse, pList, pNew); + sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); + } + } + sqlite3DbFree(db, aiCol); + + zFrom = pFKey->pFrom->zName; + nFrom = sqlite3Strlen30(zFrom); + + if( action==OE_Restrict ){ + Token tFrom; + Expr *pRaise; + + tFrom.z = zFrom; + tFrom.n = nFrom; + pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + if( pRaise ){ + pRaise->affExpr = OE_Abort; + } + pSelect = sqlite3SelectNew(pParse, + sqlite3ExprListAppend(pParse, 0, pRaise), + sqlite3SrcListAppend(pParse, 0, &tFrom, 0), + pWhere, + 0, 0, 0, 0, 0 + ); + pWhere = 0; + } + + /* Disable lookaside memory allocation */ + DisableLookaside; + + pTrigger = (Trigger *)sqlite3DbMallocZero(db, + sizeof(Trigger) + /* struct Trigger */ + sizeof(TriggerStep) + /* Single step in trigger program */ + nFrom + 1 /* Space for pStep->zTarget */ + ); + if( pTrigger ){ + pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; + pStep->zTarget = (char *)&pStep[1]; + memcpy((char *)pStep->zTarget, zFrom, nFrom); + + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); + pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( pWhen ){ + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0); + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + } + + /* Re-enable the lookaside buffer, if it was disabled earlier. */ + EnableLookaside; + + sqlite3ExprDelete(db, pWhere); + sqlite3ExprDelete(db, pWhen); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed==1 ){ + fkTriggerDelete(db, pTrigger); + return 0; + } + assert( pStep!=0 ); + assert( pTrigger!=0 ); + + switch( action ){ + case OE_Restrict: + pStep->op = TK_SELECT; + break; + case OE_Cascade: + if( !pChanges ){ + pStep->op = TK_DELETE; + break; + } + /* no break */ deliberate_fall_through + default: + pStep->op = TK_UPDATE; + } + pStep->pTrig = pTrigger; + pTrigger->pSchema = pTab->pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pFKey->apTrigger[iAction] = pTrigger; + pTrigger->op = (pChanges ? TK_UPDATE : TK_DELETE); + } + + return pTrigger; +} + +/* +** This function is called when deleting or updating a row to implement +** any required CASCADE, SET NULL or SET DEFAULT actions. +*/ +void sqlite3FkActions( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ + int regOld, /* Address of array containing old row */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + /* If foreign-key support is enabled, iterate through all FKs that + ** refer to table pTab. If there is an action associated with the FK + ** for this operation (either update or delete), invoke the associated + ** trigger sub-program. */ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *pFKey; /* Iterator variable */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + if( aChange==0 || fkParentIsModified(pTab, pFKey, aChange, bChngRowid) ){ + Trigger *pAct = fkActionTrigger(pParse, pTab, pFKey, pChanges); + if( pAct ){ + sqlite3CodeRowTriggerDirect(pParse, pAct, pTab, regOld, OE_Abort, 0); + } + } + } + } +} + +#endif /* ifndef SQLITE_OMIT_TRIGGER */ + +/* +** Free all memory associated with foreign key definitions attached to +** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash +** hash table. +*/ +void sqlite3FkDelete(sqlite3 *db, Table *pTab){ + FKey *pFKey; /* Iterator variable */ + FKey *pNext; /* Copy of pFKey->pNextFrom */ + + assert( db==0 || IsVirtual(pTab) + || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ + + /* Remove the FK from the fkeyHash hash table. */ + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } + } + + /* EV: R-30323-21917 Each foreign key constraint in SQLite is + ** classified as either immediate or deferred. + */ + assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif + + pNext = pFKey->pNextFrom; + sqlite3DbFree(db, pFKey); + } +} +#endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ diff --git a/third_party/sqlite3/fts3.c b/third_party/sqlite3/fts3.c new file mode 100644 index 000000000..f1dae05ec --- /dev/null +++ b/third_party/sqlite3/fts3.c @@ -0,0 +1,6096 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ +/* clang-format off */ + +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/fts3.inc" +#ifndef SQLITE_CORE +#include "third_party/sqlite3/sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#endif + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +int sqlite3Fts3Always(int b) { assert( b ); return b; } +int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts3 code are activated - conditions that are +** only true if it is guaranteed that the fts3 database is not corrupt. +*/ +int sqlite3_fts3_may_be_corrupt = 1; + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0] and +** not extending past pEnd[-1]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarintBounded( + const char *pBuf, + const char *pEnd, + sqlite_int64 *v +){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + const unsigned char *pX = (const unsigned char*)pEnd; + u64 b = 0; + int shift; + for(shift=0; shift<=63; shift+=7){ + u64 c = p=0 ); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Write an error message into *pzErr +*/ +void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + fts3DbExec(&rc, db, + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';" + "DROP TABLE IF EXISTS %Q.'%q_docsize';" + "DROP TABLE IF EXISTS %Q.'%q_stat';" + "%s DROP TABLE IF EXISTS %Q.'%q_content';", + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + (p->zContentTbl ? "--" : ""), zDb,p->zName + ); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + sqlite3_int64 nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc64(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + int nInt = 0; /* Output value */ + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; + } + *pnOut = nInt; + *pp += nByte; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} + +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3_int64 nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc64(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc64(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iCol0 ){ + memcpy(zCsr, z, n); + } + zCsr[n] = '\0'; + sqlite3Fts3Dequote(zCsr); + p->azColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + +#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST) + p->nMergeCount = FTS3_MERGE_COUNT; +#endif + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + if( p->bLock ){ + return SQLITE_ERROR; + } + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + +/* +** Free all resources currently held by the cursor passed as the only +** argument. +*/ +static void fts3ClearCursor(Fts3Cursor *pCsr){ + fts3CursorFinalizeStmt(pCsr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + fts3ClearCursor(pCsr); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + p->bLock++; + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; + } + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + pTab->bLock++; + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + pTab->bLock--; + return SQLITE_OK; + }else{ + pTab->bLock--; + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + i64 nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + int nBuffer = 0; /* Total term size */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrnBuffer ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + } + isFirstTerm = 0; + zCsr += fts3GetVarint32(zCsr, &nSuffix); + + assert( nPrefix>=0 && nSuffix>=0 ); + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( (i64)nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = ((i64)nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + int iNewHeight = 0; + fts3GetVarint32(zBlob, &iNewHeight); + if( iNewHeight>=iHeight ){ + rc = FTS_CORRUPT_VTAB; + }else{ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert_fts3_nc( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. +*/ +#define POSITION_LIST_END LARGEST_INT64 + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + int iVal; + *pp += fts3GetVarint32((*pp), &iVal); + *pi += iVal; + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static int fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ){ + fts3GetVarint32(&p1[1], &iCol1); + if( iCol1==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p1==POS_END ) iCol1 = 0x7fffffff; + else iCol1 = 0; + + if( *p2==POS_COLUMN ){ + fts3GetVarint32(&p2[1], &iCol2); + if( iCol2==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p2==POS_END ) iCol2 = 0x7fffffff; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + if( i1<2 || i2<2 ){ + break; + } + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert_fts3_nc( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + if( iPos1<0 || iPos2<0 ) break; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); + if( bDescIdx ){ + *pVal = (i64)((u64)*pVal - iVal); + }else{ + *pVal = (i64)((u64)*pVal + iVal); + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_uint64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); + iWrite = (u64)iVal - (u64)*piPrev; + }else{ + assert_fts3_nc( *piPrev>=iVal ); + iWrite = (u64)*piPrev - (u64)iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + int rc = SQLITE_OK; + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + rc = fts3PoslistMerge(&p, &p1, &p2); + if( rc ) break; + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + p = aOut = 0; + }else{ + assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 ); + memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING); + } + *paOut = aOut; + *pnOut = (int)(p-aOut); + return rc; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut; + + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm && zRoot ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + Fts3Table *pTab = (Fts3Table*)pCursor->pVtab; + pTab->bLock++; + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + pTab->bLock--; + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + if( p->bLock ){ + return SQLITE_ERROR; + } + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + fts3ClearCursor(pCsr); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + p->bLock++; + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; + if( pCsr->isEof ){ + fts3ClearCursor(pCsr); + pCsr->isEof = 1; + } + return pCsr->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); + break; + + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; + + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* no break */ deliberate_fall_through + } + + default: + /* A user column. Or, if this is a full-table scan, possibly the + ** language-id column. Seek the cursor. */ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + break; + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; + return fts3SetHasStat(p); +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + int rc; + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); + if( (*ppCsr)!=0 ){ + rc = SQLITE_OK; + }else{ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + /* no break */ deliberate_fall_through + case 5: iCol = sqlite3_value_int(apVal[4]); + /* no break */ deliberate_fall_through + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + /* no break */ deliberate_fall_through + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + /* no break */ deliberate_fall_through + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts3ShadowName(const char *zName){ + static const char *azName[] = { + "content", "docsize", "segdir", "segments", "stat", + }; + unsigned int i; + for(i=0; ieType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +*/ +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */ + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert_fts3_nc( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + assert( pDL->aAll!=0 || pIter==0 ); + }else{ + pIter = pDL->aAll; + } + + if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + int rc = SQLITE_OK; + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + testcase( a==0 ); /* If %_stat.value set to X'' */ + if( a ){ + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + } + + *pnPage = pCsr->nRowAvg; + return rc; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc64( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert_fts3_nc( nNew<=pPhrase->doclist.nList && nNew>0 ); + if( nNew>=0 && nNew<=pPhrase->doclist.nList ){ + assert( pPhrase->doclist.pList[nNew]=='\0' ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + } + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + pRight->bEof = pLeft->bEof = 1; + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || iCmp>0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + sqlite3_int64 nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc64(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = ( + pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId + && pExpr->pPhrase->doclist.nList>0 + ); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + do{ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + }while( iColpLeft, nCol); + fts3EvalUpdateCounts(pExpr->pRight, nCol); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot, pTab->nColumn); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + if( pRoot->bEof ) rc = FTS_CORRUPT_VTAB; + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif diff --git a/third_party/sqlite3/fts3.inc b/third_party/sqlite3/fts3.inc new file mode 100644 index 000000000..10283cff3 --- /dev/null +++ b/third_party/sqlite3/fts3.inc @@ -0,0 +1,26 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +#include "third_party/sqlite3/sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +int sqlite3Fts3Init(sqlite3 *db); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ diff --git a/third_party/sqlite3/fts3Int.inc b/third_party/sqlite3/fts3Int.inc new file mode 100644 index 000000000..52e481a60 --- /dev/null +++ b/third_party/sqlite3/fts3Int.inc @@ -0,0 +1,653 @@ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H +/* clang-format off */ + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +#include "third_party/sqlite3/sqlite3ext.h" +SQLITE_EXTENSION_INIT3 +#endif + +#include "third_party/sqlite3/fts3_hash.inc" +#include "third_party/sqlite3/fts3_tokenizer.inc" +#include "third_party/sqlite3/sqlite3.h" + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +#define FTS3_BUFFER_PADDING 8 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** The assert_fts3_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +extern int sqlite3_fts3_may_be_corrupt; +# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) +#else +# define assert_fts3_nc(x) assert(x) +#endif + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +int sqlite3Fts3Always(int b); +int sqlite3Fts3Never(int b); +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#define deliberate_fall_through + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + int bLock; /* Used to prevent recursive content= tbls */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; + + /* Number of segments in a level */ + int nMergeCount; +#endif +}; + +/* Macro to find the number of segments to merge */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +# define MergeCount(P) ((P)->nMergeCount) +#else +# define MergeCount(P) FTS3_MERGE_COUNT +#endif + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +int sqlite3Fts3PendingTermsFlush(Fts3Table *); +void sqlite3Fts3PendingTermsClear(Fts3Table *); +int sqlite3Fts3Optimize(Fts3Table *); +int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +void sqlite3Fts3SegReaderFree(Fts3SegReader *); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +void sqlite3Fts3ErrMsg(char**,const char*,...); +int sqlite3Fts3PutVarint(char *, sqlite3_int64); +int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); +int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); +int sqlite3Fts3GetVarint32(const char *, int *); +int sqlite3Fts3VarintLen(sqlite3_uint64); +void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +int sqlite3Fts3ReadInt(const char *z, int *pnOut); + +/* fts3_tokenizer.c */ +const char *sqlite3Fts3NextToken(const char *, int *); +int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); + +/* fts3_expr.c */ +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); +int sqlite3Fts3InitTerm(sqlite3 *db); +#endif + +int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +int sqlite3Fts3InitAux(sqlite3 *db); + +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +int sqlite3FtsUnicodeFold(int, int); +int sqlite3FtsUnicodeIsalnum(int); +int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ diff --git a/third_party/sqlite3/fts3_aux.c b/third_party/sqlite3/fts3_aux.c new file mode 100644 index 000000000..012edf6d3 --- /dev/null +++ b/third_party/sqlite3/fts3_aux.c @@ -0,0 +1,552 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +/* clang-format off */ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/str/str.h" + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm); + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + pCsr->nStop = (int)strlen(pCsr->zStop); + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_expr.c b/third_party/sqlite3/fts3_expr.c new file mode 100644 index 000000000..35fc10953 --- /dev/null +++ b/third_party/sqlite3/fts3_expr.c @@ -0,0 +1,1294 @@ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* clang-format off */ + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +#include "libc/assert.h" +#include "libc/str/str.h" + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(sqlite3_int64 nByte){ + void *pRet = sqlite3_malloc64(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + sqlite3_int64 nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){ + void *pRet = sqlite3_realloc64(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; +#if !defined(SQLITE_MAX_EXPR_DEPTH) + if( pParse->nNest>1000 ) return SQLITE_ERROR; +#elif SQLITE_MAX_EXPR_DEPTH>0 + if( pParse->nNest>SQLITE_MAX_EXPR_DEPTH ) return SQLITE_ERROR; +#endif + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +#include "libc/stdio/stdio.h" + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTestCommon( + int bRebalance, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); + const char *zTokenizer = 0; + char *zErr = 0; + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + zTokenizer = (const char*)sqlite3_value_text(argv[0]); + rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, zErr, -1); + } + sqlite3_free(zErr); + return; + } + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iipModule->xDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(0, context, argc, argv); +} +static void fts3ExprTestRebalance( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(1, context, argc, argv); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_hash.c b/third_party/sqlite3/fts3_hash.c new file mode 100644 index 000000000..8fa342002 --- /dev/null +++ b/third_party/sqlite3/fts3_hash.c @@ -0,0 +1,383 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ +/* clang-format off */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/fts3_hash.inc" + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(sqlite3_int64 n){ + void *p = sqlite3_malloc64(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_hash.inc b/third_party/sqlite3/fts3_hash.inc new file mode 100644 index 000000000..447d19fea --- /dev/null +++ b/third_party/sqlite3/fts3_hash.inc @@ -0,0 +1,113 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ +/* clang-format off */ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +void sqlite3Fts3HashClear(Fts3Hash*); +Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ diff --git a/third_party/sqlite3/fts3_icu.c b/third_party/sqlite3/fts3_icu.c new file mode 100644 index 000000000..f864e9c07 --- /dev/null +++ b/third_party/sqlite3/fts3_icu.c @@ -0,0 +1,259 @@ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU +/* clang-format off */ + +#include "libc/assert.h" +#include "libc/str/str.h" +#include "libc/unicode/unicode.h" +#include "third_party/sqlite3/fts3_tokenizer.inc" + +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; + +struct IcuCursor { + sqlite3_tokenizer_cursor base; + + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ + + int nBuffer; + char *zBuffer; + + int iToken; +}; + +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; + + if( argc>0 ){ + n = strlen(argv[0])+1; + } + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); + + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } + + *ppTokenizer = (sqlite3_tokenizer *)p; + + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; + + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; + + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc64( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; + + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; + + if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; + + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to icuOpen(). +*/ +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. +*/ +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + + int iStart = 0; + int iEnd = 0; + int nByte = 0; + + while( iStart==iEnd ){ + UChar32 c; + + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; + } + } + assert(iStart<=iEnd); + } + + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; + } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; + + return SQLITE_OK; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; + +/* +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_porter.c b/third_party/sqlite3/fts3_porter.c new file mode 100644 index 000000000..415905913 --- /dev/null +++ b/third_party/sqlite3/fts3_porter.c @@ -0,0 +1,662 @@ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ +/* clang-format off */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/fts3_tokenizer.inc" + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_snippet.c b/third_party/sqlite3/fts3_snippet.c new file mode 100644 index 000000000..8ab5a78ae --- /dev/null +++ b/third_party/sqlite3/fts3_snippet.c @@ -0,0 +1,1749 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ +/* clang-format off */ + +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/str/str.h" + +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + + +/* +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. +*/ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; + +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; + +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; + +/* +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; + +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; + + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; + + +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); + + pRet = sqlite3_malloc64(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + + +/* +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. +*/ +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } + return rc; +} + +/* +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. +*/ +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} + + +/* +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; + + return rc; +} + +/* +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. +*/ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ +){ + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} + +/* +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. +*/ +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; + + while( iIteriCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } + } + + return 0; +} + +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ + int j; + u64 mPhrase = (u64)1 << (i%64); + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart && (iCsr - iStart)<=64 ); + assert( i>=0 ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } + } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} + +/* +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; + + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + if( iFirst<0 ){ + rc = FTS_CORRUPT_VTAB; + }else{ + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + } + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); + } + + return rc; +} + +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + sqlite3_int64 nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ + + memset(&sIter, 0, sizeof(sIter)); + + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + *piScore = iBestScore; + } + sqlite3_free(sIter.aPhrase); + return rc; +} + + +/* +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). +*/ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100; + char *zNew = sqlite3_realloc64(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; +} + +/* +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: +** +** ........X.....X +** +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. +*/ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ + + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ + + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 ); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } + } + } + return SQLITE_OK; +} + +/* +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. +*/ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; + } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } + + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + + pMod->xClose(pC); + return rc; +} + + +/* +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. +*/ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } + + *ppCollist = pEnd; + return nEntry; +} + +/* +** This function gathers 'y' or 'b' data for a single phrase. +*/ +static int fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ +){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } + + if( pIter ) while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static int fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + int rc = SQLITE_OK; + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + rc = fts3ExprLHitGather(pExpr->pLeft, p); + if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p); + }else{ + rc = fts3ExprLHits(pExpr, p); + } + } + return rc; +} + +/* +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} + +/* +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; + + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } + } + + return rc; +} + +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; + } + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen, + const char **ppEnd +){ + sqlite3_stmt *pStmt; + const char *a; + const char *pEnd; + sqlite3_int64 nDoc; + int n; + + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); + + n = sqlite3_column_bytes(pStmt, 0); + a = sqlite3_column_blob(pStmt, 0); + if( a==0 ){ + return FTS_CORRUPT_VTAB; + } + pEnd = a + n; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + if( nDoc<=0 || a>pEnd ){ + return FTS_CORRUPT_VTAB; + } + *pnDoc = nDoc; + + if( paLen ) *paLen = a; + if( ppEnd ) *ppEnd = pEnd; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; + + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + int rc = SQLITE_OK; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) goto matchinfo_lcs_out; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(pIt); + if( pIt->pRead==0 ){ + rc = FTS_CORRUPT_VTAB; + goto matchinfo_lcs_out; + } + nLive++; + } + } + + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + matchinfo_lcs_out: + sqlite3_free(aIter); + return rc; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + const char *pEnd; /* First byte past end of length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + + +/* +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +*/ +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; + } + + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->pMIBuffer==0 ){ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ + + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; + + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } + + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; + + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; + } + + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } +} + +/* +** Implementation of snippet() function. +*/ +void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; + + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + /* Limit the snippet length to 64 tokens. */ + if( nToken<-64 ) nToken = -64; + if( nToken>+64 ) nToken = +64; + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } + + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } + + mCovered |= pFragment->covered; + } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } + + assert( nFToken>0 ); + + for(i=0; ipCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert_fts3_nc( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + + return rc; +} + +/* +** Implementation of offsets() function. +*/ +void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); + + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; + + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); + + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } + + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; + + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ + + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } + + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert_fts3_nc( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } + + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} + +/* +** Implementation of matchinfo() function. +*/ +void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; + + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + }else{ + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + } +} + +#endif diff --git a/third_party/sqlite3/fts3_tokenize_vtab.c b/third_party/sqlite3/fts3_tokenize_vtab.c new file mode 100644 index 000000000..79a43b99f --- /dev/null +++ b/third_party/sqlite3/fts3_tokenize_vtab.c @@ -0,0 +1,457 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* clang-format off */ + +#include "libc/assert.h" +#include "libc/str/str.h" + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; i1 ) azArg = (const char * const *)&azDequote[1]; + rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_tokenizer.c b/third_party/sqlite3/fts3_tokenizer.c new file mode 100644 index 000000000..617351754 --- /dev/null +++ b/third_party/sqlite3/fts3_tokenizer.c @@ -0,0 +1,515 @@ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ +/* clang-format off */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/str/str.h" + +/* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void fts3TokenizerFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } +} + +int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST +#include "libc/str/str.h" + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB + && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp) + ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_UTF8|SQLITE_DIRECTONLY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_tokenizer.inc b/third_party/sqlite3/fts3_tokenizer.inc new file mode 100644 index 000000000..725f3aa36 --- /dev/null +++ b/third_party/sqlite3/fts3_tokenizer.inc @@ -0,0 +1,162 @@ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ +/* clang-format off */ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +#include "third_party/sqlite3/sqlite3.h" + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/third_party/sqlite3/fts3_tokenizer1.c b/third_party/sqlite3/fts3_tokenizer1.c new file mode 100644 index 000000000..348ab6d69 --- /dev/null +++ b/third_party/sqlite3/fts3_tokenizer1.c @@ -0,0 +1,234 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ +/* clang-format off */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/fts3_tokenizer.inc" + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/third_party/sqlite3/fts3_unicode.c b/third_party/sqlite3/fts3_unicode.c new file mode 100644 index 000000000..648f1361a --- /dev/null +++ b/third_party/sqlite3/fts3_unicode.c @@ -0,0 +1,397 @@ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ +/* clang-format off */ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/fts3_tokenizer.inc" + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int eRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + +/* +** Destroy a tokenizer allocated by unicodeCreate(). +*/ +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. +*/ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + unsigned int iCode; + int nEntry = 0; + + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException,(p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = (int)iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} + +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; + + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->eRemoveDiacritic = 1; + + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + pCsr->aInput = (const unsigned char*)""; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + unsigned int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; + + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; +} + +/* +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. +*/ +void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ diff --git a/third_party/sqlite3/fts3_unicode2.c b/third_party/sqlite3/fts3_unicode2.c new file mode 100644 index 000000000..652c334f4 --- /dev/null +++ b/third_party/sqlite3/fts3_unicode2.c @@ -0,0 +1,384 @@ +/* +** 2012-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ +/* clang-format off */ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +#include "libc/assert.h" + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ diff --git a/third_party/sqlite3/fts3_write.c b/third_party/sqlite3/fts3_write.c new file mode 100644 index 000000000..fa5aae5fe --- /dev/null +++ b/third_party/sqlite3/fts3_write.c @@ -0,0 +1,5799 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ +/* clang-format off */ + +#include "third_party/sqlite3/fts3Int.inc" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "libc/alg/alg.h" +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM (SELECT * FROM %Q.'%q_segdir' " + " WHERE level = ? ORDER BY idx ASC LIMIT ?" + " )", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + f &= ~SQLITE_PREPARE_NO_VTAB; + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert_fts3_nc( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=MergeCount(p) ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + }else if( rc==SQLITE_ERROR ){ + rc = FTS_CORRUPT_VTAB; + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nSuffix<=0 + || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTerm + ){ + return FTS_CORRUPT_VTAB; + } + + /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are + ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer + ** overflow - hence the (i64) casts. */ + if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){ + i64 nNew = ((i64)nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc64(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); + }else{ + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); + } + } + } + } + + return rc; +} + + +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( zRoot!=0 || nRoot==0 ); +#ifdef CORRUPT_DB + assert( zRoot!=0 || CORRUPT_DB ); +#endif + + if( iStartLeaf==0 ){ + if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB; + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + assert( nSuffix>0 ); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + assert( nDoclist>0 ); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + assert( nTerm>0 ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ + + for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( p0){ + memset(&pList[nList], 0, pEnd - &pList[nList]); + } + *ppList = pList; + *pnList = nList; +} + +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + assert( nList>0 ); + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ + if( nReq>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = nReq*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + return SQLITE_OK; +} + + +int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } + + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } + + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iPrev - (u64)iDocid); + }else{ + if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iDocid - (u64)iPrev); + } + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + + rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist); + if( rc ) return rc; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } + + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + + +void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + u64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = (i64)iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = ((i64)iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ + rc = SQLITE_DONE; + goto finished; + } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; + + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; + + assert( csr.nSegment>0 ); + assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnLeafData); + } + } + } + + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} + + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + sqlite3_free(a); +} + +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + } + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + sqlite3Fts3SegmentsClose(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ + return FTS_CORRUPT_VTAB; + } + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + if( (p->nNode-p->iOff)nDoclist ){ + return FTS_CORRUPT_VTAB; + } + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert_fts3_nc( p->iOff<=p->nNode ); + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( aNode && aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return aNode ? nodeReaderNext(p) : SQLITE_OK; +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + if( aRoot==0 ){ + sqlite3_reset(pSelect); + return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB; + } + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + if( nHeight<1 || nHeight>=FTS_MAX_APPENDABLE_HEIGHT ){ + sqlite3_reset(pSelect); + return FTS_CORRUPT_VTAB; + } + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + assert_fts3_nc( reader.term.n>0 || reader.aNode==0 ); + if( reader.term.n>0 ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + } + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); + } + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-1; + if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + if( i==0 ) return FTS_CORRUPT_VTAB; + i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + assert( i<=nHint ); + if( i!=nHint ) return FTS_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. +*/ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ + iAbsLevel = iHintAbsLevel; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<=0 ) break; + + assert( nMod<=0x7FFFFFFF ); + if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){ + rc = FTS_CORRUPT_VTAB; + break; + } + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + ){ + int bEmpty = 0; + rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( rc==SQLITE_OK ){ + bEmpty = 1; + }else if( rc!=SQLITE_ROW ){ + sqlite3Fts3SegReaderFinish(pCsr); + break; + } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + if( bEmpty==0 ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (MergeCount(p) / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + u64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); + }else{ + iDocid = (i64)((u64)iDocid + iVal); + } + } + }else{ + iPos += (iVal - 2); + cksum = cksum ^ fts3ChecksumEntry( + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos + ); + } + } + } + } + } + sqlite3Fts3SegReaderFinish(&csr); + + *pRc = rc; + return cksum; +} + +/* +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. +*/ +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; + return rc; +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc = SQLITE_ERROR; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + }else{ + int v; + if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + v = atoi(&zVal[9]); + if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + v = atoi(&zVal[11]); + if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){ + v = atoi(&zVal[11]); + if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; + rc = SQLITE_OK; + } +#endif + } + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } + + return rc; +} + +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} +#endif + +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + } + + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE
( +** , +**
HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; + + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; + } + + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + } + + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +#endif diff --git a/third_party/sqlite3/fts5.c b/third_party/sqlite3/fts5.c new file mode 100644 index 000000000..3020f8342 --- /dev/null +++ b/third_party/sqlite3/fts5.c @@ -0,0 +1,22814 @@ + +/* clang-format off */ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +#line 1 "fts5.h" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + +#include "third_party/sqlite3/sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
+** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + +#line 1 "fts5Int.h" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTS5INT_H +#define _FTS5INT_H + +/* #include "third_party/sqlite3/fts5.h" */ +#include "third_party/sqlite3/sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#include "libc/assert.h" +#include "libc/str/str.h" + +#ifndef SQLITE_AMALGAMATION + +typedef unsigned char u8; +typedef unsigned int u32; +typedef unsigned short u16; +typedef short i16; +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif + +#define testcase(x) +#define ALWAYS(x) 1 +#define NEVER(x) 0 + +#define MIN(x,y) (((x) < (y)) ? (x) : (y)) +#define MAX(x,y) (((x) > (y)) ? (x) : (y)) + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +*/ +# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#endif + +/* Truncate very long tokens to this many bytes. Hard limit is +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset +** field that occurs at the start of each leaf page (see fts5_index.c). */ +#define FTS5_MAX_TOKEN_SIZE 32768 + +/* +** Maximum number of prefix indexes on single FTS5 table. This must be +** less than 32. If it is set to anything large than that, an #error +** directive in fts5_index.c will cause the build to fail. +*/ +#define FTS5_MAX_PREFIX_INDEXES 31 + +/* +** Maximum segments permitted in a single index +*/ +#define FTS5_MAX_SEGMENT 2000 + +#define FTS5_DEFAULT_NEARDIST 10 +#define FTS5_DEFAULT_RANK "bm25" + +/* Name of rank and rowid columns */ +#define FTS5_RANK_NAME "rank" +#define FTS5_ROWID_NAME "rowid" + +#ifdef SQLITE_DEBUG +# define FTS5_CORRUPT sqlite3Fts5Corrupt() +static int sqlite3Fts5Corrupt(void); +#else +# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB +#endif + +/* +** The assert_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +extern int sqlite3_fts5_may_be_corrupt; +# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) +#else +# define assert_nc(x) assert(x) +#endif + +/* +** A version of memcmp() that does not cause asan errors if one of the pointer +** parameters is NULL and the number of bytes to compare is zero. +*/ +#define fts5Memcmp(s1, s2, n) ((n)==0 ? 0 : memcmp((s1), (s2), (n))) + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef UNUSED_PARAM2 +# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y) +#endif + +typedef struct Fts5Global Fts5Global; +typedef struct Fts5Colset Fts5Colset; + +/* If a NEAR() clump or phrase may only match a specific set of columns, +** then an object of the following type is used to record the set of columns. +** Each entry in the aiCol[] array is a column that may be matched. +** +** This object is used by fts5_expr.c and fts5_index.c. +*/ +struct Fts5Colset { + int nCol; + int aiCol[1]; +}; + + + +/************************************************************************** +** Interface to code in fts5_config.c. fts5_config.c contains contains code +** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. +*/ + +typedef struct Fts5Config Fts5Config; + +/* +** An instance of the following structure encodes all information that can +** be gleaned from the CREATE VIRTUAL TABLE statement. +** +** And all information loaded from the %_config table. +** +** nAutomerge: +** The minimum number of segments that an auto-merge operation should +** attempt to merge together. A value of 1 sets the object to use the +** compile time default. Zero disables auto-merge altogether. +** +** zContent: +** +** zContentRowid: +** The value of the content_rowid= option, if one was specified. Or +** the string "rowid" otherwise. This text is not quoted - if it is +** used as part of an SQL statement it needs to be quoted appropriately. +** +** zContentExprlist: +** +** pzErrmsg: +** This exists in order to allow the fts5_index.c module to return a +** decent error message if it encounters a file-format version it does +** not understand. +** +** bColumnsize: +** True if the %_docsize table is created. +** +** bPrefixIndex: +** This is only used for debugging. If set to false, any prefix indexes +** are ignored. This value is configured using: +** +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); +** +*/ +struct Fts5Config { + sqlite3 *db; /* Database handle */ + char *zDb; /* Database holding FTS index (e.g. "main") */ + char *zName; /* Name of FTS index */ + int nCol; /* Number of columns */ + char **azCol; /* Column names */ + u8 *abUnindexed; /* True for unindexed columns */ + int nPrefix; /* Number of prefix indexes */ + int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ + int eContent; /* An FTS5_CONTENT value */ + char *zContent; /* content table */ + char *zContentRowid; /* "content_rowid=" option value */ + int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int eDetail; /* FTS5_DETAIL_XXX value */ + char *zContentExprlist; + Fts5Tokenizer *pTok; + fts5_tokenizer *pTokApi; + int bLock; /* True when table is preparing statement */ + int ePattern; /* FTS_PATTERN_XXX constant */ + + /* Values loaded from the %_config table */ + int iCookie; /* Incremented when %_config is modified */ + int pgsz; /* Approximate page size used in %_data */ + int nAutomerge; /* 'automerge' setting */ + int nCrisisMerge; /* Maximum allowed segments per level */ + int nUsermerge; /* 'usermerge' setting */ + int nHashSize; /* Bytes of memory for in-memory hash */ + char *zRank; /* Name of rank function */ + char *zRankArgs; /* Arguments to rank function */ + + /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ + char **pzErrmsg; + +#ifdef SQLITE_DEBUG + int bPrefixIndex; /* True to use prefix-indexes */ +#endif +}; + +/* Current expected value of %_config table 'version' field */ +#define FTS5_CURRENT_VERSION 4 + +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 + +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 + +#define FTS5_PATTERN_NONE 0 +#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */ +#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */ + +static int sqlite3Fts5ConfigParse( + Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** +); +static void sqlite3Fts5ConfigFree(Fts5Config*); + +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); + +static int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +); + +static void sqlite3Fts5Dequote(char *z); + +/* Load the contents of the %_config table */ +static int sqlite3Fts5ConfigLoad(Fts5Config*, int); + +/* Set the value of a single config attribute */ +static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); + +static int sqlite3Fts5ConfigParseRank(const char*, char**, char**); + +/* +** End of interface to code in fts5_config.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_buffer.c. +*/ + +/* +** Buffer object for the incremental building of string data. +*/ +typedef struct Fts5Buffer Fts5Buffer; +struct Fts5Buffer { + u8 *p; + int n; + int nSpace; +}; + +static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); +static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); +static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); +static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); +static void sqlite3Fts5BufferFree(Fts5Buffer*); +static void sqlite3Fts5BufferZero(Fts5Buffer*); +static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); +static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); + +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); + +#define fts5BufferZero(x) sqlite3Fts5BufferZero(x) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferFree(a) sqlite3Fts5BufferFree(a) +#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) +#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) + +#define fts5BufferGrow(pRc,pBuf,nn) ( \ + (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ + sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ +) + +/* Write and decode big-endian 32-bit integer values */ +static void sqlite3Fts5Put32(u8*, int); +static int sqlite3Fts5Get32(const u8*); + +#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) + +typedef struct Fts5PoslistReader Fts5PoslistReader; +struct Fts5PoslistReader { + /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ + const u8 *a; /* Position list to iterate through */ + int n; /* Size of buffer at a[] in bytes */ + int i; /* Current offset in a[] */ + + u8 bFlag; /* For client use (any custom purpose) */ + + /* Output variables */ + u8 bEof; /* Set to true at EOF */ + i64 iPos; /* (iCol<<32) + iPos */ +}; +static int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +); +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); + +typedef struct Fts5PoslistWriter Fts5PoslistWriter; +struct Fts5PoslistWriter { + i64 iPrev; +}; +static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); +static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); + +static int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +); + +/* Malloc utility */ +static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte); +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); + +/* Character set tests (like isspace(), isalpha() etc.) */ +static int sqlite3Fts5IsBareword(char t); + + +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ +typedef struct Fts5Termset Fts5Termset; +static int sqlite3Fts5TermsetNew(Fts5Termset**); +static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); +static void sqlite3Fts5TermsetFree(Fts5Termset*); + +/* +** End of interface to code in fts5_buffer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_index.c. fts5_index.c contains contains code +** to access the data stored in the %_data table. +*/ + +typedef struct Fts5Index Fts5Index; +typedef struct Fts5IndexIter Fts5IndexIter; + +struct Fts5IndexIter { + i64 iRowid; + const u8 *pData; + int nData; + u8 bEof; +}; + +#define sqlite3Fts5IterEof(x) ((x)->bEof) + +/* +** Values used as part of the flags argument passed to IndexQuery(). +*/ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ + +/* The following are used internally by the fts5_index.c module. They are +** defined here only to make it easier to avoid clashes with the flags +** above. */ +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 + +/* +** Create/destroy an Fts5Index object. +*/ +static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); +static int sqlite3Fts5IndexClose(Fts5Index *p); + +/* +** Return a simple checksum value based on the arguments. +*/ +static u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +); + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +static int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +); + +/* +** Open a new iterator to iterate though all rowids that match the +** specified token or token prefix. +*/ +static int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +); + +/* +** The various operations on open token or token prefix iterators opened +** using sqlite3Fts5IndexQuery(). +*/ +static int sqlite3Fts5IterNext(Fts5IndexIter*); +static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); + +/* +** Close an iterator opened by sqlite3Fts5IndexQuery(). +*/ +static void sqlite3Fts5IterClose(Fts5IndexIter*); + +/* +** Close the reader blob handle, if it is open. +*/ +static void sqlite3Fts5IndexCloseReader(Fts5Index*); + +/* +** This interface is used by the fts5vocab module. +*/ +static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); +static int sqlite3Fts5IterNextScan(Fts5IndexIter*); + + +/* +** Insert or remove data to or from the index. Each time a document is +** added to or removed from the index, this function is called one or more +** times. +** +** For an insert, it must be called once for each token in the new document. +** If the operation is a delete, it must be called (at least) once for each +** unique token in the document with an iCol value less than zero. The iPos +** argument is ignored for a delete. +*/ +static int sqlite3Fts5IndexWrite( + Fts5Index *p, /* Index to write to */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to +** document iDocid. +*/ +static int sqlite3Fts5IndexBeginWrite( + Fts5Index *p, /* Index to write to */ + int bDelete, /* True if current operation is a delete */ + i64 iDocid /* Docid to add or remove data from */ +); + +/* +** Flush any data stored in the in-memory hash tables to the database. +** Also close any open blob handles. +*/ +static int sqlite3Fts5IndexSync(Fts5Index *p); + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +static int sqlite3Fts5IndexRollback(Fts5Index *p); + +/* +** Get or set the "averages" values. +*/ +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); + +/* +** Functions called by the storage module as part of integrity-check. +*/ +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum); + +/* +** Called during virtual module initialization to register UDF +** fts5_decode() with SQLite +*/ +static int sqlite3Fts5IndexInit(sqlite3*); + +static int sqlite3Fts5IndexSetCookie(Fts5Index*, int); + +/* +** Return the total number of entries read from the %_data table by +** this connection since it was created. +*/ +static int sqlite3Fts5IndexReads(Fts5Index *p); + +static int sqlite3Fts5IndexReinit(Fts5Index *p); +static int sqlite3Fts5IndexOptimize(Fts5Index *p); +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); +static int sqlite3Fts5IndexReset(Fts5Index *p); + +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); + +/* +** End of interface to code in fts5_index.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_varint.c. +*/ +static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); +static int sqlite3Fts5GetVarintLen(u32 iVal); +static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); + +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) +#define fts5GetVarint sqlite3Fts5GetVarint + +#define fts5FastGetVarint32(a, iOff, nVal) { \ + nVal = (a)[iOff++]; \ + if( nVal & 0x80 ){ \ + iOff--; \ + iOff += fts5GetVarint32(&(a)[iOff], nVal); \ + } \ +} + + +/* +** End of interface to code in fts5_varint.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5_main.c. +*/ + +/* +** Virtual-table object. +*/ +typedef struct Fts5Table Fts5Table; +struct Fts5Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts5Config *pConfig; /* Virtual table configuration */ + Fts5Index *pIndex; /* Full-text index */ +}; + +static int sqlite3Fts5GetTokenizer( + Fts5Global*, + const char **azArg, + int nArg, + Fts5Config*, + char **pzErr +); + +static Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); + +static int sqlite3Fts5FlushToDisk(Fts5Table*); + +/* +** End of interface to code in fts5.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_hash.c. +*/ +typedef struct Fts5Hash Fts5Hash; + +/* +** Create a hash table, free a hash table. +*/ +static int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); +static void sqlite3Fts5HashFree(Fts5Hash*); + +static int sqlite3Fts5HashWrite( + Fts5Hash*, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Empty (but do not delete) a hash table. +*/ +static void sqlite3Fts5HashClear(Fts5Hash*); + +static int sqlite3Fts5HashQuery( + Fts5Hash*, /* Hash table to query */ + int nPre, + const char *pTerm, int nTerm, /* Query term */ + void **ppObj, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +); + +static int sqlite3Fts5HashScanInit( + Fts5Hash*, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +); +static void sqlite3Fts5HashScanNext(Fts5Hash*); +static int sqlite3Fts5HashScanEof(Fts5Hash*); +static void sqlite3Fts5HashScanEntry(Fts5Hash *, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +); + + +/* +** End of interface to code in fts5_hash.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_storage.c. fts5_storage.c contains contains +** code to access the data stored in the %_content and %_docsize tables. +*/ + +#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */ +#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */ +#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */ + +typedef struct Fts5Storage Fts5Storage; + +static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); +static int sqlite3Fts5StorageClose(Fts5Storage *p); +static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); + +static int sqlite3Fts5DropAll(Fts5Config*); +static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); + +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); +static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); + +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg); + +static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); +static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); + +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); + +static int sqlite3Fts5StorageSync(Fts5Storage *p); +static int sqlite3Fts5StorageRollback(Fts5Storage *p); + +static int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, const char*, sqlite3_value*, int +); + +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); +static int sqlite3Fts5StorageRebuild(Fts5Storage *p); +static int sqlite3Fts5StorageOptimize(Fts5Storage *p); +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); +static int sqlite3Fts5StorageReset(Fts5Storage *p); + +/* +** End of interface to code in fts5_storage.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5_expr.c. +*/ +typedef struct Fts5Expr Fts5Expr; +typedef struct Fts5ExprNode Fts5ExprNode; +typedef struct Fts5Parse Fts5Parse; +typedef struct Fts5Token Fts5Token; +typedef struct Fts5ExprPhrase Fts5ExprPhrase; +typedef struct Fts5ExprNearset Fts5ExprNearset; + +struct Fts5Token { + const char *p; /* Token text (not NULL terminated) */ + int n; /* Size of buffer p in bytes */ +}; + +/* Parse a MATCH expression. */ +static int sqlite3Fts5ExprNew( + Fts5Config *pConfig, + int bPhraseToAnd, + int iCol, /* Column on LHS of MATCH operator */ + const char *zExpr, + Fts5Expr **ppNew, + char **pzErr +); +static int sqlite3Fts5ExprPattern( + Fts5Config *pConfig, + int bGlob, + int iCol, + const char *zText, + Fts5Expr **pp +); + +/* +** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); +** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr); +** rc = sqlite3Fts5ExprNext(pExpr) +** ){ +** // The document with rowid iRowid matches the expression! +** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); +** } +*/ +static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); +static int sqlite3Fts5ExprEof(Fts5Expr*); +static i64 sqlite3Fts5ExprRowid(Fts5Expr*); + +static void sqlite3Fts5ExprFree(Fts5Expr*); +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2); + +/* Called during startup to register a UDF with SQLite */ +static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); + +static int sqlite3Fts5ExprPhraseCount(Fts5Expr*); +static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); +static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); + +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; +static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); +static int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int +); +static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); + +static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); + +static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); + +/******************************************* +** The fts5_expr.c API above this point is used by the other hand-written +** C code in this module. The interfaces below this point are called by +** the parser code in fts5parse.y. */ + +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); + +static Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, + int eType, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight, + Fts5ExprNearset *pNear +); + +static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight +); + +static Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, + Fts5ExprPhrase *pPhrase, + Fts5Token *pToken, + int bPrefix +); + +static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*); + +static Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse*, + Fts5ExprNearset*, + Fts5ExprPhrase* +); + +static Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse*, + Fts5Colset*, + Fts5Token * +); + +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); + +static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); +static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*); +static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); + +/* +** End of interface to code in fts5_expr.c. +**************************************************************************/ + + + +/************************************************************************** +** Interface to code in fts5_aux.c. +*/ + +static int sqlite3Fts5AuxInit(fts5_api*); +/* +** End of interface to code in fts5_aux.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_tokenizer.c. +*/ + +static int sqlite3Fts5TokenizerInit(fts5_api*); +static int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +); +/* +** End of interface to code in fts5_tokenizer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_vocab.c. +*/ + +static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); + +/* +** End of interface to code in fts5_vocab.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to automatically generated code in fts5_unicode2.c. +*/ +static int sqlite3Fts5UnicodeIsdiacritic(int c); +static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); + +static int sqlite3Fts5UnicodeCatParse(const char*, u8*); +static int sqlite3Fts5UnicodeCategory(u32 iCode); +static void sqlite3Fts5UnicodeAscii(u8*, u8*); +/* +** End of interface to code in fts5_unicode2.c. +**************************************************************************/ + +#endif + +#line 1 "fts5parse.h" +#define FTS5_OR 1 +#define FTS5_AND 2 +#define FTS5_NOT 3 +#define FTS5_TERM 4 +#define FTS5_COLON 5 +#define FTS5_MINUS 6 +#define FTS5_LCP 7 +#define FTS5_RCP 8 +#define FTS5_STRING 9 +#define FTS5_LP 10 +#define FTS5_RP 11 +#define FTS5_CARET 12 +#define FTS5_COMMA 13 +#define FTS5_PLUS 14 +#define FTS5_STAR 15 + +#line 1 "fts5parse.c" +/* This file is automatically generated by Lemon from input grammar +** source file "fts5parse.y". */ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +/************ Begin %include sections from the grammar ************************/ +#line 47 "fts5parse.y" + +/* #include "third_party/sqlite3/fts5Int.h" */ +/* #include "third_party/sqlite3/fts5parse.h" */ + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define fts5YYNOERRORRECOVERY 1 + +/* +** Make fts5yytestcase() the same as testcase() +*/ +#define fts5yytestcase(X) testcase(X) + +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define fts5YYPARSEFREENOTNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define fts5YYMALLOCARGTYPE u64 + +#line 57 "fts5parse.c" +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef FTS5_OR +#define FTS5_OR 1 +#define FTS5_AND 2 +#define FTS5_NOT 3 +#define FTS5_TERM 4 +#define FTS5_COLON 5 +#define FTS5_MINUS 6 +#define FTS5_LCP 7 +#define FTS5_RCP 8 +#define FTS5_STRING 9 +#define FTS5_LP 10 +#define FTS5_RP 11 +#define FTS5_CARET 12 +#define FTS5_COMMA 13 +#define FTS5_PLUS 14 +#define FTS5_STAR 15 +#endif +/**************** End token definitions ***************************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** fts5YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** fts5YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** fts5YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** fts5YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union +** for terminal symbols is called "fts5yy0". +** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3Fts5ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter +** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser +** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser +** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** fts5YYNSTATE the combined number of states. +** fts5YYNRULE the number of rules in the grammar +** fts5YYNFTS5TOKEN Number of terminal symbols +** fts5YY_MAX_SHIFT Maximum value for shift actions +** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error +** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept +** fts5YY_NO_ACTION The fts5yy_action[] code for no-op +** fts5YY_MIN_REDUCE Minimum value for reduce actions +** fts5YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define fts5YYCODETYPE unsigned char +#define fts5YYNOCODE 27 +#define fts5YYACTIONTYPE unsigned char +#define sqlite3Fts5ParserFTS5TOKENTYPE Fts5Token +typedef union { + int fts5yyinit; + sqlite3Fts5ParserFTS5TOKENTYPE fts5yy0; + int fts5yy4; + Fts5Colset* fts5yy11; + Fts5ExprNode* fts5yy24; + Fts5ExprNearset* fts5yy46; + Fts5ExprPhrase* fts5yy53; +} fts5YYMINORTYPE; +#ifndef fts5YYSTACKDEPTH +#define fts5YYSTACKDEPTH 100 +#endif +#define sqlite3Fts5ParserARG_SDECL Fts5Parse *pParse; +#define sqlite3Fts5ParserARG_PDECL ,Fts5Parse *pParse +#define sqlite3Fts5ParserARG_PARAM ,pParse +#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; +#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define sqlite3Fts5ParserCTX_SDECL +#define sqlite3Fts5ParserCTX_PDECL +#define sqlite3Fts5ParserCTX_PARAM +#define sqlite3Fts5ParserCTX_FETCH +#define sqlite3Fts5ParserCTX_STORE +#define fts5YYNSTATE 35 +#define fts5YYNRULE 28 +#define fts5YYNRULE_WITH_ACTION 28 +#define fts5YYNFTS5TOKEN 16 +#define fts5YY_MAX_SHIFT 34 +#define fts5YY_MIN_SHIFTREDUCE 52 +#define fts5YY_MAX_SHIFTREDUCE 79 +#define fts5YY_ERROR_ACTION 80 +#define fts5YY_ACCEPT_ACTION 81 +#define fts5YY_NO_ACTION 82 +#define fts5YY_MIN_REDUCE 83 +#define fts5YY_MAX_REDUCE 110 +/************* End control #defines *******************************************/ +#define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) + +/* Define the fts5yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define fts5yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the fts5yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef fts5yytestcase +# define fts5yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE. +** +** N == fts5YY_ERROR_ACTION A syntax error has occurred. +** +** N == fts5YY_ACCEPT_ACTION The parser accepts its input. +** +** N == fts5YY_NO_ACTION No such action. Denotes unused +** slots in the fts5yy_action[] table. +** +** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE +** and fts5YY_MAX_REDUCE +** +** The action table is constructed as a single large table named fts5yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = fts5yy_action[ fts5yy_shift_ofst[S] + X ] +** (B) N = fts5yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of +** the fts5yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** fts5yy_action[] A single table containing all actions. +** fts5yy_lookahead[] A table containing the lookahead for each entry in +** fts5yy_action. Used to detect hash collisions. +** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for +** shifting terminals. +** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for +** shifting non-terminals after a reduce. +** fts5yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define fts5YY_ACTTAB_COUNT (105) +static const fts5YYACTIONTYPE fts5yy_action[] = { + /* 0 */ 81, 20, 96, 6, 28, 99, 98, 26, 26, 18, + /* 10 */ 96, 6, 28, 17, 98, 56, 26, 19, 96, 6, + /* 20 */ 28, 14, 98, 14, 26, 31, 92, 96, 6, 28, + /* 30 */ 108, 98, 25, 26, 21, 96, 6, 28, 78, 98, + /* 40 */ 58, 26, 29, 96, 6, 28, 107, 98, 22, 26, + /* 50 */ 24, 16, 12, 11, 1, 13, 13, 24, 16, 23, + /* 60 */ 11, 33, 34, 13, 97, 8, 27, 32, 98, 7, + /* 70 */ 26, 3, 4, 5, 3, 4, 5, 3, 83, 4, + /* 80 */ 5, 3, 63, 5, 3, 62, 12, 2, 86, 13, + /* 90 */ 9, 30, 10, 10, 54, 57, 75, 78, 78, 53, + /* 100 */ 57, 15, 82, 82, 71, +}; +static const fts5YYCODETYPE fts5yy_lookahead[] = { + /* 0 */ 16, 17, 18, 19, 20, 22, 22, 24, 24, 17, + /* 10 */ 18, 19, 20, 7, 22, 9, 24, 17, 18, 19, + /* 20 */ 20, 9, 22, 9, 24, 13, 17, 18, 19, 20, + /* 30 */ 26, 22, 24, 24, 17, 18, 19, 20, 15, 22, + /* 40 */ 9, 24, 17, 18, 19, 20, 26, 22, 21, 24, + /* 50 */ 6, 7, 9, 9, 10, 12, 12, 6, 7, 21, + /* 60 */ 9, 24, 25, 12, 18, 5, 20, 14, 22, 5, + /* 70 */ 24, 3, 1, 2, 3, 1, 2, 3, 0, 1, + /* 80 */ 2, 3, 11, 2, 3, 11, 9, 10, 5, 12, + /* 90 */ 23, 24, 10, 10, 8, 9, 9, 15, 15, 8, + /* 100 */ 9, 9, 27, 27, 11, 27, 27, 27, 27, 27, + /* 110 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + /* 120 */ 27, +}; +#define fts5YY_SHIFT_COUNT (34) +#define fts5YY_SHIFT_MIN (0) +#define fts5YY_SHIFT_MAX (93) +static const unsigned char fts5yy_shift_ofst[] = { + /* 0 */ 44, 44, 44, 44, 44, 44, 51, 77, 43, 12, + /* 10 */ 14, 83, 82, 14, 23, 23, 31, 31, 71, 74, + /* 20 */ 78, 81, 86, 91, 6, 53, 53, 60, 64, 68, + /* 30 */ 53, 87, 92, 53, 93, +}; +#define fts5YY_REDUCE_COUNT (17) +#define fts5YY_REDUCE_MIN (-17) +#define fts5YY_REDUCE_MAX (67) +static const signed char fts5yy_reduce_ofst[] = { + /* 0 */ -16, -8, 0, 9, 17, 25, 46, -17, -17, 37, + /* 10 */ 67, 4, 4, 8, 4, 20, 27, 38, +}; +static const fts5YYACTIONTYPE fts5yy_default[] = { + /* 0 */ 80, 80, 80, 80, 80, 80, 95, 80, 80, 105, + /* 10 */ 80, 110, 110, 80, 110, 110, 80, 80, 80, 80, + /* 20 */ 80, 91, 80, 80, 80, 101, 100, 80, 80, 90, + /* 30 */ 103, 80, 80, 104, 80, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef fts5YYFALLBACK +static const fts5YYCODETYPE fts5yyFallback[] = { +}; +#endif /* fts5YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct fts5yyStackEntry { + fts5YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + fts5YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + fts5YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct fts5yyStackEntry fts5yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct fts5yyParser { + fts5yyStackEntry *fts5yytos; /* Pointer to top element of the stack */ +#ifdef fts5YYTRACKMAXSTACKDEPTH + int fts5yyhwm; /* High-water mark of the stack */ +#endif +#ifndef fts5YYNOERRORRECOVERY + int fts5yyerrcnt; /* Shifts left before out of the error */ +#endif + sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ + sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ +#if fts5YYSTACKDEPTH<=0 + int fts5yystksz; /* Current side of the stack */ + fts5yyStackEntry *fts5yystack; /* The parser's stack */ + fts5yyStackEntry fts5yystk0; /* First stack entry */ +#else + fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct fts5yyParser fts5yyParser; + +#ifndef NDEBUG +#include "libc/assert.h" +#include "libc/stdio/stdio.h" +static FILE *fts5yyTraceFILE = 0; +static char *fts5yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + fts5yyTraceFILE = TraceFILE; + fts5yyTracePrompt = zTracePrompt; + if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0; + else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(fts5YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const fts5yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "NOT", + /* 4 */ "TERM", + /* 5 */ "COLON", + /* 6 */ "MINUS", + /* 7 */ "LCP", + /* 8 */ "RCP", + /* 9 */ "STRING", + /* 10 */ "LP", + /* 11 */ "RP", + /* 12 */ "CARET", + /* 13 */ "COMMA", + /* 14 */ "PLUS", + /* 15 */ "STAR", + /* 16 */ "input", + /* 17 */ "expr", + /* 18 */ "cnearset", + /* 19 */ "exprlist", + /* 20 */ "colset", + /* 21 */ "colsetlist", + /* 22 */ "nearset", + /* 23 */ "nearphrases", + /* 24 */ "phrase", + /* 25 */ "neardist_opt", + /* 26 */ "star_opt", +}; +#endif /* defined(fts5YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const fts5yyRuleName[] = { + /* 0 */ "input ::= expr", + /* 1 */ "colset ::= MINUS LCP colsetlist RCP", + /* 2 */ "colset ::= LCP colsetlist RCP", + /* 3 */ "colset ::= STRING", + /* 4 */ "colset ::= MINUS STRING", + /* 5 */ "colsetlist ::= colsetlist STRING", + /* 6 */ "colsetlist ::= STRING", + /* 7 */ "expr ::= expr AND expr", + /* 8 */ "expr ::= expr OR expr", + /* 9 */ "expr ::= expr NOT expr", + /* 10 */ "expr ::= colset COLON LP expr RP", + /* 11 */ "expr ::= LP expr RP", + /* 12 */ "expr ::= exprlist", + /* 13 */ "exprlist ::= cnearset", + /* 14 */ "exprlist ::= exprlist cnearset", + /* 15 */ "cnearset ::= nearset", + /* 16 */ "cnearset ::= colset COLON nearset", + /* 17 */ "nearset ::= phrase", + /* 18 */ "nearset ::= CARET phrase", + /* 19 */ "nearset ::= STRING LP nearphrases neardist_opt RP", + /* 20 */ "nearphrases ::= phrase", + /* 21 */ "nearphrases ::= nearphrases phrase", + /* 22 */ "neardist_opt ::=", + /* 23 */ "neardist_opt ::= COMMA STRING", + /* 24 */ "phrase ::= phrase PLUS STRING star_opt", + /* 25 */ "phrase ::= STRING star_opt", + /* 26 */ "star_opt ::= STAR", + /* 27 */ "star_opt ::=", +}; +#endif /* NDEBUG */ + + +#if fts5YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int fts5yyGrowStack(fts5yyParser *p){ + int newSize; + int idx; + fts5yyStackEntry *pNew; + + newSize = p->fts5yystksz*2 + 100; + idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; + if( p->fts5yystack==&p->fts5yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->fts5yystk0; + }else{ + pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, p->fts5yystksz, newSize); + } +#endif + p->fts5yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef fts5YYMALLOCARGTYPE +# define fts5YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PDECL){ + fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yypRawParser; + sqlite3Fts5ParserCTX_STORE +#ifdef fts5YYTRACKMAXSTACKDEPTH + fts5yypParser->fts5yyhwm = 0; +#endif +#if fts5YYSTACKDEPTH<=0 + fts5yypParser->fts5yytos = NULL; + fts5yypParser->fts5yystack = NULL; + fts5yypParser->fts5yystksz = 0; + if( fts5yyGrowStack(fts5yypParser) ){ + fts5yypParser->fts5yystack = &fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystksz = 1; + } +#endif +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; + fts5yypParser->fts5yystack[0].stateno = 0; + fts5yypParser->fts5yystack[0].major = 0; +#if fts5YYSTACKDEPTH>0 + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; +#endif +} + +#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Fts5Parser and sqlite3Fts5ParserFree. +*/ +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE) sqlite3Fts5ParserCTX_PDECL){ + fts5yyParser *fts5yypParser; + fts5yypParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) ); + if( fts5yypParser ){ + sqlite3Fts5ParserCTX_STORE + sqlite3Fts5ParserInit(fts5yypParser sqlite3Fts5ParserCTX_PARAM); + } + return (void*)fts5yypParser; +} +#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void fts5yy_destructor( + fts5yyParser *fts5yypParser, /* The parser */ + fts5YYCODETYPE fts5yymajor, /* Type code for object to destroy */ + fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */ +){ + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH + switch( fts5yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + case 16: /* input */ +{ +#line 83 "fts5parse.y" + (void)pParse; +#line 586 "fts5parse.c" +} + break; + case 17: /* expr */ + case 18: /* cnearset */ + case 19: /* exprlist */ +{ +#line 89 "fts5parse.y" + sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy24)); +#line 595 "fts5parse.c" +} + break; + case 20: /* colset */ + case 21: /* colsetlist */ +{ +#line 93 "fts5parse.y" + sqlite3_free((fts5yypminor->fts5yy11)); +#line 603 "fts5parse.c" +} + break; + case 22: /* nearset */ + case 23: /* nearphrases */ +{ +#line 148 "fts5parse.y" + sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy46)); +#line 611 "fts5parse.c" +} + break; + case 24: /* phrase */ +{ +#line 183 "fts5parse.y" + sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy53)); +#line 618 "fts5parse.c" +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ + fts5yyStackEntry *fts5yytos; + assert( pParser->fts5yytos!=0 ); + assert( pParser->fts5yytos > pParser->fts5yystack ); + fts5yytos = pParser->fts5yytos--; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +static void sqlite3Fts5ParserFinalize(void *p){ + fts5yyParser *pParser = (fts5yyParser*)p; + while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); +#if fts5YYSTACKDEPTH<=0 + if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); +#endif +} + +#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +static void sqlite3Fts5ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef fts5YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + sqlite3Fts5ParserFinalize(p); + (*freeProc)(p); +} +#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef fts5YYTRACKMAXSTACKDEPTH +static int sqlite3Fts5ParserStackPeak(void *p){ + fts5yyParser *pParser = (fts5yyParser*)p; + return pParser->fts5yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element fts5yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(fts5YYCOVERAGE) +static unsigned char fts5yycoverage[fts5YYNSTATE][fts5YYNFTS5TOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(fts5YYCOVERAGE) +static int sqlite3Fts5ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenofts5YY_MAX_SHIFT ) return stateno; + assert( stateno <= fts5YY_SHIFT_COUNT ); +#if defined(fts5YYCOVERAGE) + fts5yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = fts5yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=fts5YY_ACTTAB_COUNT ); + assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD ); + assert( iLookAhead!=fts5YYNOCODE ); + assert( iLookAhead < fts5YYNFTS5TOKEN ); + i += iLookAhead; + assert( i<(int)fts5YY_NLOOKAHEAD ); + if( fts5yy_lookahead[i]!=iLookAhead ){ +#ifdef fts5YYFALLBACK + fts5YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]); + } +#endif + assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef fts5YYWILDCARD + { + int j = i - iLookAhead + fts5YYWILDCARD; + assert( j<(int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])) ); + if( fts5yy_lookahead[j]==fts5YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], + fts5yyTokenName[fts5YYWILDCARD]); + } +#endif /* NDEBUG */ + return fts5yy_action[j]; + } + } +#endif /* fts5YYWILDCARD */ + return fts5yy_default[stateno]; + }else{ + assert( i>=0 && i<(int)(sizeof(fts5yy_action)/sizeof(fts5yy_action[0])) ); + return fts5yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static fts5YYACTIONTYPE fts5yy_find_reduce_action( + fts5YYACTIONTYPE stateno, /* Current state number */ + fts5YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef fts5YYERRORSYMBOL + if( stateno>fts5YY_REDUCE_COUNT ){ + return fts5yy_default[stateno]; + } +#else + assert( stateno<=fts5YY_REDUCE_COUNT ); +#endif + i = fts5yy_reduce_ofst[stateno]; + assert( iLookAhead!=fts5YYNOCODE ); + i += iLookAhead; +#ifdef fts5YYERRORSYMBOL + if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ + return fts5yy_default[stateno]; + } +#else + assert( i>=0 && ifts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +#line 36 "fts5parse.y" + + sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); +#line 839 "fts5parse.c" +/******** End %stack_overflow code ********************************************/ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3Fts5ParserCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState, const char *zTag){ + if( fts5yyTraceFILE ){ + if( fts5yyNewStatefts5yytos->major], + fts5yyNewState); + }else{ + fprintf(fts5yyTraceFILE,"%s%s '%s', pending reduce %d\n", + fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major], + fts5yyNewState - fts5YY_MIN_REDUCE); + } + } +} +#else +# define fts5yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void fts5yy_shift( + fts5yyParser *fts5yypParser, /* The parser to be shifted */ + fts5YYACTIONTYPE fts5yyNewState, /* The new state to shift in */ + fts5YYCODETYPE fts5yyMajor, /* The major token to shift in */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyMinor /* The minor token to shift in */ +){ + fts5yyStackEntry *fts5yytos; + fts5yypParser->fts5yytos++; +#ifdef fts5YYTRACKMAXSTACKDEPTH + if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ + fts5yypParser->fts5yyhwm++; + assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); + } +#endif +#if fts5YYSTACKDEPTH>0 + if( fts5yypParser->fts5yytos>fts5yypParser->fts5yystackEnd ){ + fts5yypParser->fts5yytos--; + fts5yyStackOverflow(fts5yypParser); + return; + } +#else + if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ + if( fts5yyGrowStack(fts5yypParser) ){ + fts5yypParser->fts5yytos--; + fts5yyStackOverflow(fts5yypParser); + return; + } + } +#endif + if( fts5yyNewState > fts5YY_MAX_SHIFT ){ + fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; + } + fts5yytos = fts5yypParser->fts5yytos; + fts5yytos->stateno = fts5yyNewState; + fts5yytos->major = fts5yyMajor; + fts5yytos->minor.fts5yy0 = fts5yyMinor; + fts5yyTraceShift(fts5yypParser, fts5yyNewState, "Shift"); +} + +/* For rule J, fts5yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const fts5YYCODETYPE fts5yyRuleInfoLhs[] = { + 16, /* (0) input ::= expr */ + 20, /* (1) colset ::= MINUS LCP colsetlist RCP */ + 20, /* (2) colset ::= LCP colsetlist RCP */ + 20, /* (3) colset ::= STRING */ + 20, /* (4) colset ::= MINUS STRING */ + 21, /* (5) colsetlist ::= colsetlist STRING */ + 21, /* (6) colsetlist ::= STRING */ + 17, /* (7) expr ::= expr AND expr */ + 17, /* (8) expr ::= expr OR expr */ + 17, /* (9) expr ::= expr NOT expr */ + 17, /* (10) expr ::= colset COLON LP expr RP */ + 17, /* (11) expr ::= LP expr RP */ + 17, /* (12) expr ::= exprlist */ + 19, /* (13) exprlist ::= cnearset */ + 19, /* (14) exprlist ::= exprlist cnearset */ + 18, /* (15) cnearset ::= nearset */ + 18, /* (16) cnearset ::= colset COLON nearset */ + 22, /* (17) nearset ::= phrase */ + 22, /* (18) nearset ::= CARET phrase */ + 22, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */ + 23, /* (20) nearphrases ::= phrase */ + 23, /* (21) nearphrases ::= nearphrases phrase */ + 25, /* (22) neardist_opt ::= */ + 25, /* (23) neardist_opt ::= COMMA STRING */ + 24, /* (24) phrase ::= phrase PLUS STRING star_opt */ + 24, /* (25) phrase ::= STRING star_opt */ + 26, /* (26) star_opt ::= STAR */ + 26, /* (27) star_opt ::= */ +}; + +/* For rule J, fts5yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char fts5yyRuleInfoNRhs[] = { + -1, /* (0) input ::= expr */ + -4, /* (1) colset ::= MINUS LCP colsetlist RCP */ + -3, /* (2) colset ::= LCP colsetlist RCP */ + -1, /* (3) colset ::= STRING */ + -2, /* (4) colset ::= MINUS STRING */ + -2, /* (5) colsetlist ::= colsetlist STRING */ + -1, /* (6) colsetlist ::= STRING */ + -3, /* (7) expr ::= expr AND expr */ + -3, /* (8) expr ::= expr OR expr */ + -3, /* (9) expr ::= expr NOT expr */ + -5, /* (10) expr ::= colset COLON LP expr RP */ + -3, /* (11) expr ::= LP expr RP */ + -1, /* (12) expr ::= exprlist */ + -1, /* (13) exprlist ::= cnearset */ + -2, /* (14) exprlist ::= exprlist cnearset */ + -1, /* (15) cnearset ::= nearset */ + -3, /* (16) cnearset ::= colset COLON nearset */ + -1, /* (17) nearset ::= phrase */ + -2, /* (18) nearset ::= CARET phrase */ + -5, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */ + -1, /* (20) nearphrases ::= phrase */ + -2, /* (21) nearphrases ::= nearphrases phrase */ + 0, /* (22) neardist_opt ::= */ + -2, /* (23) neardist_opt ::= COMMA STRING */ + -4, /* (24) phrase ::= phrase PLUS STRING star_opt */ + -2, /* (25) phrase ::= STRING star_opt */ + -1, /* (26) star_opt ::= STAR */ + 0, /* (27) star_opt ::= */ +}; + +static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The fts5yyLookahead and fts5yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The fts5yyLookahead will be fts5YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static fts5YYACTIONTYPE fts5yy_reduce( + fts5yyParser *fts5yypParser, /* The parser */ + unsigned int fts5yyruleno, /* Number of the rule by which to reduce */ + int fts5yyLookahead, /* Lookahead token, or fts5YYNOCODE if none */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyLookaheadToken /* Value of the lookahead token */ + sqlite3Fts5ParserCTX_PDECL /* %extra_context */ +){ + int fts5yygoto; /* The next state */ + fts5YYACTIONTYPE fts5yyact; /* The next action */ + fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */ + int fts5yysize; /* Amount to pop the stack */ + sqlite3Fts5ParserARG_FETCH + (void)fts5yyLookahead; + (void)fts5yyLookaheadToken; + fts5yymsp = fts5yypParser->fts5yytos; + + switch( fts5yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + fts5YYMINORTYPE fts5yylhsminor; + case 0: /* input ::= expr */ +#line 82 "fts5parse.y" +{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy24); } +#line 1016 "fts5parse.c" + break; + case 1: /* colset ::= MINUS LCP colsetlist RCP */ +#line 97 "fts5parse.y" +{ + fts5yymsp[-3].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); +} +#line 1023 "fts5parse.c" + break; + case 2: /* colset ::= LCP colsetlist RCP */ +#line 100 "fts5parse.y" +{ fts5yymsp[-2].minor.fts5yy11 = fts5yymsp[-1].minor.fts5yy11; } +#line 1028 "fts5parse.c" + break; + case 3: /* colset ::= STRING */ +#line 101 "fts5parse.y" +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); +} +#line 1035 "fts5parse.c" + fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; + break; + case 4: /* colset ::= MINUS STRING */ +#line 104 "fts5parse.y" +{ + fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); + fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); +} +#line 1044 "fts5parse.c" + break; + case 5: /* colsetlist ::= colsetlist STRING */ +#line 109 "fts5parse.y" +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy11, &fts5yymsp[0].minor.fts5yy0); } +#line 1050 "fts5parse.c" + fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11; + break; + case 6: /* colsetlist ::= STRING */ +#line 111 "fts5parse.y" +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); +} +#line 1058 "fts5parse.c" + fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; + break; + case 7: /* expr ::= expr AND expr */ +#line 115 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} +#line 1066 "fts5parse.c" + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 8: /* expr ::= expr OR expr */ +#line 118 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} +#line 1074 "fts5parse.c" + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 9: /* expr ::= expr NOT expr */ +#line 121 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} +#line 1082 "fts5parse.c" + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 10: /* expr ::= colset COLON LP expr RP */ +#line 125 "fts5parse.y" +{ + sqlite3Fts5ParseSetColset(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[-4].minor.fts5yy11); + fts5yylhsminor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24; +} +#line 1091 "fts5parse.c" + fts5yymsp[-4].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 11: /* expr ::= LP expr RP */ +#line 129 "fts5parse.y" +{fts5yymsp[-2].minor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;} +#line 1097 "fts5parse.c" + break; + case 12: /* expr ::= exprlist */ + case 13: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==13); +#line 130 "fts5parse.y" +{fts5yylhsminor.fts5yy24 = fts5yymsp[0].minor.fts5yy24;} +#line 1103 "fts5parse.c" + fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 14: /* exprlist ::= exprlist cnearset */ +#line 133 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24); +} +#line 1111 "fts5parse.c" + fts5yymsp[-1].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 15: /* cnearset ::= nearset */ +#line 137 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); +} +#line 1119 "fts5parse.c" + fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 16: /* cnearset ::= colset COLON nearset */ +#line 140 "fts5parse.y" +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); + sqlite3Fts5ParseSetColset(pParse, fts5yylhsminor.fts5yy24, fts5yymsp[-2].minor.fts5yy11); +} +#line 1128 "fts5parse.c" + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 17: /* nearset ::= phrase */ +#line 151 "fts5parse.y" +{ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); } +#line 1134 "fts5parse.c" + fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 18: /* nearset ::= CARET phrase */ +#line 152 "fts5parse.y" +{ + sqlite3Fts5ParseSetCaret(fts5yymsp[0].minor.fts5yy53); + fts5yymsp[-1].minor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); +} +#line 1143 "fts5parse.c" + break; + case 19: /* nearset ::= STRING LP nearphrases neardist_opt RP */ +#line 156 "fts5parse.y" +{ + sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0); + sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy46, &fts5yymsp[-1].minor.fts5yy0); + fts5yylhsminor.fts5yy46 = fts5yymsp[-2].minor.fts5yy46; +} +#line 1152 "fts5parse.c" + fts5yymsp[-4].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 20: /* nearphrases ::= phrase */ +#line 162 "fts5parse.y" +{ + fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); +} +#line 1160 "fts5parse.c" + fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 21: /* nearphrases ::= nearphrases phrase */ +#line 165 "fts5parse.y" +{ + fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy46, fts5yymsp[0].minor.fts5yy53); +} +#line 1168 "fts5parse.c" + fts5yymsp[-1].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 22: /* neardist_opt ::= */ +#line 172 "fts5parse.y" +{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; } +#line 1174 "fts5parse.c" + break; + case 23: /* neardist_opt ::= COMMA STRING */ +#line 173 "fts5parse.y" +{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; } +#line 1179 "fts5parse.c" + break; + case 24: /* phrase ::= phrase PLUS STRING star_opt */ +#line 185 "fts5parse.y" +{ + fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy53, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); +} +#line 1186 "fts5parse.c" + fts5yymsp[-3].minor.fts5yy53 = fts5yylhsminor.fts5yy53; + break; + case 25: /* phrase ::= STRING star_opt */ +#line 188 "fts5parse.y" +{ + fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); +} +#line 1194 "fts5parse.c" + fts5yymsp[-1].minor.fts5yy53 = fts5yylhsminor.fts5yy53; + break; + case 26: /* star_opt ::= STAR */ +#line 196 "fts5parse.y" +{ fts5yymsp[0].minor.fts5yy4 = 1; } +#line 1200 "fts5parse.c" + break; + case 27: /* star_opt ::= */ +#line 197 "fts5parse.y" +{ fts5yymsp[1].minor.fts5yy4 = 0; } +#line 1205 "fts5parse.c" + break; + default: + break; +/********** End reduce actions ************************************************/ + }; + assert( fts5yyrulenofts5YY_MAX_SHIFT && fts5yyact<=fts5YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( fts5yyact!=fts5YY_ERROR_ACTION ); + + fts5yymsp += fts5yysize+1; + fts5yypParser->fts5yytos = fts5yymsp; + fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact; + fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto; + fts5yyTraceShift(fts5yypParser, fts5yyact, "... then shift"); + return fts5yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef fts5YYNOERRORRECOVERY +static void fts5yy_parse_failed( + fts5yyParser *fts5yypParser /* The parser */ +){ + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt); + } +#endif + while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE +} +#endif /* fts5YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void fts5yy_syntax_error( + fts5yyParser *fts5yypParser, /* The parser */ + int fts5yymajor, /* The major type of the error token */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The minor type of the error token */ +){ + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH +#define FTS5TOKEN fts5yyminor +/************ Begin %syntax_error code ****************************************/ +#line 30 "fts5parse.y" + + UNUSED_PARAM(fts5yymajor); /* Silence a compiler warning */ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKEN.n,FTS5TOKEN.p + ); +#line 1273 "fts5parse.c" +/************ End %syntax_error code ******************************************/ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void fts5yy_accept( + fts5yyParser *fts5yypParser /* The parser */ +){ + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt); + } +#endif +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + assert( fts5yypParser->fts5yytos==fts5yypParser->fts5yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3Fts5ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +static void sqlite3Fts5Parser( + void *fts5yyp, /* The parser */ + int fts5yymajor, /* The major token code number */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The value for the token */ + sqlite3Fts5ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + fts5YYMINORTYPE fts5yyminorunion; + fts5YYACTIONTYPE fts5yyact; /* The parser action. */ +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) + int fts5yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef fts5YYERRORSYMBOL + int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */ +#endif + fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yyp; /* The parser */ + sqlite3Fts5ParserCTX_FETCH + sqlite3Fts5ParserARG_STORE + + assert( fts5yypParser->fts5yytos!=0 ); +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) + fts5yyendofinput = (fts5yymajor==0); +#endif + + fts5yyact = fts5yypParser->fts5yytos->stateno; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + if( fts5yyact < fts5YY_MIN_REDUCE ){ + fprintf(fts5yyTraceFILE,"%sInput '%s' in state %d\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact); + }else{ + fprintf(fts5yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact-fts5YY_MIN_REDUCE); + } + } +#endif + + while(1){ /* Exit by "break" */ + assert( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystack ); + assert( fts5yyact==fts5yypParser->fts5yytos->stateno ); + fts5yyact = fts5yy_find_shift_action((fts5YYCODETYPE)fts5yymajor,fts5yyact); + if( fts5yyact >= fts5YY_MIN_REDUCE ){ + unsigned int fts5yyruleno = fts5yyact - fts5YY_MIN_REDUCE; /* Reduce by this rule */ + assert( fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ); +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + int fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno]; + if( fts5yysize ){ + fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + fts5yyTracePrompt, + fts5yyruleno, fts5yyRuleName[fts5yyruleno], + fts5yyrulenofts5yytos[fts5yysize].stateno); + }else{ + fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s.\n", + fts5yyTracePrompt, fts5yyruleno, fts5yyRuleName[fts5yyruleno], + fts5yyrulenofts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ + fts5yypParser->fts5yyhwm++; + assert( fts5yypParser->fts5yyhwm == + (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); + } +#endif +#if fts5YYSTACKDEPTH>0 + if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ + fts5yyStackOverflow(fts5yypParser); + break; + } +#else + if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ + if( fts5yyGrowStack(fts5yypParser) ){ + fts5yyStackOverflow(fts5yypParser); + break; + } + } +#endif + } + fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); + }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ + fts5yy_shift(fts5yypParser,fts5yyact,(fts5YYCODETYPE)fts5yymajor,fts5yyminor); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt--; +#endif + break; + }else if( fts5yyact==fts5YY_ACCEPT_ACTION ){ + fts5yypParser->fts5yytos--; + fts5yy_accept(fts5yypParser); + return; + }else{ + assert( fts5yyact == fts5YY_ERROR_ACTION ); + fts5yyminorunion.fts5yy0 = fts5yyminor; +#ifdef fts5YYERRORSYMBOL + int fts5yymx; +#endif +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt); + } +#endif +#ifdef fts5YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( fts5yypParser->fts5yyerrcnt<0 ){ + fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor); + } + fts5yymx = fts5yypParser->fts5yytos->major; + if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); + } +#endif + fts5yy_destructor(fts5yypParser, (fts5YYCODETYPE)fts5yymajor, &fts5yyminorunion); + fts5yymajor = fts5YYNOCODE; + }else{ + while( fts5yypParser->fts5yytos >= fts5yypParser->fts5yystack + && (fts5yyact = fts5yy_find_reduce_action( + fts5yypParser->fts5yytos->stateno, + fts5YYERRORSYMBOL)) > fts5YY_MAX_SHIFTREDUCE + ){ + fts5yy_pop_parser_stack(fts5yypParser); + } + if( fts5yypParser->fts5yytos < fts5yypParser->fts5yystack || fts5yymajor==0 ){ + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + fts5yy_parse_failed(fts5yypParser); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + fts5yymajor = fts5YYNOCODE; + }else if( fts5yymx!=fts5YYERRORSYMBOL ){ + fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor); + } + } + fts5yypParser->fts5yyerrcnt = 3; + fts5yyerrorhit = 1; + if( fts5yymajor==fts5YYNOCODE ) break; + fts5yyact = fts5yypParser->fts5yytos->stateno; +#elif defined(fts5YYNOERRORRECOVERY) + /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + break; +#else /* fts5YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( fts5yypParser->fts5yyerrcnt<=0 ){ + fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); + } + fts5yypParser->fts5yyerrcnt = 3; + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); + if( fts5yyendofinput ){ + fts5yy_parse_failed(fts5yypParser); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + } + break; +#endif + } + } +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fts5yyStackEntry *i; + char cDiv = '['; + fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt); + for(i=&fts5yypParser->fts5yystack[1]; i<=fts5yypParser->fts5yytos; i++){ + fprintf(fts5yyTraceFILE,"%c%s", cDiv, fts5yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(fts5yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +static int sqlite3Fts5ParserFallback(int iToken){ +#ifdef fts5YYFALLBACK + assert( iToken<(int)(sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])) ); + return fts5yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} + +#line 1 "fts5_aux.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ +#include "libc/math.h" /* amalgamator: keep */ + +/* +** Object used to iterate through all "coalesced phrase instances" in +** a single column of the current row. If the phrase instances in the +** column being considered do not overlap, this object simply iterates +** through them. Or, if they do overlap (share one or more tokens in +** common), each set of overlapping instances is treated as a single +** match. See documentation for the highlight() auxiliary function for +** details. +** +** Usage is: +** +** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter); +** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter); +** rc = fts5CInstIterNext(&iter) +** ){ +** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd); +** } +** +*/ +typedef struct CInstIter CInstIter; +struct CInstIter { + const Fts5ExtensionApi *pApi; /* API offered by current FTS version */ + Fts5Context *pFts; /* First arg to pass to pApi functions */ + int iCol; /* Column to search */ + int iInst; /* Next phrase instance index */ + int nInst; /* Total number of phrase instances */ + + /* Output variables */ + int iStart; /* First token in coalesced phrase instance */ + int iEnd; /* Last token in coalesced phrase instance */ +}; + +/* +** Advance the iterator to the next coalesced phrase instance. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +static int fts5CInstIterNext(CInstIter *pIter){ + int rc = SQLITE_OK; + pIter->iStart = -1; + pIter->iEnd = -1; + + while( rc==SQLITE_OK && pIter->iInstnInst ){ + int ip; int ic; int io; + rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + if( ic==pIter->iCol ){ + int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip); + if( pIter->iStart<0 ){ + pIter->iStart = io; + pIter->iEnd = iEnd; + }else if( io<=pIter->iEnd ){ + if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd; + }else{ + break; + } + } + pIter->iInst++; + } + } + + return rc; +} + +/* +** Initialize the iterator object indicated by the final parameter to +** iterate through coalesced phrase instances in column iCol. +*/ +static int fts5CInstIterInit( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + int iCol, + CInstIter *pIter +){ + int rc; + + memset(pIter, 0, sizeof(CInstIter)); + pIter->pApi = pApi; + pIter->pFts = pFts; + pIter->iCol = iCol; + rc = pApi->xInstCount(pFts, &pIter->nInst); + + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(pIter); + } + + return rc; +} + + + +/************************************************************************* +** Start of highlight() implementation. +*/ +typedef struct HighlightContext HighlightContext; +struct HighlightContext { + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iRangeStart; /* First token to include */ + int iRangeEnd; /* If non-zero, last token to include */ + const char *zOpen; /* Opening highlight */ + const char *zClose; /* Closing highlight */ + const char *zIn; /* Input text */ + int nIn; /* Size of input text in bytes */ + int iOff; /* Current offset within zIn[] */ + char *zOut; /* Output value */ +}; + +/* +** Append text to the HighlightContext output string - p->zOut. Argument +** z points to a buffer containing n bytes of text to append. If n is +** negative, everything up until the first '\0' is appended to the output. +** +** If *pRc is set to any value other than SQLITE_OK when this function is +** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, +** *pRc is set to an error code before returning. +*/ +static void fts5HighlightAppend( + int *pRc, + HighlightContext *p, + const char *z, int n +){ + if( *pRc==SQLITE_OK && z ){ + if( n<0 ) n = (int)strlen(z); + p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); + if( p->zOut==0 ) *pRc = SQLITE_NOMEM; + } +} + +/* +** Tokenizer callback used by implementation of highlight() function. +*/ +static int fts5HighlightCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + HighlightContext *p = (HighlightContext*)pContext; + int rc = SQLITE_OK; + int iPos; + + UNUSED_PARAM2(pToken, nToken); + + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; + + if( p->iRangeEnd>0 ){ + if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; + if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; + } + + if( iPos==p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); + fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->iOff = iStartOff; + } + + if( iPos==p->iter.iEnd ){ + if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + fts5HighlightAppend(&rc, p, p->zOpen, -1); + } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->iOff = iEndOff; + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(&p->iter); + } + } + + if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + if( iPos>=p->iter.iStart && iPositer.iEnd ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + } + } + + return rc; +} + +/* +** Implementation of highlight() function. +*/ +static void fts5HighlightFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc; + int iCol; + + if( nVal!=3 ){ + const char *zErr = "wrong number of arguments to function highlight()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + memset(&ctx, 0, sizeof(HighlightContext)); + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); + + if( ctx.zIn ){ + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); + } + + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + } + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + } + sqlite3_free(ctx.zOut); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} +/* +** End of highlight() implementation. +**************************************************************************/ + +/* +** Context object passed to the fts5SentenceFinderCb() function. +*/ +typedef struct Fts5SFinder Fts5SFinder; +struct Fts5SFinder { + int iPos; /* Current token position */ + int nFirstAlloc; /* Allocated size of aFirst[] */ + int nFirst; /* Number of entries in aFirst[] */ + int *aFirst; /* Array of first token in each sentence */ + const char *zDoc; /* Document being tokenized */ +}; + +/* +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an +** error occurs. +*/ +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ + if( p->nFirstAlloc==p->nFirst ){ + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; + int *aNew; + + aNew = (int*)sqlite3_realloc64(p->aFirst, nNew*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + p->aFirst = aNew; + p->nFirstAlloc = nNew; + } + p->aFirst[p->nFirst++] = iAdd; + return SQLITE_OK; +} + +/* +** This function is an xTokenize() callback used by the auxiliary snippet() +** function. Its job is to identify tokens that are the first in a sentence. +** For each such token, an entry is added to the SFinder.aFirst[] array. +*/ +static int fts5SentenceFinderCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + int rc = SQLITE_OK; + + UNUSED_PARAM2(pToken, nToken); + UNUSED_PARAM(iEndOff); + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + Fts5SFinder *p = (Fts5SFinder*)pContext; + if( p->iPos>0 ){ + int i; + char c = 0; + for(i=iStartOff-1; i>=0; i--){ + c = p->zDoc[i]; + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; + } + if( i!=iStartOff-1 && (c=='.' || c==':') ){ + rc = fts5SentenceFinderAdd(p, p->iPos); + } + }else{ + rc = fts5SentenceFinderAdd(p, 0); + } + p->iPos++; + } + return rc; +} + +static int fts5SnippetScore( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + int nDocsize, /* Size of column in tokens */ + unsigned char *aSeen, /* Array with one element per query phrase */ + int iCol, /* Column to score */ + int iPos, /* Starting offset to score */ + int nToken, /* Max tokens per snippet */ + int *pnScore, /* OUT: Score */ + int *piPos /* OUT: Adjusted offset */ +){ + int rc; + int i; + int ip = 0; + int ic = 0; + int iOff = 0; + int iFirst = -1; + int nInst; + int nScore = 0; + int iLast = 0; + sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken; + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; ixInst(pFts, i, &ip, &ic, &iOff); + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOffxPhraseSize(pFts, ip); + } + } + + *pnScore = nScore; + if( piPos ){ + sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; + if( iAdj<0 ) iAdj = 0; + *piPos = (int)iAdj; + } + + return rc; +} + +/* +** Return the value in pVal interpreted as utf-8 text. Except, if pVal +** contains a NULL value, return a pointer to a static string zero +** bytes in length instead of a NULL pointer. +*/ +static const char *fts5ValueToText(sqlite3_value *pVal){ + const char *zRet = (const char*)sqlite3_value_text(pVal); + return zRet ? zRet : ""; +} + +/* +** Implementation of snippet() function. +*/ +static void fts5SnippetFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc = SQLITE_OK; /* Return code */ + int iCol; /* 1st argument to snippet() */ + const char *zEllips; /* 4th argument to snippet() */ + int nToken; /* 5th argument to snippet() */ + int nInst = 0; /* Number of instance matches this row */ + int i; /* Used to iterate through instances */ + int nPhrase; /* Number of phrases in query */ + unsigned char *aSeen; /* Array of "seen instance" flags */ + int iBestCol; /* Column containing best snippet */ + int iBestStart = 0; /* First token of best snippet */ + int nBestScore = 0; /* Score of best snippet */ + int nColSize = 0; /* Total size of iBestCol in tokens */ + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ + int nCol; + + if( nVal!=5 ){ + const char *zErr = "wrong number of arguments to function snippet()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + nCol = pApi->xColumnCount(pFts); + memset(&ctx, 0, sizeof(HighlightContext)); + iCol = sqlite3_value_int(apVal[0]); + ctx.zOpen = fts5ValueToText(apVal[1]); + ctx.zClose = fts5ValueToText(apVal[2]); + zEllips = fts5ValueToText(apVal[3]); + nToken = sqlite3_value_int(apVal[4]); + + iBestCol = (iCol>=0 ? iCol : 0); + nPhrase = pApi->xPhraseCount(pFts); + aSeen = sqlite3_malloc(nPhrase); + if( aSeen==0 ){ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + rc = pApi->xInstCount(pFts, &nInst); + } + + memset(&sFinder, 0, sizeof(Fts5SFinder)); + for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize(pFts, + sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + ); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnSize(pFts, i, &nDocsize); + if( rc!=SQLITE_OK ) break; + + for(ii=0; rc==SQLITE_OK && iixInst(pFts, ii, &ip, &ic, &io); + if( ic!=i ) continue; + if( io>nDocsize ) rc = FTS5_CORRUPT; + if( rc!=SQLITE_OK ) continue; + memset(aSeen, 0, nPhrase); + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, + io, nToken, &nScore, &iAdj + ); + if( rc==SQLITE_OK && nScore>nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = iAdj; + nColSize = nDocsize; + } + + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ + for(jj=0; jj<(sFinder.nFirst-1); jj++){ + if( sFinder.aFirst[jj+1]>io ) break; + } + + if( sFinder.aFirst[jj]nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = sFinder.aFirst[jj]; + nColSize = nDocsize; + } + } + } + } + } + } + + if( rc==SQLITE_OK ){ + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); + } + if( rc==SQLITE_OK && nColSize==0 ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } + if( ctx.zIn ){ + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); + } + + ctx.iRangeStart = iBestStart; + ctx.iRangeEnd = iBestStart + nToken - 1; + + if( iBestStart>0 ){ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + + /* Advance iterator ctx.iter so that it points to the first coalesced + ** phrase instance at or following position iBestStart. */ + while( ctx.iter.iStart>=0 && ctx.iter.iStartxTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + } + if( ctx.iRangeEnd>=(nColSize-1) ){ + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + }else{ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + } + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); + sqlite3_free(aSeen); + sqlite3_free(sFinder.aFirst); +} + +/************************************************************************/ + +/* +** The first time the bm25() function is called for a query, an instance +** of the following structure is allocated and populated. +*/ +typedef struct Fts5Bm25Data Fts5Bm25Data; +struct Fts5Bm25Data { + int nPhrase; /* Number of phrases in query */ + double avgdl; /* Average number of tokens in each row */ + double *aIDF; /* IDF for each phrase */ + double *aFreq; /* Array used to calculate phrase freq. */ +}; + +/* +** Callback used by fts5Bm25GetData() to count the number of rows in the +** table matched by each individual phrase within the query. +*/ +static int fts5CountCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData /* Pointer to sqlite3_int64 variable */ +){ + sqlite3_int64 *pn = (sqlite3_int64*)pUserData; + UNUSED_PARAM2(pApi, pFts); + (*pn)++; + return SQLITE_OK; +} + +/* +** Set *ppData to point to the Fts5Bm25Data object for the current query. +** If the object has not already been allocated, allocate and populate it +** now. +*/ +static int fts5Bm25GetData( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Bm25Data *p; /* Object to return */ + + p = (Fts5Bm25Data*)pApi->xGetAuxdata(pFts, 0); + if( p==0 ){ + int nPhrase; /* Number of phrases in query */ + sqlite3_int64 nRow = 0; /* Number of rows in table */ + sqlite3_int64 nToken = 0; /* Number of tokens in table */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + int i; + + /* Allocate the Fts5Bm25Data object */ + nPhrase = pApi->xPhraseCount(pFts); + nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double); + p = (Fts5Bm25Data*)sqlite3_malloc64(nByte); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(p, 0, (size_t)nByte); + p->nPhrase = nPhrase; + p->aIDF = (double*)&p[1]; + p->aFreq = &p->aIDF[nPhrase]; + } + + /* Calculate the average document length for this FTS5 table */ + if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow); + assert( rc!=SQLITE_OK || nRow>0 ); + if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken); + if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow; + + /* Calculate an IDF for each phrase in the query */ + for(i=0; rc==SQLITE_OK && ixQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb); + if( rc==SQLITE_OK ){ + /* Calculate the IDF (Inverse Document Frequency) for phrase i. + ** This is done using the standard BM25 formula as found on wikipedia: + ** + ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) ) + ** + ** where "N" is the total number of documents in the set and nHit + ** is the number that contain at least one instance of the phrase + ** under consideration. + ** + ** The problem with this is that if (N < 2*nHit), the IDF is + ** negative. Which is undesirable. So the mimimum allowable IDF is + ** (1e-6) - roughly the same as a term that appears in just over + ** half of set of 5,000,000 documents. */ + double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); + if( idf<=0.0 ) idf = 1e-6; + p->aIDF[i] = idf; + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + if( rc!=SQLITE_OK ) p = 0; + } + *ppData = p; + return rc; +} + +/* +** Implementation of bm25() function. +*/ +static void fts5Bm25Function( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const double k1 = 1.2; /* Constant "k1" from BM25 formula */ + const double b = 0.75; /* Constant "b" from BM25 formula */ + int rc; /* Error code */ + double score = 0.0; /* SQL function return value */ + Fts5Bm25Data *pData; /* Values allocated/calculated once only */ + int i; /* Iterator variable */ + int nInst = 0; /* Value returned by xInstCount() */ + double D = 0.0; /* Total number of tokens in row */ + double *aFreq = 0; /* Array of phrase freq. for current row */ + + /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation) + ** for each phrase in the query for the current row. */ + rc = fts5Bm25GetData(pApi, pFts, &pData); + if( rc==SQLITE_OK ){ + aFreq = pData->aFreq; + memset(aFreq, 0, sizeof(double) * pData->nPhrase); + rc = pApi->xInstCount(pFts, &nInst); + } + for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0; + aFreq[ip] += w; + } + } + + /* Figure out the total size of the current row in tokens. */ + if( rc==SQLITE_OK ){ + int nTok; + rc = pApi->xColumnSize(pFts, -1, &nTok); + D = (double)nTok; + } + + /* Determine and return the BM25 score for the current row. Or, if an + ** error has occurred, throw an exception. */ + if( rc==SQLITE_OK ){ + for(i=0; inPhrase; i++){ + score += pData->aIDF[i] * ( + ( aFreq[i] * (k1 + 1.0) ) / + ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) + ); + } + sqlite3_result_double(pCtx, -1.0 * score); + }else{ + sqlite3_result_error_code(pCtx, rc); + } +} + +static int sqlite3Fts5AuxInit(fts5_api *pApi){ + struct Builtin { + const char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc;/* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + } aBuiltin [] = { + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + }; + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && ixCreateFunction(pApi, + aBuiltin[i].zFunc, + aBuiltin[i].pUserData, + aBuiltin[i].xFunc, + aBuiltin[i].xDestroy + ); + } + + return rc; +} + +#line 1 "fts5_buffer.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ + if( (u32)pBuf->nSpacenSpace ? pBuf->nSpace : 64; + u8 *pNew; + while( nNewp, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = (int)nNew; + pBuf->p = pNew; + } + } + return 0; +} + + +/* +** Encode value iVal as an SQLite varint and append it to the buffer object +** pBuf. If an OOM error occurs, set the error code in p. +*/ +static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ + if( fts5BufferGrow(pRc, pBuf, 9) ) return; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal); +} + +static void sqlite3Fts5Put32(u8 *aBuf, int iVal){ + aBuf[0] = (iVal>>24) & 0x00FF; + aBuf[1] = (iVal>>16) & 0x00FF; + aBuf[2] = (iVal>> 8) & 0x00FF; + aBuf[3] = (iVal>> 0) & 0x00FF; +} + +static int sqlite3Fts5Get32(const u8 *aBuf){ + return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]); +} + +/* +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static void sqlite3Fts5BufferAppendBlob( + int *pRc, + Fts5Buffer *pBuf, + u32 nData, + const u8 *pData +){ + assert_nc( *pRc || nData>=0 ); + if( nData ){ + if( fts5BufferGrow(pRc, pBuf, nData) ) return; + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; + } +} + +/* +** Append the nul-terminated string zStr to the buffer pBuf. This function +** ensures that the byte following the buffer data is set to 0x00, even +** though this byte is not included in the pBuf->n count. +*/ +static void sqlite3Fts5BufferAppendString( + int *pRc, + Fts5Buffer *pBuf, + const char *zStr +){ + int nStr = (int)strlen(zStr); + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr); + pBuf->n--; +} + +/* +** Argument zFmt is a printf() style format string. This function performs +** the printf() style processing, then appends the results to buffer pBuf. +** +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte +** following the buffer data is set to 0x00, even though this byte is not +** included in the pBuf->n count. +*/ +static void sqlite3Fts5BufferAppendPrintf( + int *pRc, + Fts5Buffer *pBuf, + char *zFmt, ... +){ + if( *pRc==SQLITE_OK ){ + char *zTmp; + va_list ap; + va_start(ap, zFmt); + zTmp = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + if( zTmp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); + sqlite3_free(zTmp); + } + } +} + +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Free any buffer allocated by pBuf. Zero the structure before returning. +*/ +static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ + sqlite3_free(pBuf->p); + memset(pBuf, 0, sizeof(Fts5Buffer)); +} + +/* +** Zero the contents of the buffer object. But do not free the associated +** memory allocation. +*/ +static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ + pBuf->n = 0; +} + +/* +** Set the buffer to contain nData/pData. If an OOM error occurs, leave an +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static void sqlite3Fts5BufferSet( + int *pRc, + Fts5Buffer *pBuf, + int nData, + const u8 *pData +){ + pBuf->n = 0; + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData); +} + +static int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +){ + int i = *pi; + if( i>=n ){ + /* EOF */ + *piOff = -1; + return 1; + }else{ + i64 iOff = *piOff; + int iVal; + fts5FastGetVarint32(a, i, iVal); + if( iVal<=1 ){ + if( iVal==0 ){ + *pi = i; + return 0; + } + fts5FastGetVarint32(a, i, iVal); + iOff = ((i64)iVal) << 32; + fts5FastGetVarint32(a, i, iVal); + if( iVal<2 ){ + /* This is a corrupt record. So stop parsing it here. */ + *piOff = -1; + return 1; + } + } + *piOff = iOff + ((iVal-2) & 0x7FFFFFFF); + *pi = i; + return 0; + } +} + + +/* +** Advance the iterator object passed as the only argument. Return true +** if the iterator reaches EOF, or false otherwise. +*/ +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ + pIter->bEof = 1; + } + return pIter->bEof; +} + +static int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +){ + memset(pIter, 0, sizeof(*pIter)); + pIter->a = a; + pIter->n = n; + sqlite3Fts5PoslistReaderNext(pIter); + return pIter->bEof; +} + +/* +** Append position iPos to the position list being accumulated in buffer +** pBuf, which must be already be large enough to hold the new data. +** The previous position written to this list is *piPrev. *piPrev is set +** to iPos before returning. +*/ +static void sqlite3Fts5PoslistSafeAppend( + Fts5Buffer *pBuf, + i64 *piPrev, + i64 iPos +){ + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; + if( (iPos & colmask) != (*piPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + *piPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); + *piPrev = iPos; +} + +static int sqlite3Fts5PoslistWriterAppend( + Fts5Buffer *pBuf, + Fts5PoslistWriter *pWriter, + i64 iPos +){ + int rc = 0; /* Initialized only to suppress erroneous warning from Clang */ + if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; + sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); + return SQLITE_OK; +} + +static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + if( nByte>0 ) *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, (size_t)nByte); + } + } + return pRet; +} + +/* +** Return a nul-terminated copy of the string indicated by pIn. If nIn +** is non-negative, then it is the length of the string in bytes. Otherwise, +** the length of the string is determined using strlen(). +** +** It is the responsibility of the caller to eventually free the returned +** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. +*/ +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + if( nIn<0 ){ + nIn = (int)strlen(pIn); + } + zRet = (char*)sqlite3_malloc(nIn+1); + if( zRet ){ + memcpy(zRet, pIn, nIn); + zRet[nIn] = '\0'; + }else{ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Return true if character 't' may be part of an FTS5 bareword, or false +** otherwise. Characters that may be part of barewords: +** +** * All non-ASCII characters, +** * The 52 upper and lower case ASCII characters, and +** * The 10 integer ASCII characters. +** * The underscore character "_" (0x5F). +** * The unicode "subsitute" character (0x1A). +*/ +static int sqlite3Fts5IsBareword(char t){ + u8 aBareword[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ + }; + + return (t & 0x80) || aBareword[(int)t]; +} + + +/************************************************************************* +*/ +typedef struct Fts5TermsetEntry Fts5TermsetEntry; +struct Fts5TermsetEntry { + char *pTerm; + int nTerm; + int iIdx; /* Index (main or aPrefix[] entry) */ + Fts5TermsetEntry *pNext; +}; + +struct Fts5Termset { + Fts5TermsetEntry *apHash[512]; +}; + +static int sqlite3Fts5TermsetNew(Fts5Termset **pp){ + int rc = SQLITE_OK; + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); + return rc; +} + +static int sqlite3Fts5TermsetAdd( + Fts5Termset *p, + int iIdx, + const char *pTerm, int nTerm, + int *pbPresent +){ + int rc = SQLITE_OK; + *pbPresent = 0; + if( p ){ + int i; + u32 hash = 13; + Fts5TermsetEntry *pEntry; + + /* Calculate a hash value for this term. This is the same hash checksum + ** used by the fts5_hash.c module. This is not important for correct + ** operation of the module, but is necessary to ensure that some tests + ** designed to produce hash table collisions really do work. */ + for(i=nTerm-1; i>=0; i--){ + hash = (hash << 3) ^ hash ^ pTerm[i]; + } + hash = (hash << 3) ^ hash ^ iIdx; + hash = hash % ArraySize(p->apHash); + + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + pEntry->iIdx = iIdx; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } + } + } + + return rc; +} + +static void sqlite3Fts5TermsetFree(Fts5Termset *p){ + if( p ){ + u32 i; + for(i=0; iapHash); i++){ + Fts5TermsetEntry *pEntry = p->apHash[i]; + while( pEntry ){ + Fts5TermsetEntry *pDel = pEntry; + pEntry = pEntry->pNext; + sqlite3_free(pDel); + } + } + sqlite3_free(p); + } +} + +#line 1 "fts5_config.c" +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +#define FTS5_DEFAULT_PAGE_SIZE 4050 +#define FTS5_DEFAULT_AUTOMERGE 4 +#define FTS5_DEFAULT_USERMERGE 4 +#define FTS5_DEFAULT_CRISISMERGE 16 +#define FTS5_DEFAULT_HASHSIZE (1024*1024) + +/* Maximum allowed page size */ +#define FTS5_MAX_PAGE_SIZE (64*1024) + +static int fts5_iswhitespace(char x){ + return (x==' '); +} + +static int fts5_isopenquote(char x){ + return (x=='"' || x=='\'' || x=='[' || x=='`'); +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a white-space character. +*/ +static const char *fts5ConfigSkipWhitespace(const char *pIn){ + const char *p = pIn; + if( p ){ + while( fts5_iswhitespace(*p) ){ p++; } + } + return p; +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a "bareword" character. +*/ +static const char *fts5ConfigSkipBareword(const char *pIn){ + const char *p = pIn; + while ( sqlite3Fts5IsBareword(*p) ) p++; + if( p==pIn ) p = 0; + return p; +} + +static int fts5_isdigit(char a){ + return (a>='0' && a<='9'); +} + + + +static const char *fts5ConfigSkipLiteral(const char *pIn){ + const char *p = pIn; + switch( *p ){ + case 'n': case 'N': + if( sqlite3_strnicmp("null", p, 4)==0 ){ + p = &p[4]; + }else{ + p = 0; + } + break; + + case 'x': case 'X': + p++; + if( *p=='\'' ){ + p++; + while( (*p>='a' && *p<='f') + || (*p>='A' && *p<='F') + || (*p>='0' && *p<='9') + ){ + p++; + } + if( *p=='\'' && 0==((p-pIn)%2) ){ + p++; + }else{ + p = 0; + } + }else{ + p = 0; + } + break; + + case '\'': + p++; + while( p ){ + if( *p=='\'' ){ + p++; + if( *p!='\'' ) break; + } + p++; + if( *p==0 ) p = 0; + } + break; + + default: + /* maybe a number */ + if( *p=='+' || *p=='-' ) p++; + while( fts5_isdigit(*p) ) p++; + + /* At this point, if the literal was an integer, the parse is + ** finished. Or, if it is a floating point value, it may continue + ** with either a decimal point or an 'E' character. */ + if( *p=='.' && fts5_isdigit(p[1]) ){ + p += 2; + while( fts5_isdigit(*p) ) p++; + } + if( p==pIn ) p = 0; + + break; + } + + return p; +} + +/* +** The first character of the string pointed to by argument z is guaranteed +** to be an open-quote character (see function fts5_isopenquote()). +** +** This function searches for the corresponding close-quote character within +** the string and, if found, dequotes the string in place and adds a new +** nul-terminator byte. +** +** If the close-quote is found, the value returned is the byte offset of +** the character immediately following it. Or, if the close-quote is not +** found, -1 is returned. If -1 is returned, the buffer is left in an +** undefined state. +*/ +static int fts5Dequote(char *z){ + char q; + int iIn = 1; + int iOut = 0; + q = z[0]; + + /* Set stack variable q to the close-quote character */ + assert( q=='[' || q=='\'' || q=='"' || q=='`' ); + if( q=='[' ) q = ']'; + + while( z[iIn] ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + + z[iOut] = '\0'; + return iIn; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static void sqlite3Fts5Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + assert( 0==fts5_iswhitespace(z[0]) ); + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + fts5Dequote(z); + } +} + + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = (int)strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** Parse a "special" CREATE VIRTUAL TABLE directive and update +** configuration object pConfig as appropriate. +** +** If successful, object pConfig is updated and SQLITE_OK returned. If +** an error occurs, an SQLite error code is returned and an error message +** may be left in *pzErr. It is the responsibility of the caller to +** eventually free any such error message using sqlite3_free(). +*/ +static int fts5ConfigParseSpecial( + Fts5Global *pGlobal, + Fts5Config *pConfig, /* Configuration object to update */ + const char *zCmd, /* Special command to parse */ + const char *zArg, /* Argument to parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + int nCmd = (int)strlen(zCmd); + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ + const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; + const char *p; + int bFirst = 1; + if( pConfig->aPrefix==0 ){ + pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte); + if( rc ) return rc; + } + + p = zArg; + while( 1 ){ + int nPre = 0; + + while( p[0]==' ' ) p++; + if( bFirst==0 && p[0]==',' ){ + p++; + while( p[0]==' ' ) p++; + }else if( p[0]=='\0' ){ + break; + } + if( p[0]<'0' || p[0]>'9' ){ + *pzErr = sqlite3_mprintf("malformed prefix=... directive"); + rc = SQLITE_ERROR; + break; + } + + if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){ + *pzErr = sqlite3_mprintf( + "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES + ); + rc = SQLITE_ERROR; + break; + } + + while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ + nPre = nPre*10 + (p[0] - '0'); + p++; + } + + if( nPre<=0 || nPre>=1000 ){ + *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); + rc = SQLITE_ERROR; + break; + } + + pConfig->aPrefix[pConfig->nPrefix] = nPre; + pConfig->nPrefix++; + bFirst = 0; + } + assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES ); + return rc; + } + + if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ + const char *p = (const char*)zArg; + sqlite3_int64 nArg = strlen(zArg) + 1; + char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); + char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); + char *pSpace = pDel; + + if( azArg && pSpace ){ + if( pConfig->pTok ){ + *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); + rc = SQLITE_ERROR; + }else{ + for(nArg=0; p && *p; nArg++){ + const char *p2 = fts5ConfigSkipWhitespace(p); + if( *p2=='\'' ){ + p = fts5ConfigSkipLiteral(p2); + }else{ + p = fts5ConfigSkipBareword(p2); + } + if( p ){ + memcpy(pSpace, p2, p-p2); + azArg[nArg] = pSpace; + sqlite3Fts5Dequote(pSpace); + pSpace += (p - p2) + 1; + p = fts5ConfigSkipWhitespace(p); + } + } + if( p==0 ){ + *pzErr = sqlite3_mprintf("parse error in tokenize directive"); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5GetTokenizer(pGlobal, + (const char**)azArg, (int)nArg, pConfig, + pzErr + ); + } + } + } + + sqlite3_free(azArg); + sqlite3_free(pDel); + return rc; + } + + if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + *pzErr = sqlite3_mprintf("multiple content=... directives"); + rc = SQLITE_ERROR; + }else{ + if( zArg[0] ){ + pConfig->eContent = FTS5_CONTENT_EXTERNAL; + pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg); + }else{ + pConfig->eContent = FTS5_CONTENT_NONE; + } + } + return rc; + } + + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ + if( pConfig->zContentRowid ){ + *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); + rc = SQLITE_ERROR; + }else{ + pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1); + } + return rc; + } + + if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bColumnsize = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); + } + return rc; + } + + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); + return SQLITE_ERROR; +} + +/* +** Allocate an instance of the default tokenizer ("simple") at +** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error +** code if an error occurs. +*/ +static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ + assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); + return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0); +} + +/* +** Gobble up the first bareword or quoted word from the input buffer zIn. +** Return a pointer to the character immediately following the last in +** the gobbled word if successful, or a NULL pointer otherwise (failed +** to find close-quote character). +** +** Before returning, set pzOut to point to a new buffer containing a +** nul-terminated, dequoted copy of the gobbled word. If the word was +** quoted, *pbQuoted is also set to 1 before returning. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op (NULL is returned). Otherwise, if an OOM occurs within this +** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not* +** set if a parse error (failed to find close quote) occurs. +*/ +static const char *fts5ConfigGobbleWord( + int *pRc, /* IN/OUT: Error code */ + const char *zIn, /* Buffer to gobble string/bareword from */ + char **pzOut, /* OUT: malloc'd buffer containing str/bw */ + int *pbQuoted /* OUT: Set to true if dequoting required */ +){ + const char *zRet = 0; + + sqlite3_int64 nIn = strlen(zIn); + char *zOut = sqlite3_malloc64(nIn+1); + + assert( *pRc==SQLITE_OK ); + *pbQuoted = 0; + *pzOut = 0; + + if( zOut==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memcpy(zOut, zIn, (size_t)(nIn+1)); + if( fts5_isopenquote(zOut[0]) ){ + int ii = fts5Dequote(zOut); + zRet = &zIn[ii]; + *pbQuoted = 1; + }else{ + zRet = fts5ConfigSkipBareword(zIn); + if( zRet ){ + zOut[zRet-zIn] = '\0'; + } + } + } + + if( zRet==0 ){ + sqlite3_free(zOut); + }else{ + *pzOut = zOut; + } + + return zRet; +} + +static int fts5ConfigParseColumn( + Fts5Config *p, + char *zCol, + char *zArg, + char **pzErr +){ + int rc = SQLITE_OK; + if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) + || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) + ){ + *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol); + rc = SQLITE_ERROR; + }else if( zArg ){ + if( 0==sqlite3_stricmp(zArg, "unindexed") ){ + p->abUnindexed[p->nCol] = 1; + }else{ + *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); + rc = SQLITE_ERROR; + } + } + + p->azCol[p->nCol++] = zCol; + return rc; +} + +/* +** Populate the Fts5Config.zContentExprlist string. +*/ +static int fts5ConfigMakeExprlist(Fts5Config *p){ + int i; + int rc = SQLITE_OK; + Fts5Buffer buf = {0, 0, 0}; + + sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); + if( p->eContent!=FTS5_CONTENT_NONE ){ + for(i=0; inCol; i++){ + if( p->eContent==FTS5_CONTENT_EXTERNAL ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + } + } + } + + assert( p->zContentExprlist==0 ); + p->zContentExprlist = (char*)buf.p; + return rc; +} + +/* +** Arguments nArg/azArg contain the string arguments passed to the xCreate +** or xConnect method of the virtual table. This function attempts to +** allocate an instance of Fts5Config containing the results of parsing +** those arguments. +** +** If successful, SQLITE_OK is returned and *ppOut is set to point to the +** new Fts5Config object. If an error occurs, an SQLite error code is +** returned, *ppOut is set to NULL and an error message may be left in +** *pzErr. It is the responsibility of the caller to eventually free any +** such error message using sqlite3_free(). +*/ +static int sqlite3Fts5ConfigParse( + Fts5Global *pGlobal, + sqlite3 *db, + int nArg, /* Number of arguments */ + const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */ + Fts5Config **ppOut, /* OUT: Results of parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pRet; /* New object to return */ + int i; + sqlite3_int64 nByte; + + *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + if( pRet==0 ) return SQLITE_NOMEM; + memset(pRet, 0, sizeof(Fts5Config)); + pRet->db = db; + pRet->iCookie = -1; + + nByte = nArg * (sizeof(char*) + sizeof(u8)); + pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); + pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; + pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); + pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); + pRet->bColumnsize = 1; + pRet->eDetail = FTS5_DETAIL_FULL; +#ifdef SQLITE_DEBUG + pRet->bPrefixIndex = 1; +#endif + if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ + *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); + rc = SQLITE_ERROR; + } + + for(i=3; rc==SQLITE_OK && ipTok==0 ){ + rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); + } + + /* If no zContent option was specified, fill in the default values. */ + if( rc==SQLITE_OK && pRet->zContent==0 ){ + const char *zTail = 0; + assert( pRet->eContent==FTS5_CONTENT_NORMAL + || pRet->eContent==FTS5_CONTENT_NONE + ); + if( pRet->eContent==FTS5_CONTENT_NORMAL ){ + zTail = "content"; + }else if( pRet->bColumnsize ){ + zTail = "docsize"; + } + + if( zTail ){ + pRet->zContent = sqlite3Fts5Mprintf( + &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail + ); + } + } + + if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ + pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1); + } + + /* Formulate the zContentExprlist text */ + if( rc==SQLITE_OK ){ + rc = fts5ConfigMakeExprlist(pRet); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts5ConfigFree(pRet); + *ppOut = 0; + } + return rc; +} + +/* +** Free the configuration object passed as the only argument. +*/ +static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ + if( pConfig ){ + int i; + if( pConfig->pTok ){ + pConfig->pTokApi->xDelete(pConfig->pTok); + } + sqlite3_free(pConfig->zDb); + sqlite3_free(pConfig->zName); + for(i=0; inCol; i++){ + sqlite3_free(pConfig->azCol[i]); + } + sqlite3_free(pConfig->azCol); + sqlite3_free(pConfig->aPrefix); + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + sqlite3_free(pConfig->zContent); + sqlite3_free(pConfig->zContentRowid); + sqlite3_free(pConfig->zContentExprlist); + sqlite3_free(pConfig); + } +} + +/* +** Call sqlite3_declare_vtab() based on the contents of the configuration +** object passed as the only argument. Return SQLITE_OK if successful, or +** an SQLite error code if an error occurs. +*/ +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ + int i; + int rc = SQLITE_OK; + char *zSql; + + zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x("); + for(i=0; zSql && inCol; i++){ + const char *zSep = (i==0?"":", "); + zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]); + } + zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", + zSql, pConfig->zName, FTS5_RANK_NAME + ); + + assert( zSql || rc==SQLITE_NOMEM ); + if( zSql ){ + rc = sqlite3_declare_vtab(pConfig->db, zSql); + sqlite3_free(zSql); + } + + return rc; +} + +/* +** Tokenize the text passed via the second and third arguments. +** +** The callback is invoked once for each token in the input text. The +** arguments passed to it are, in order: +** +** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize() +** const char *pToken // Pointer to buffer containing token +** int nToken // Size of token in bytes +** int iStart // Byte offset of start of token within input text +** int iEnd // Byte offset of end of token within input text +** int iPos // Position of token in input (first token is 0) +** +** If the callback returns a non-zero value the tokenization is abandoned +** and no further callbacks are issued. +** +** This function returns SQLITE_OK if successful or an SQLite error code +** if an error occurs. If the tokenization was abandoned early because +** the callback returned SQLITE_DONE, this is not an error and this function +** still returns SQLITE_OK. Or, if the tokenization was abandoned early +** because the callback returned another non-zero value, it is assumed +** to be an SQLite error code and returned to the caller. +*/ +static int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +){ + if( pText==0 ) return SQLITE_OK; + return pConfig->pTokApi->xTokenize( + pConfig->pTok, pCtx, flags, pText, nText, xToken + ); +} + +/* +** Argument pIn points to the first character in what is expected to be +** a comma-separated list of SQL literals followed by a ')' character. +** If it actually is this, return a pointer to the ')'. Otherwise, return +** NULL to indicate a parse error. +*/ +static const char *fts5ConfigSkipArgs(const char *pIn){ + const char *p = pIn; + + while( 1 ){ + p = fts5ConfigSkipWhitespace(p); + p = fts5ConfigSkipLiteral(p); + p = fts5ConfigSkipWhitespace(p); + if( p==0 || *p==')' ) break; + if( *p!=',' ){ + p = 0; + break; + } + p++; + } + + return p; +} + +/* +** Parameter zIn contains a rank() function specification. The format of +** this is: +** +** + Bareword (function name) +** + Open parenthesis - "(" +** + Zero or more SQL literals in a comma separated list +** + Close parenthesis - ")" +*/ +static int sqlite3Fts5ConfigParseRank( + const char *zIn, /* Input string */ + char **pzRank, /* OUT: Rank function name */ + char **pzRankArgs /* OUT: Rank function arguments */ +){ + const char *p = zIn; + const char *pRank; + char *zRank = 0; + char *zRankArgs = 0; + int rc = SQLITE_OK; + + *pzRank = 0; + *pzRankArgs = 0; + + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + p = fts5ConfigSkipWhitespace(p); + pRank = p; + p = fts5ConfigSkipBareword(p); + + if( p ){ + zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank); + if( zRank ) memcpy(zRank, pRank, p-pRank); + }else{ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + p = fts5ConfigSkipWhitespace(p); + if( *p!='(' ) rc = SQLITE_ERROR; + p++; + } + if( rc==SQLITE_OK ){ + const char *pArgs; + p = fts5ConfigSkipWhitespace(p); + pArgs = p; + if( *p!=')' ){ + p = fts5ConfigSkipArgs(p); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs); + if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs); + } + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRank); + assert( zRankArgs==0 ); + }else{ + *pzRank = zRank; + *pzRankArgs = zRankArgs; + } + return rc; +} + +static int sqlite3Fts5ConfigSetValue( + Fts5Config *pConfig, + const char *zKey, + sqlite3_value *pVal, + int *pbBadkey +){ + int rc = SQLITE_OK; + + if( 0==sqlite3_stricmp(zKey, "pgsz") ){ + int pgsz = 0; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + pgsz = sqlite3_value_int(pVal); + } + if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE ){ + *pbBadkey = 1; + }else{ + pConfig->pgsz = pgsz; + } + } + + else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ + int nHashSize = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nHashSize = sqlite3_value_int(pVal); + } + if( nHashSize<=0 ){ + *pbBadkey = 1; + }else{ + pConfig->nHashSize = nHashSize; + } + } + + else if( 0==sqlite3_stricmp(zKey, "automerge") ){ + int nAutomerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nAutomerge = sqlite3_value_int(pVal); + } + if( nAutomerge<0 || nAutomerge>64 ){ + *pbBadkey = 1; + }else{ + if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nAutomerge = nAutomerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ + int nUsermerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nUsermerge = sqlite3_value_int(pVal); + } + if( nUsermerge<2 || nUsermerge>16 ){ + *pbBadkey = 1; + }else{ + pConfig->nUsermerge = nUsermerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ + int nCrisisMerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nCrisisMerge = sqlite3_value_int(pVal); + } + if( nCrisisMerge<0 ){ + *pbBadkey = 1; + }else{ + if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1; + pConfig->nCrisisMerge = nCrisisMerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "rank") ){ + const char *zIn = (const char*)sqlite3_value_text(pVal); + char *zRank; + char *zRankArgs; + rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); + if( rc==SQLITE_OK ){ + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + pConfig->zRank = zRank; + pConfig->zRankArgs = zRankArgs; + }else if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + *pbBadkey = 1; + } + }else{ + *pbBadkey = 1; + } + return rc; +} + +/* +** Load the contents of the %_config table into memory. +*/ +static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ + const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; + char *zSql; + sqlite3_stmt *p = 0; + int rc = SQLITE_OK; + int iVersion = 0; + + /* Set default values */ + pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; + pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; + pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + + zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); + sqlite3_free(zSql); + } + + assert( rc==SQLITE_OK || p==0 ); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(p) ){ + const char *zK = (const char*)sqlite3_column_text(p, 0); + sqlite3_value *pVal = sqlite3_column_value(p, 1); + if( 0==sqlite3_stricmp(zK, "version") ){ + iVersion = sqlite3_value_int(pVal); + }else{ + int bDummy = 0; + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy); + } + } + rc = sqlite3_finalize(p); + } + + if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + rc = SQLITE_ERROR; + if( pConfig->pzErrmsg ){ + assert( 0==*pConfig->pzErrmsg ); + *pConfig->pzErrmsg = sqlite3_mprintf( + "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION + ); + } + } + + if( rc==SQLITE_OK ){ + pConfig->iCookie = iCookie; + } + return rc; +} + +#line 1 "fts5_expr.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ +/* #include "third_party/sqlite3/fts5parse.h" */ + +/* +** All token types in the generated fts5parse.h file are greater than 0. +*/ +#define FTS5_EOF 0 + +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + +typedef struct Fts5ExprTerm Fts5ExprTerm; + +/* +** Functions generated by lemon from fts5parse.y. +*/ +// static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64)); +// static void sqlite3Fts5ParserFree(void *, void (*freeProc)(void *)); +// static void sqlite3Fts5Parser(void *, int, Fts5Token, Fts5Parse *); +#ifndef NDEBUG +#include "libc/stdio/stdio.h" +static void sqlite3Fts5ParserTrace(FILE *, char *); +#endif +// static int sqlite3Fts5ParserFallback(int); + + +struct Fts5Expr { + Fts5Index *pIndex; + Fts5Config *pConfig; + Fts5ExprNode *pRoot; + int bDesc; /* Iterate in descending rowid order */ + int nPhrase; /* Number of phrases in expression */ + Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ +}; + +/* +** eType: +** Expression node type. Always one of: +** +** FTS5_AND (nChild, apChild valid) +** FTS5_OR (nChild, apChild valid) +** FTS5_NOT (nChild, apChild valid) +** FTS5_STRING (pNear valid) +** FTS5_TERM (pNear valid) +*/ +struct Fts5ExprNode { + int eType; /* Node type */ + int bEof; /* True at EOF */ + int bNomatch; /* True if entry is not a match */ + + /* Next method for this node. */ + int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); + + i64 iRowid; /* Current rowid */ + Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ + + /* Child nodes. For a NOT node, this array always contains 2 entries. For + ** AND or OR nodes, it contains 2 or more entries. */ + int nChild; /* Number of child nodes */ + Fts5ExprNode *apChild[1]; /* Array of child nodes */ +}; + +#define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) + +/* +** Invoke the xNext method of an Fts5ExprNode object. This macro should be +** used as if it has the same signature as the xNext() methods themselves. +*/ +#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d)) + +/* +** An instance of the following structure represents a single search term +** or term prefix. +*/ +struct Fts5ExprTerm { + u8 bPrefix; /* True for a prefix term */ + u8 bFirst; /* True if token must be first in column */ + char *zTerm; /* nul-terminated term */ + Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ +}; + +/* +** A phrase. One or more terms that must appear in a contiguous sequence +** within a document for it to match. +*/ +struct Fts5ExprPhrase { + Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ + Fts5Buffer poslist; /* Current position list */ + int nTerm; /* Number of entries in aTerm[] */ + Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ +}; + +/* +** One or more phrases that must appear within a certain token distance of +** each other within each matching document. +*/ +struct Fts5ExprNearset { + int nNear; /* NEAR parameter */ + Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ + int nPhrase; /* Number of entries in aPhrase[] array */ + Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ +}; + + +/* +** Parse context. +*/ +struct Fts5Parse { + Fts5Config *pConfig; + char *zErr; + int rc; + int nPhrase; /* Size of apPhrase array */ + Fts5ExprPhrase **apPhrase; /* Array of all phrases */ + Fts5ExprNode *pExpr; /* Result of a successful parse */ + int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ +}; + +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pParse->rc==SQLITE_OK ){ + pParse->zErr = sqlite3_vmprintf(zFmt, ap); + pParse->rc = SQLITE_ERROR; + } + va_end(ap); +} + +static int fts5ExprIsspace(char t){ + return t==' ' || t=='\t' || t=='\n' || t=='\r'; +} + +/* +** Read the first token from the nul-terminated string at *pz. +*/ +static int fts5ExprGetToken( + Fts5Parse *pParse, + const char **pz, /* IN/OUT: Pointer into buffer */ + Fts5Token *pToken +){ + const char *z = *pz; + int tok; + + /* Skip past any whitespace */ + while( fts5ExprIsspace(*z) ) z++; + + pToken->p = z; + pToken->n = 1; + switch( *z ){ + case '(': tok = FTS5_LP; break; + case ')': tok = FTS5_RP; break; + case '{': tok = FTS5_LCP; break; + case '}': tok = FTS5_RCP; break; + case ':': tok = FTS5_COLON; break; + case ',': tok = FTS5_COMMA; break; + case '+': tok = FTS5_PLUS; break; + case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; + case '^': tok = FTS5_CARET; break; + case '\0': tok = FTS5_EOF; break; + + case '"': { + const char *z2; + tok = FTS5_STRING; + + for(z2=&z[1]; 1; z2++){ + if( z2[0]=='"' ){ + z2++; + if( z2[0]!='"' ) break; + } + if( z2[0]=='\0' ){ + sqlite3Fts5ParseError(pParse, "unterminated string"); + return FTS5_EOF; + } + } + pToken->n = (z2 - z); + break; + } + + default: { + const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } + tok = FTS5_STRING; + for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); + pToken->n = (z2 - z); + if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR; + if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT; + if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND; + break; + } + } + + *pz = &pToken->p[pToken->n]; + return tok; +} + +static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64((sqlite3_int64)t);} +static void fts5ParseFree(void *p){ sqlite3_free(p); } + +static int sqlite3Fts5ExprNew( + Fts5Config *pConfig, /* FTS5 Configuration */ + int bPhraseToAnd, + int iCol, + const char *zExpr, /* Expression text */ + Fts5Expr **ppNew, + char **pzErr +){ + Fts5Parse sParse; + Fts5Token token; + const char *z = zExpr; + int t; /* Next token type */ + void *pEngine; + Fts5Expr *pNew; + + *ppNew = 0; + *pzErr = 0; + memset(&sParse, 0, sizeof(sParse)); + sParse.bPhraseToAnd = bPhraseToAnd; + pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc); + if( pEngine==0 ){ return SQLITE_NOMEM; } + sParse.pConfig = pConfig; + + do { + t = fts5ExprGetToken(&sParse, &z, &token); + sqlite3Fts5Parser(pEngine, t, token, &sParse); + }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); + sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + + /* If the LHS of the MATCH expression was a user column, apply the + ** implicit column-filter. */ + if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ + int n = sizeof(Fts5Colset); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); + if( pColset ){ + pColset->nCol = 1; + pColset->aiCol[0] = iCol; + sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset); + } + } + + assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); + if( sParse.rc==SQLITE_OK ){ + *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + if( pNew==0 ){ + sParse.rc = SQLITE_NOMEM; + sqlite3Fts5ParseNodeFree(sParse.pExpr); + }else{ + if( !sParse.pExpr ){ + const int nByte = sizeof(Fts5ExprNode); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); + if( pNew->pRoot ){ + pNew->pRoot->bEof = 1; + } + }else{ + pNew->pRoot = sParse.pExpr; + } + pNew->pIndex = 0; + pNew->pConfig = pConfig; + pNew->apExprPhrase = sParse.apPhrase; + pNew->nPhrase = sParse.nPhrase; + pNew->bDesc = 0; + sParse.apPhrase = 0; + } + }else{ + sqlite3Fts5ParseNodeFree(sParse.pExpr); + } + + sqlite3_free(sParse.apPhrase); + *pzErr = sParse.zErr; + return sParse.rc; +} + +/* +** This function is only called when using the special 'trigram' tokenizer. +** Argument zText contains the text of a LIKE or GLOB pattern matched +** against column iCol. This function creates and compiles an FTS5 MATCH +** expression that will match a superset of the rows matched by the LIKE or +** GLOB. If successful, SQLITE_OK is returned. Otherwise, an SQLite error +** code. +*/ +static int sqlite3Fts5ExprPattern( + Fts5Config *pConfig, int bGlob, int iCol, const char *zText, Fts5Expr **pp +){ + i64 nText = strlen(zText); + char *zExpr = (char*)sqlite3_malloc64(nText*4 + 1); + int rc = SQLITE_OK; + + if( zExpr==0 ){ + rc = SQLITE_NOMEM; + }else{ + char aSpec[3]; + int iOut = 0; + int i = 0; + int iFirst = 0; + + if( bGlob==0 ){ + aSpec[0] = '_'; + aSpec[1] = '%'; + aSpec[2] = 0; + }else{ + aSpec[0] = '*'; + aSpec[1] = '?'; + aSpec[2] = '['; + } + + while( i<=nText ){ + if( i==nText + || zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2] + ){ + if( i-iFirst>=3 ){ + int jj; + zExpr[iOut++] = '"'; + for(jj=iFirst; jj0 ){ + int bAnd = 0; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + bAnd = 1; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + iCol = pConfig->nCol; + } + } + zExpr[iOut] = '\0'; + rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg); + }else{ + *pp = 0; + } + sqlite3_free(zExpr); + } + + return rc; +} + +/* +** Free the expression node object passed as the only argument. +*/ +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ + if( p ){ + int i; + for(i=0; inChild; i++){ + sqlite3Fts5ParseNodeFree(p->apChild[i]); + } + sqlite3Fts5ParseNearsetFree(p->pNear); + sqlite3_free(p); + } +} + +/* +** Free the expression object passed as the only argument. +*/ +static void sqlite3Fts5ExprFree(Fts5Expr *p){ + if( p ){ + sqlite3Fts5ParseNodeFree(p->pRoot); + sqlite3_free(p->apExprPhrase); + sqlite3_free(p); + } +} + +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ + Fts5Parse sParse; + memset(&sParse, 0, sizeof(sParse)); + + if( *pp1 ){ + Fts5Expr *p1 = *pp1; + int nPhrase = p1->nPhrase + p2->nPhrase; + + p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0); + p2->pRoot = 0; + + if( sParse.rc==SQLITE_OK ){ + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) + ); + if( ap==0 ){ + sParse.rc = SQLITE_NOMEM; + }else{ + int i; + memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*)); + for(i=0; inPhrase; i++){ + ap[i] = p2->apExprPhrase[i]; + } + p1->nPhrase = nPhrase; + p1->apExprPhrase = ap; + } + } + sqlite3_free(p2->apExprPhrase); + sqlite3_free(p2); + }else{ + *pp1 = p2; + } + + return sParse.rc; +} + +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = p->pIter->iRowid; + if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ + if( pIter->nData==0 ) continue; + if( nIter==nAlloc ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); + aIter = aNew; + } + sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; + } + } + + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + i64 iPrev = -1; + fts5BufferZero(pBuf); + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; ip; + *pn = pBuf->n; + } + } + + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + + +/* +** All individual term iterators in pPhrase are guaranteed to be valid and +** pointing to the same rowid when this function is called. This function +** checks if the current rowid really is a match, and if so populates +** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch +** is set to true if this is really a match, or false otherwise. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if the current rowid is +** not a match. +*/ +static int fts5ExprPhraseIsMatch( + Fts5ExprNode *pNode, /* Node pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbMatch /* OUT: Set to true if really a match */ +){ + Fts5PoslistWriter writer = {0}; + Fts5PoslistReader aStatic[4]; + Fts5PoslistReader *aIter = aStatic; + int i; + int rc = SQLITE_OK; + int bFirst = pPhrase->aTerm[0].bFirst; + + fts5BufferZero(&pPhrase->poslist); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pPhrase->nTerm>ArraySize(aStatic) ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; + aIter = (Fts5PoslistReader*)sqlite3_malloc64(nByte); + if( !aIter ) return SQLITE_NOMEM; + } + memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); + + /* Initialize a term iterator for each term in the phrase */ + for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + int n = 0; + int bFlag = 0; + u8 *a = 0; + if( pTerm->pSynonym ){ + Fts5Buffer buf = {0, 0, 0}; + rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n); + if( rc ){ + sqlite3_free(a); + goto ismatch_out; + } + if( a==buf.p ) bFlag = 1; + }else{ + a = (u8*)pTerm->pIter->pData; + n = pTerm->pIter->nData; + } + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); + aIter[i].bFlag = (u8)bFlag; + if( aIter[i].bEof ) goto ismatch_out; + } + + while( 1 ){ + int bMatch; + i64 iPos = aIter[0].iPos; + do { + bMatch = 1; + for(i=0; inTerm; i++){ + Fts5PoslistReader *pPos = &aIter[i]; + i64 iAdj = iPos + i; + if( pPos->iPos!=iAdj ){ + bMatch = 0; + while( pPos->iPosiPos>iAdj ) iPos = pPos->iPos-i; + } + } + }while( bMatch==0 ); + + /* Append position iPos to the output */ + if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; + } + + for(i=0; inTerm; i++){ + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; + } + } + + ismatch_out: + *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + +typedef struct Fts5LookaheadReader Fts5LookaheadReader; +struct Fts5LookaheadReader { + const u8 *a; /* Buffer containing position list */ + int n; /* Size of buffer a[] in bytes */ + int i; /* Current offset in position list */ + i64 iPos; /* Current position */ + i64 iLookahead; /* Next position */ +}; + +#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62) + +static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){ + p->iPos = p->iLookahead; + if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){ + p->iLookahead = FTS5_LOOKAHEAD_EOF; + } + return (p->iPos==FTS5_LOOKAHEAD_EOF); +} + +static int fts5LookaheadReaderInit( + const u8 *a, int n, /* Buffer to read position list from */ + Fts5LookaheadReader *p /* Iterator object to initialize */ +){ + memset(p, 0, sizeof(Fts5LookaheadReader)); + p->a = a; + p->n = n; + fts5LookaheadReaderNext(p); + return fts5LookaheadReaderNext(p); +} + +typedef struct Fts5NearTrimmer Fts5NearTrimmer; +struct Fts5NearTrimmer { + Fts5LookaheadReader reader; /* Input iterator */ + Fts5PoslistWriter writer; /* Writer context */ + Fts5Buffer *pOut; /* Output poslist */ +}; + +/* +** The near-set object passed as the first argument contains more than +** one phrase. All phrases currently point to the same row. The +** Fts5ExprPhrase.poslist buffers are populated accordingly. This function +** tests if the current row contains instances of each phrase sufficiently +** close together to meet the NEAR constraint. Non-zero is returned if it +** does, or zero otherwise. +** +** If in/out parameter (*pRc) is set to other than SQLITE_OK when this +** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM) +** occurs within this function (*pRc) is set accordingly before returning. +** The return value is undefined in both these cases. +** +** If no error occurs and non-zero (a match) is returned, the position-list +** of each phrase object is edited to contain only those entries that +** meet the constraint before returning. +*/ +static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ + Fts5NearTrimmer aStatic[4]; + Fts5NearTrimmer *a = aStatic; + Fts5ExprPhrase **apPhrase = pNear->apPhrase; + + int i; + int rc = *pRc; + int bMatch; + + assert( pNear->nPhrase>1 ); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pNear->nPhrase>ArraySize(aStatic) ){ + sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; + a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); + }else{ + memset(aStatic, 0, sizeof(aStatic)); + } + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + } + + /* Initialize a lookahead iterator for each phrase. After passing the + ** buffer and buffer size to the lookaside-reader init function, zero + ** the phrase poslist buffer. The new poslist for the phrase (containing + ** the same entries as the original with some entries removed on account + ** of the NEAR constraint) is written over the original even as it is + ** being read. This is safe as the entries for the new poslist are a + ** subset of the old, so it is not possible for data yet to be read to + ** be overwritten. */ + for(i=0; inPhrase; i++){ + Fts5Buffer *pPoslist = &apPhrase[i]->poslist; + fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader); + pPoslist->n = 0; + a[i].pOut = pPoslist; + } + + while( 1 ){ + int iAdv; + i64 iMin; + i64 iMax; + + /* This block advances the phrase iterators until they point to a set of + ** entries that together comprise a match. */ + iMax = a[0].reader.iPos; + do { + bMatch = 1; + for(i=0; inPhrase; i++){ + Fts5LookaheadReader *pPos = &a[i].reader; + iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear; + if( pPos->iPosiPos>iMax ){ + bMatch = 0; + while( pPos->iPosiPos>iMax ) iMax = pPos->iPos; + } + } + }while( bMatch==0 ); + + /* Add an entry to each output position list */ + for(i=0; inPhrase; i++){ + i64 iPos = a[i].reader.iPos; + Fts5PoslistWriter *pWriter = &a[i].writer; + if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){ + sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos); + } + } + + iAdv = 0; + iMin = a[0].reader.iLookahead; + for(i=0; inPhrase; i++){ + if( a[i].reader.iLookahead < iMin ){ + iMin = a[i].reader.iLookahead; + iAdv = i; + } + } + if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out; + } + + ismatch_out: { + int bRet = a[0].pOut->n>0; + *pRc = rc; + if( a!=aStatic ) sqlite3_free(a); + return bRet; + } +} + +/* +** Advance iterator pIter until it points to a value equal to or laster +** than the initial value of *piLast. If this means the iterator points +** to a value laster than *piLast, update *piLast to the new lastest value. +** +** If the iterator reaches EOF, set *pbEof to true before returning. If +** an error occurs, set *pRc to an error code. If either *pbEof or *pRc +** are set, return a non-zero value. Otherwise, return zero. +*/ +static int fts5ExprAdvanceto( + Fts5IndexIter *pIter, /* Iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + i64 iLast = *piLast; + i64 iRowid; + + iRowid = pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastiRowid; + assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); + } + *piLast = iRowid; + + return 0; +} + +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc /* OUT: Error code */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + int bEof = 0; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = p->pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pRc = rc; + bEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); + } + return bEof; +} + + +static int fts5ExprNearTest( + int *pRc, + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ +){ + Fts5ExprNearset *pNear = pNode->pNear; + int rc = *pRc; + + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ + pPhrase->poslist.n = 1; + } + } + } + return pPhrase->poslist.n; + }else{ + int i; + + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym + || pNear->pColset || pPhrase->aTerm[0].bFirst + ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); + } + } + + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } +} + + +/* +** Initialize all term iterators in the pNear object. If any term is found +** to match no documents at all, return immediately without initializing any +** further iterators. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. It is not considered an error if some term matches zero +** documents. +*/ +static int fts5ExprNearInitAll( + Fts5Expr *pExpr, + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + int i; + + assert( pNode->bNomatch==0 ); + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + }else{ + int j; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bHit = 0; + + for(p=pTerm; p; p=p->pSynonym){ + int rc; + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, + &p->pIter + ); + assert( (rc==SQLITE_OK)==(p->pIter!=0) ); + if( rc!=SQLITE_OK ) return rc; + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + bHit = 1; + } + } + + if( bHit==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + } + } + } + } + + pNode->bEof = 0; + return SQLITE_OK; +} + +/* +** If pExpr is an ASC iterator, this function returns a value with the +** same sign as: +** +** (iLhs - iRhs) +** +** Otherwise, if this is a DESC iterator, the opposite is returned: +** +** (iRhs - iLhs) +*/ +static int fts5RowidCmp( + Fts5Expr *pExpr, + i64 iLhs, + i64 iRhs +){ + assert( pExpr->bDesc==0 || pExpr->bDesc==1 ); + if( pExpr->bDesc==0 ){ + if( iLhs iRhs); + }else{ + if( iLhs>iRhs ) return -1; + return (iLhs < iRhs); + } +} + +static void fts5ExprSetEof(Fts5ExprNode *pNode){ + int i; + pNode->bEof = 1; + pNode->bNomatch = 0; + for(i=0; inChild; i++){ + fts5ExprSetEof(pNode->apChild[i]); + } +} + +static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + int i; + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + pPhrase->poslist.n = 0; + } + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprNodeZeroPoslist(pNode->apChild[i]); + } + } +} + + + +/* +** Compare the values currently indicated by the two nodes as follows: +** +** res = (*p1) - (*p2) +** +** Nodes that point to values that come later in the iteration order are +** considered to be larger. Nodes at EOF are the largest of all. +** +** This means that if the iteration order is ASC, then numerically larger +** rowids are considered larger. Or if it is the default DESC, numerically +** smaller rowids are larger. +*/ +static int fts5NodeCompare( + Fts5Expr *pExpr, + Fts5ExprNode *p1, + Fts5ExprNode *p2 +){ + if( p2->bEof ) return -1; + if( p1->bEof ) return +1; + return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); +} + +/* +** All individual term iterators in pNear are guaranteed to be valid when +** this function is called. This function checks if all term iterators +** point to the same rowid, and if not, advances them until they do. +** If an EOF is reached before this happens, *pbEof is set to true before +** returning. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprNodeTest_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; + int rc = SQLITE_OK; + i64 iLast; /* Lastest rowid any iterator points to */ + int i, j; /* Phrase and token index, respectively */ + int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; + + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + || pNear->apPhrase[0]->aTerm[0].bFirst + ); + + /* Initialize iLast, the "lastest" rowid any iterator points to. If the + ** iterator skips through rowids in the default ascending order, this means + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it + ** means the minimum rowid. */ + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); + }else{ + iLast = pLeft->aTerm[0].pIter->iRowid; + } + + do { + bMatch = 1; + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bNomatch = 0; + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + if( pIter->iRowid==iLast || pIter->bEof ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } + } + } + } + }while( bMatch==0 ); + + pNode->iRowid = iLast; + pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); + assert( pNode->bEof==0 || pNode->bNomatch==0 ); + + return rc; +} + +/* +** Advance the first term iterator in the first phrase of pNear. Set output +** variable *pbEof to true if it reaches EOF or if an error occurs. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5ExprNodeNext_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ + int bFromValid, + i64 iFrom +){ + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; + int rc = SQLITE_OK; + + pNode->bNomatch = 0; + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); + + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 ii = p->pIter->iRowid; + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } + if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); + }else{ + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); + } + + if( pNode->bEof==0 ){ + assert( rc==SQLITE_OK ); + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + } + + return rc; +} + + +static int fts5ExprNodeTest_TERM( + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ +){ + /* As this "NEAR" object is actually a single phrase that consists + ** of a single term only, grab pointers into the poslist managed by the + ** fts5_index.c iterator object. This is much faster than synthesizing + ** a new poslist the way we have to for more complicated phrase or NEAR + ** expressions. */ + Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + + assert( pNode->eType==FTS5_TERM ); + assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); + + pPhrase->poslist.n = pIter->nData; + if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pPhrase->poslist.p = (u8*)pIter->pData; + } + pNode->iRowid = pIter->iRowid; + pNode->bNomatch = (pPhrase->poslist.n==0); + return SQLITE_OK; +} + +/* +** xNext() method for a node of type FTS5_TERM. +*/ +static int fts5ExprNodeNext_TERM( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc; + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + + assert( pNode->bEof==0 ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + }else{ + pNode->bEof = 1; + pNode->bNomatch = 0; + } + return rc; +} + +static void fts5ExprNodeTest_OR( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + Fts5ExprNode *pNext = pNode->apChild[0]; + int i; + + for(i=1; inChild; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + int cmp = fts5NodeCompare(pExpr, pNext, pChild); + if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ + pNext = pChild; + } + } + pNode->iRowid = pNext->iRowid; + pNode->bEof = pNext->bEof; + pNode->bNomatch = pNext->bNomatch; +} + +static int fts5ExprNodeNext_OR( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int i; + i64 iLast = pNode->iRowid; + + for(i=0; inChild; i++){ + Fts5ExprNode *p1 = pNode->apChild[i]; + assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); + if( p1->bEof==0 ){ + if( (p1->iRowid==iLast) + || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) + ){ + int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } + } + } + } + + fts5ExprNodeTest_OR(pExpr, pNode); + return SQLITE_OK; +} + +/* +** Argument pNode is an FTS5_AND node. +*/ +static int fts5ExprNodeTest_AND( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pAnd /* FTS5_AND node to advance */ +){ + int iChild; + i64 iLast = pAnd->iRowid; + int rc = SQLITE_OK; + int bMatch; + + assert( pAnd->bEof==0 ); + do { + pAnd->bNomatch = 0; + bMatch = 1; + for(iChild=0; iChildnChild; iChild++){ + Fts5ExprNode *pChild = pAnd->apChild[iChild]; + int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); + if( cmp>0 ){ + /* Advance pChild until it points to iLast or laster */ + rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } + } + + /* If the child node is now at EOF, so is the parent AND node. Otherwise, + ** the child node is guaranteed to have advanced at least as far as + ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the + ** new lastest rowid seen so far. */ + assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); + if( pChild->bEof ){ + fts5ExprSetEof(pAnd); + bMatch = 1; + break; + }else if( iLast!=pChild->iRowid ){ + bMatch = 0; + iLast = pChild->iRowid; + } + + if( pChild->bNomatch ){ + pAnd->bNomatch = 1; + } + } + }while( bMatch==0 ); + + if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ + fts5ExprNodeZeroPoslist(pAnd); + } + pAnd->iRowid = iLast; + return SQLITE_OK; +} + +static int fts5ExprNodeNext_AND( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; + } + return rc; +} + +static int fts5ExprNodeTest_NOT( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode /* FTS5_NOT node to advance */ +){ + int rc = SQLITE_OK; + Fts5ExprNode *p1 = pNode->apChild[0]; + Fts5ExprNode *p2 = pNode->apChild[1]; + assert( pNode->nChild==2 ); + + while( rc==SQLITE_OK && p1->bEof==0 ){ + int cmp = fts5NodeCompare(pExpr, p1, p2); + if( cmp>0 ){ + rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); + cmp = fts5NodeCompare(pExpr, p1, p2); + } + assert( rc!=SQLITE_OK || cmp<=0 ); + if( cmp || p2->bNomatch ) break; + rc = fts5ExprNodeNext(pExpr, p1, 0, 0); + } + pNode->bEof = p1->bEof; + pNode->bNomatch = p1->bNomatch; + pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); + } + return rc; +} + +static int fts5ExprNodeNext_NOT( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } + return rc; +} + +/* +** If pNode currently points to a match, this function returns SQLITE_OK +** without modifying it. Otherwise, pNode is advanced until it does point +** to a match or EOF is reached. +*/ +static int fts5ExprNodeTest( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + int rc = SQLITE_OK; + if( pNode->bEof==0 ){ + switch( pNode->eType ){ + + case FTS5_STRING: { + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + break; + } + + case FTS5_TERM: { + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + break; + } + + case FTS5_AND: { + rc = fts5ExprNodeTest_AND(pExpr, pNode); + break; + } + + case FTS5_OR: { + fts5ExprNodeTest_OR(pExpr, pNode); + break; + } + + default: assert( pNode->eType==FTS5_NOT ); { + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + break; + } + } + } + return rc; +} + + +/* +** Set node pNode, which is part of expression pExpr, to point to the first +** match. If there are no matches, set the Node.bEof flag to indicate EOF. +** +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +** It is not an error if there are no matches. +*/ +static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ + int rc = SQLITE_OK; + pNode->bEof = 0; + pNode->bNomatch = 0; + + if( Fts5NodeIsString(pNode) ){ + /* Initialize all term iterators in the NEAR object. */ + rc = fts5ExprNearInitAll(pExpr, pNode); + }else if( pNode->xNext==0 ){ + pNode->bEof = 1; + }else{ + int i; + int nEof = 0; + for(i=0; inChild && rc==SQLITE_OK; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); + assert( pChild->bEof==0 || pChild->bEof==1 ); + nEof += pChild->bEof; + } + pNode->iRowid = pNode->apChild[0]->iRowid; + + switch( pNode->eType ){ + case FTS5_AND: + if( nEof>0 ) fts5ExprSetEof(pNode); + break; + + case FTS5_OR: + if( pNode->nChild==nEof ) fts5ExprSetEof(pNode); + break; + + default: + assert( pNode->eType==FTS5_NOT ); + pNode->bEof = pNode->apChild[0]->bEof; + break; + } + } + + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest(pExpr, pNode); + } + return rc; +} + + +/* +** Begin iterating through the set of documents in index pIdx matched by +** the MATCH expression passed as the first argument. If the "bDesc" +** parameter is passed a non-zero value, iteration is in descending rowid +** order. Or, if it is zero, in ascending order. +** +** If iterating in ascending rowid order (bDesc==0), the first document +** visited is that with the smallest rowid that is larger than or equal +** to parameter iFirst. Or, if iterating in ascending order (bDesc==1), +** then the first document visited must have a rowid smaller than or +** equal to iFirst. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ + Fts5ExprNode *pRoot = p->pRoot; + int rc; /* Return code */ + + p->pIndex = pIdx; + p->bDesc = bDesc; + rc = fts5ExprNodeFirst(p, pRoot); + + /* If not at EOF but the current rowid occurs earlier than iFirst in + ** the iteration order, move to document iFirst or later. */ + if( rc==SQLITE_OK + && 0==pRoot->bEof + && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 + ){ + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); + } + + /* If the iterator is not at a real match, skip forward until it is. */ + while( pRoot->bNomatch && rc==SQLITE_OK ){ + assert( pRoot->bEof==0 ); + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + } + return rc; +} + +/* +** Move to the next document +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ + int rc; + Fts5ExprNode *pRoot = p->pRoot; + assert( pRoot->bEof==0 && pRoot->bNomatch==0 ); + do { + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) ); + }while( pRoot->bNomatch ); + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } + return rc; +} + +static int sqlite3Fts5ExprEof(Fts5Expr *p){ + return p->pRoot->bEof; +} + +static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ + return p->pRoot->iRowid; +} + +static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ + int rc = SQLITE_OK; + *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n); + return rc; +} + +/* +** Free the phrase object passed as the only argument. +*/ +static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ + if( pPhrase ){ + int i; + for(i=0; inTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + sqlite3_free(pTerm->zTerm); + sqlite3Fts5IterClose(pTerm->pIter); + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + fts5BufferFree((Fts5Buffer*)&pSyn[1]); + sqlite3_free(pSyn); + } + } + if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); + sqlite3_free(pPhrase); + } +} + +/* +** Set the "bFirst" flag on the first token of the phrase passed as the +** only argument. +*/ +static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){ + if( pPhrase && pPhrase->nTerm ){ + pPhrase->aTerm[0].bFirst = 1; + } +} + +/* +** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated +** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is +** appended to it and the results returned. +** +** If an OOM error occurs, both the pNear and pPhrase objects are freed and +** NULL returned. +*/ +static Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNearset *pNear, /* Existing nearset, or NULL */ + Fts5ExprPhrase *pPhrase /* Recently parsed phrase */ +){ + const int SZALLOC = 8; + Fts5ExprNearset *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + if( pPhrase==0 ){ + return pNear; + } + if( pNear==0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, (size_t)nByte); + } + }else if( (pNear->nPhrase % SZALLOC)==0 ){ + int nNew = pNear->nPhrase + SZALLOC; + sqlite3_int64 nByte; + + nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + } + }else{ + pRet = pNear; + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNearsetFree(pNear); + sqlite3Fts5ParsePhraseFree(pPhrase); + }else{ + if( pRet->nPhrase>0 ){ + Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); + if( pPhrase->nTerm==0 ){ + fts5ExprPhraseFree(pPhrase); + pRet->nPhrase--; + pParse->nPhrase--; + pPhrase = pLast; + }else if( pLast->nTerm==0 ){ + fts5ExprPhraseFree(pLast); + pParse->apPhrase[pParse->nPhrase-2] = pPhrase; + pParse->nPhrase--; + pRet->nPhrase--; + } + } + pRet->apPhrase[pRet->nPhrase++] = pPhrase; + } + return pRet; +} + +typedef struct TokenCtx TokenCtx; +struct TokenCtx { + Fts5ExprPhrase *pPhrase; + int rc; +}; + +/* +** Callback for tokenizing terms used by ParseTerm(). +*/ +static int fts5ParseTokenize( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int rc = SQLITE_OK; + const int SZALLOC = 8; + TokenCtx *pCtx = (TokenCtx*)pContext; + Fts5ExprPhrase *pPhrase = pCtx->pPhrase; + + UNUSED_PARAM2(iUnused1, iUnused2); + + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc64(nByte); + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, (size_t)nByte); + pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + + pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } + } + + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + } + } + + pCtx->rc = rc; + return rc; +} + + +/* +** Free the phrase object passed as the only argument. +*/ +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){ + fts5ExprPhraseFree(pPhrase); +} + +/* +** Free the phrase object passed as the second argument. +*/ +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){ + if( pNear ){ + int i; + for(i=0; inPhrase; i++){ + fts5ExprPhraseFree(pNear->apPhrase[i]); + } + sqlite3_free(pNear->pColset); + sqlite3_free(pNear); + } +} + +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){ + assert( pParse->pExpr==0 ); + pParse->pExpr = p; +} + +static int parseGrowPhraseArray(Fts5Parse *pParse){ + if( (pParse->nPhrase % 8)==0 ){ + sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); + Fts5ExprPhrase **apNew; + apNew = (Fts5ExprPhrase**)sqlite3_realloc64(pParse->apPhrase, nByte); + if( apNew==0 ){ + pParse->rc = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + pParse->apPhrase = apNew; + } + return SQLITE_OK; +} + +/* +** This function is called by the parser to process a string token. The +** string may or may not be quoted. In any case it is tokenized and a +** phrase object consisting of all tokens returned. +*/ +static Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprPhrase *pAppend, /* Phrase to append to */ + Fts5Token *pToken, /* String to tokenize */ + int bPrefix /* True if there is a trailing "*" */ +){ + Fts5Config *pConfig = pParse->pConfig; + TokenCtx sCtx; /* Context object passed to callback */ + int rc; /* Tokenize return code */ + char *z = 0; + + memset(&sCtx, 0, sizeof(TokenCtx)); + sCtx.pPhrase = pAppend; + + rc = fts5ParseStringFromToken(pToken, &z); + if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); + int n; + sqlite3Fts5Dequote(z); + n = (int)strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); + } + sqlite3_free(z); + if( rc || (rc = sCtx.rc) ){ + pParse->rc = rc; + fts5ExprPhraseFree(sCtx.pPhrase); + sCtx.pPhrase = 0; + }else{ + + if( pAppend==0 ){ + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(sCtx.pPhrase); + return 0; + } + pParse->nPhrase++; + } + + if( sCtx.pPhrase==0 ){ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + }else if( sCtx.pPhrase->nTerm ){ + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; + } + pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; + } + + return sCtx.pPhrase; +} + +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +static int sqlite3Fts5ExprClonePhrase( + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ + + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; + if( pColsetOrig ){ + sqlite3_int64 nByte; + Fts5Colset *pColset; + nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); + if( pColset ){ + memcpy(pColset, pColsetOrig, (size_t)nByte); + } + pNew->pRoot->pNear->pColset = pColset; + } + } + + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), + 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } + } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 + && pOrig->aTerm[0].pSynonym==0 + && pOrig->aTerm[0].bFirst==0 + ){ + pNew->pRoot->eType = FTS5_TERM; + pNew->pRoot->xNext = fts5ExprNodeNext_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + pNew->pRoot->xNext = fts5ExprNodeNext_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + +/* +** Token pTok has appeared in a MATCH expression where the NEAR operator +** is expected. If token pTok does not contain "NEAR", store an error +** in the pParse object. +*/ +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){ + if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p + ); + } +} + +static void sqlite3Fts5ParseSetDistance( + Fts5Parse *pParse, + Fts5ExprNearset *pNear, + Fts5Token *p +){ + if( pNear ){ + int nNear = 0; + int i; + if( p->n ){ + for(i=0; in; i++){ + char c = (char)p->p[i]; + if( c<'0' || c>'9' ){ + sqlite3Fts5ParseError( + pParse, "expected integer, got \"%.*s\"", p->n, p->p + ); + return; + } + nNear = nNear * 10 + (p->p[i] - '0'); + } + }else{ + nNear = FTS5_DEFAULT_NEARDIST; + } + pNear->nNear = nNear; + } +} + +/* +** The second argument passed to this function may be NULL, or it may be +** an existing Fts5Colset object. This function returns a pointer to +** a new colset object containing the contents of (p) with new value column +** number iCol appended. +** +** If an OOM error occurs, store an error code in pParse and return NULL. +** The old colset object (if any) is not freed in this case. +*/ +static Fts5Colset *fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *p, /* Existing colset object */ + int iCol /* New column to add to colset object */ +){ + int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */ + Fts5Colset *pNew; /* New colset object to return */ + + assert( pParse->rc==SQLITE_OK ); + assert( iCol>=0 && iColpConfig->nCol ); + + pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + if( pNew==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + int *aiCol = pNew->aiCol; + int i, j; + for(i=0; iiCol ) break; + } + for(j=nCol; j>i; j--){ + aiCol[j] = aiCol[j-1]; + } + aiCol[i] = iCol; + pNew->nCol = nCol+1; + +#ifndef NDEBUG + /* Check that the array is in order and contains no duplicate entries. */ + for(i=1; inCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] ); +#endif + } + + return pNew; +} + +/* +** Allocate and return an Fts5Colset object specifying the inverse of +** the colset passed as the second argument. Free the colset passed +** as the second argument before returning. +*/ +static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ + Fts5Colset *pRet; + int nCol = pParse->pConfig->nCol; + + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet ){ + int i; + int iOld = 0; + for(i=0; i=p->nCol || p->aiCol[iOld]!=i ){ + pRet->aiCol[pRet->nCol++] = i; + }else{ + iOld++; + } + } + } + + sqlite3_free(p); + return pRet; +} + +static Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *pColset, /* Existing colset object */ + Fts5Token *p +){ + Fts5Colset *pRet = 0; + int iCol; + char *z; /* Dequoted copy of token p */ + + z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n); + if( pParse->rc==SQLITE_OK ){ + Fts5Config *pConfig = pParse->pConfig; + sqlite3Fts5Dequote(z); + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break; + } + if( iCol==pConfig->nCol ){ + sqlite3Fts5ParseError(pParse, "no such column: %s", z); + }else{ + pRet = fts5ParseColset(pParse, pColset, iCol); + } + sqlite3_free(z); + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3_free(pColset); + } + + return pRet; +} + +/* +** If argument pOrig is NULL, or if (*pRc) is set to anything other than +** SQLITE_OK when this function is called, NULL is returned. +** +** Otherwise, a copy of (*pOrig) is made into memory obtained from +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned. +*/ +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ + Fts5Colset *pRet; + if( pOrig ){ + sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); + if( pRet ){ + memcpy(pRet, pOrig, (size_t)nByte); + } + }else{ + pRet = 0; + } + return pRet; +} + +/* +** Remove from colset pColset any columns that are not also in colset pMerge. +*/ +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){ + int iIn = 0; /* Next input in pColset */ + int iMerge = 0; /* Next input in pMerge */ + int iOut = 0; /* Next output slot in pColset */ + + while( iInnCol && iMergenCol ){ + int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge]; + if( iDiff==0 ){ + pColset->aiCol[iOut++] = pMerge->aiCol[iMerge]; + iMerge++; + iIn++; + }else if( iDiff>0 ){ + iMerge++; + }else{ + iIn++; + } + } + pColset->nCol = iOut; +} + +/* +** Recursively apply colset pColset to expression node pNode and all of +** its decendents. If (*ppFree) is not NULL, it contains a spare copy +** of pColset. This function may use the spare copy and set (*ppFree) to +** zero, or it may create copies of pColset using fts5CloneColset(). +*/ +static void fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pNode, + Fts5Colset *pColset, + Fts5Colset **ppFree +){ + if( pParse->rc==SQLITE_OK ){ + assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING + || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR + || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF + ); + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->pColset ){ + fts5MergeColset(pNear->pColset, pColset); + if( pNear->pColset->nCol==0 ){ + pNode->eType = FTS5_EOF; + pNode->xNext = 0; + } + }else if( *ppFree ){ + pNear->pColset = pColset; + *ppFree = 0; + }else{ + pNear->pColset = fts5CloneColset(&pParse->rc, pColset); + } + }else{ + int i; + assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 ); + for(i=0; inChild; i++){ + fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree); + } + } + } +} + +/* +** Apply colset pColset to expression node pExpr and all of its descendents. +*/ +static void sqlite3Fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pExpr, + Fts5Colset *pColset +){ + Fts5Colset *pFree = pColset; + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pParse->rc = SQLITE_ERROR; + pParse->zErr = sqlite3_mprintf( + "fts5: column queries are not supported (detail=none)" + ); + }else{ + fts5ParseSetColset(pParse, pExpr, pColset, &pFree); + } + sqlite3_free(pFree); +} + +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ + switch( pNode->eType ){ + case FTS5_STRING: { + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + && pNear->apPhrase[0]->aTerm[0].bFirst==0 + ){ + pNode->eType = FTS5_TERM; + pNode->xNext = fts5ExprNodeNext_TERM; + }else{ + pNode->xNext = fts5ExprNodeNext_STRING; + } + break; + }; + + case FTS5_OR: { + pNode->xNext = fts5ExprNodeNext_OR; + break; + }; + + case FTS5_AND: { + pNode->xNext = fts5ExprNodeNext_AND; + break; + }; + + default: assert( pNode->eType==FTS5_NOT ); { + pNode->xNext = fts5ExprNodeNext_NOT; + break; + }; + } +} + +static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ + int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; + memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); + p->nChild += pSub->nChild; + sqlite3_free(pSub); + }else{ + p->apChild[p->nChild++] = pSub; + } +} + +/* +** This function is used when parsing LIKE or GLOB patterns against +** trigram indexes that specify either detail=column or detail=none. +** It converts a phrase: +** +** abc + def + ghi +** +** into an AND tree: +** +** abc AND def AND ghi +*/ +static Fts5ExprNode *fts5ParsePhraseToAnd( + Fts5Parse *pParse, + Fts5ExprNearset *pNear +){ + int nTerm = pNear->apPhrase[0]->nTerm; + int ii; + int nByte; + Fts5ExprNode *pRet; + + assert( pNear->nPhrase==1 ); + assert( pParse->bPhraseToAnd ); + + nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + if( pRet ){ + pRet->eType = FTS5_AND; + pRet->nChild = nTerm; + fts5ExprAssignXNext(pRet); + pParse->nPhrase--; + for(ii=0; iirc, sizeof(Fts5ExprPhrase) + ); + if( pPhrase ){ + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(pPhrase); + }else{ + pParse->apPhrase[pParse->nPhrase++] = pPhrase; + pPhrase->nTerm = 1; + pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup( + &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1 + ); + pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, + 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) + ); + } + } + } + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + }else{ + sqlite3Fts5ParseNearsetFree(pNear); + } + } + + return pRet; +} + +/* +** Allocate and return a new expression object. If anything goes wrong (i.e. +** OOM error), leave an error code in pParse and return NULL. +*/ +static Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, /* Parse context */ + int eType, /* FTS5_STRING, AND, OR or NOT */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight, /* Right hand child expression */ + Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */ +){ + Fts5ExprNode *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + int nChild = 0; /* Number of children of returned node */ + sqlite3_int64 nByte; /* Bytes of space to allocate for this node */ + + assert( (eType!=FTS5_STRING && !pNear) + || (eType==FTS5_STRING && !pLeft && !pRight) + ); + if( eType==FTS5_STRING && pNear==0 ) return 0; + if( eType!=FTS5_STRING && pLeft==0 ) return pRight; + if( eType!=FTS5_STRING && pRight==0 ) return pLeft; + + if( eType==FTS5_STRING + && pParse->bPhraseToAnd + && pNear->apPhrase[0]->nTerm>1 + ){ + pRet = fts5ParsePhraseToAnd(pParse, pNear); + }else{ + if( eType==FTS5_NOT ){ + nChild = 2; + }else if( eType==FTS5_AND || eType==FTS5_OR ){ + nChild = 2; + if( pLeft->eType==eType ) nChild += pLeft->nChild-1; + if( pRight->eType==eType ) nChild += pRight->nChild-1; + } + + nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + + if( pRet ){ + pRet->eType = eType; + pRet->pNear = pNear; + fts5ExprAssignXNext(pRet); + if( eType==FTS5_STRING ){ + int iPhrase; + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + pNear->apPhrase[iPhrase]->pNode = pRet; + if( pNear->apPhrase[iPhrase]->nTerm==0 ){ + pRet->xNext = 0; + pRet->eType = FTS5_EOF; + } + } + + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + if( pNear->nPhrase!=1 + || pPhrase->nTerm>1 + || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst) + ){ + assert( pParse->rc==SQLITE_OK ); + pParse->rc = SQLITE_ERROR; + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_mprintf( + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3_free(pRet); + pRet = 0; + } + } + }else{ + fts5ExprAddChildren(pRet, pLeft); + fts5ExprAddChildren(pRet, pRight); + } + } + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + sqlite3Fts5ParseNearsetFree(pNear); + } + return pRet; +} + +static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight /* Right hand child expression */ +){ + Fts5ExprNode *pRet = 0; + Fts5ExprNode *pPrev; + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + }else{ + + assert( pLeft->eType==FTS5_STRING + || pLeft->eType==FTS5_TERM + || pLeft->eType==FTS5_EOF + || pLeft->eType==FTS5_AND + ); + assert( pRight->eType==FTS5_STRING + || pRight->eType==FTS5_TERM + || pRight->eType==FTS5_EOF + ); + + if( pLeft->eType==FTS5_AND ){ + pPrev = pLeft->apChild[pLeft->nChild-1]; + }else{ + pPrev = pLeft; + } + assert( pPrev->eType==FTS5_STRING + || pPrev->eType==FTS5_TERM + || pPrev->eType==FTS5_EOF + ); + + if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); + sqlite3Fts5ParseNodeFree(pRight); + pRet = pLeft; + pParse->nPhrase--; + } + else if( pPrev->eType==FTS5_EOF ){ + Fts5ExprPhrase **ap; + + if( pPrev==pLeft ){ + pRet = pRight; + }else{ + pLeft->apChild[pLeft->nChild-1] = pRight; + pRet = pLeft; + } + + ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; + assert( ap[0]==pPrev->pNear->apPhrase[0] ); + memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); + pParse->nPhrase--; + + sqlite3Fts5ParseNodeFree(pPrev); + } + else{ + pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); + } + } + + return pRet; +} + +static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ + sqlite3_int64 nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc64(nByte); + + if( zQuoted ){ + int i = 0; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->zTerm; + zQuoted[i++] = '"'; + while( *zIn ){ + if( *zIn=='"' ) zQuoted[i++] = '"'; + zQuoted[i++] = *zIn++; + } + zQuoted[i++] = '"'; + if( p->pSynonym ) zQuoted[i++] = '|'; + } + if( pTerm->bPrefix ){ + zQuoted[i++] = ' '; + zQuoted[i++] = '*'; + } + zQuoted[i++] = '\0'; + } + return zQuoted; +} + +static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){ + char *zNew; + va_list ap; + va_start(ap, zFmt); + zNew = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zApp && zNew ){ + char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); + sqlite3_free(zNew); + zNew = zNew2; + } + sqlite3_free(zApp); + return zNew; +} + +/* +** Compose a tcl-readable representation of expression pExpr. Return a +** pointer to a buffer containing that representation. It is the +** responsibility of the caller to at some point free the buffer using +** sqlite3_free(). +*/ +static char *fts5ExprPrintTcl( + Fts5Config *pConfig, + const char *zNearsetCmd, + Fts5ExprNode *pExpr +){ + char *zRet = 0; + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd); + if( zRet==0 ) return 0; + if( pNear->pColset ){ + int *aiCol = pNear->pColset->aiCol; + int nCol = pNear->pColset->nCol; + if( nCol==1 ){ + zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]); + }else{ + zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]); + for(i=1; ipColset->nCol; i++){ + zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]); + } + zRet = fts5PrintfAppend(zRet, "} "); + } + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear); + if( zRet==0 ) return 0; + } + + zRet = fts5PrintfAppend(zRet, "--"); + if( zRet==0 ) return 0; + + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + + zRet = fts5PrintfAppend(zRet, " {"); + for(iTerm=0; zRet && iTermnTerm; iTerm++){ + char *zTerm = pPhrase->aTerm[iTerm].zTerm; + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } + } + + if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); + if( zRet==0 ) return 0; + } + + }else{ + char const *zOp = 0; + int i; + switch( pExpr->eType ){ + case FTS5_AND: zOp = "AND"; break; + case FTS5_NOT: zOp = "NOT"; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = "OR"; + break; + } + + zRet = sqlite3_mprintf("%s", zOp); + for(i=0; zRet && inChild; i++){ + char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]); + if( !z ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + zRet = fts5PrintfAppend(zRet, " [%z]", z); + } + } + } + + return zRet; +} + +static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ + char *zRet = 0; + if( pExpr->eType==0 ){ + return sqlite3_mprintf("\"\""); + }else + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + if( pNear->pColset ){ + int ii; + Fts5Colset *pColset = pNear->pColset; + if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{"); + for(ii=0; iinCol; ii++){ + zRet = fts5PrintfAppend(zRet, "%s%s", + pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " " + ); + } + if( zRet ){ + zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : ""); + } + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "NEAR("); + if( zRet==0 ) return 0; + } + + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( i!=0 ){ + zRet = fts5PrintfAppend(zRet, " "); + if( zRet==0 ) return 0; + } + for(iTerm=0; iTermnTerm; iTerm++){ + char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]); + if( zTerm ){ + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm); + sqlite3_free(zTerm); + } + if( zTerm==0 || zRet==0 ){ + sqlite3_free(zRet); + return 0; + } + } + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear); + if( zRet==0 ) return 0; + } + + }else{ + char const *zOp = 0; + int i; + + switch( pExpr->eType ){ + case FTS5_AND: zOp = " AND "; break; + case FTS5_NOT: zOp = " NOT "; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = " OR "; + break; + } + + for(i=0; inChild; i++){ + char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); + if( z==0 ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + int e = pExpr->apChild[i]->eType; + int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); + zRet = fts5PrintfAppend(zRet, "%s%s%z%s", + (i==0 ? "" : zOp), + (b?"(":""), z, (b?")":"") + ); + } + if( zRet==0 ) break; + } + } + + return zRet; +} + +/* +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0) +** and fts5_expr_tcl() (bTcl!=0). +*/ +static void fts5ExprFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal, /* Function arguments */ + int bTcl +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zExpr = 0; + char *zErr = 0; + Fts5Expr *pExpr = 0; + int rc; + int i; + + const char **azConfig; /* Array of arguments for Fts5Config */ + const char *zNearsetCmd = "nearset"; + int nConfig; /* Size of azConfig[] */ + Fts5Config *pConfig = 0; + int iArg = 1; + + if( nArg<1 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function %s", + bTcl ? "fts5_expr_tcl" : "fts5_expr" + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + + if( bTcl && nArg>1 ){ + zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); + iArg = 2; + } + + nConfig = 3 + (nArg-iArg); + azConfig = (const char**)sqlite3_malloc64(sizeof(char*) * nConfig); + if( azConfig==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + azConfig[0] = 0; + azConfig[1] = "main"; + azConfig[2] = "tbl"; + for(i=3; iArgnCol, zExpr, &pExpr, &zErr); + } + if( rc==SQLITE_OK ){ + char *zText; + if( pExpr->pRoot->xNext==0 ){ + zText = sqlite3_mprintf(""); + }else if( bTcl ){ + zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); + }else{ + zText = fts5ExprPrint(pConfig, pExpr->pRoot); + } + if( zText==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + } + + if( rc!=SQLITE_OK ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free((void *)azConfig); + sqlite3Fts5ConfigFree(pConfig); + sqlite3Fts5ExprFree(pExpr); +} + +static void fts5ExprFunctionHr( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 0); +} +static void fts5ExprFunctionTcl( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 1); +} + +/* +** The implementation of an SQLite user-defined-function that accepts a +** single integer as an argument. If the integer is an alpha-numeric +** unicode code point, 1 is returned. Otherwise 0. +*/ +static void fts5ExprIsAlnum( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + int iCode; + u8 aArr[32]; + if( nArg!=1 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_isalnum", -1 + ); + return; + } + memset(aArr, 0, sizeof(aArr)); + sqlite3Fts5UnicodeCatParse("L*", aArr); + sqlite3Fts5UnicodeCatParse("N*", aArr); + sqlite3Fts5UnicodeCatParse("Co", aArr); + iCode = sqlite3_value_int(apVal[0]); + sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]); +} + +static void fts5ExprFold( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_fold", -1 + ); + }else{ + int iCode; + int bRemoveDiacritics = 0; + iCode = sqlite3_value_int(apVal[0]); + if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]); + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); + } +} + +/* +** This is called during initialization to register the fts5_expr() scalar +** UDF with the SQLite handle passed as the only argument. +*/ +static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ + struct Fts5ExprFunc { + const char *z; + void (*x)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "fts5_expr", fts5ExprFunctionHr }, + { "fts5_expr_tcl", fts5ExprFunctionTcl }, + { "fts5_isalnum", fts5ExprIsAlnum }, + { "fts5_fold", fts5ExprFold }, + }; + int i; + int rc = SQLITE_OK; + void *pCtx = (void*)pGlobal; + + for(i=0; rc==SQLITE_OK && iz, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); + } + + /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and + ** sqlite3Fts5ParserFallback() are unused */ +#ifndef NDEBUG + (void)sqlite3Fts5ParserTrace; +#endif + (void)sqlite3Fts5ParserFallback; + + return rc; +} + +/* +** Return the number of phrases in expression pExpr. +*/ +static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){ + return (pExpr ? pExpr->nPhrase : 0); +} + +/* +** Return the number of terms in the iPhrase'th phrase in pExpr. +*/ +static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){ + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0; + return pExpr->apExprPhrase[iPhrase]->nTerm; +} + +/* +** This function is used to access the current position list for phrase +** iPhrase. +*/ +static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ + int nRet; + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ + *pa = pPhrase->poslist.p; + nRet = pPhrase->poslist.n; + }else{ + *pa = 0; + nRet = 0; + } + return nRet; +} + +struct Fts5PoslistPopulator { + Fts5PoslistWriter writer; + int bOk; /* True if ok to populate */ + int bMiss; +}; + +static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ + Fts5PoslistPopulator *pRet; + pRet = sqlite3_malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + if( pRet ){ + int i; + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + if( bLive && + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) + ){ + pRet[i].bMiss = 1; + }else{ + pBuf->n = 0; + } + } + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistPopulator *aPopulator; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +/* +** TODO: Make this more efficient! +*/ +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static int fts5ExprPopulatePoslistsCb( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Byte offset of token within input text */ + int iUnused2 /* Byte offset of end of token within input text */ +){ + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; + Fts5Expr *pExpr = p->pExpr; + int i; + + UNUSED_PARAM2(iUnused1, iUnused2); + + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pTerm; + if( p->aPopulator[i].bOk==0 ) continue; + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + int nTerm = (int)strlen(pTerm->zTerm); + if( (nTerm==nToken || (nTermbPrefix)) + && memcmp(pTerm->zTerm, pToken, nTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff + ); + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +static int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistPopulator *aPopulator, + int iCol, + const char *z, int n +){ + int i; + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aPopulator = aPopulator; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + for(i=0; inPhrase; i++){ + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + Fts5Colset *pColset = pNode->pNear->pColset; + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + || aPopulator[i].bMiss + ){ + aPopulator[i].bOk = 0; + }else{ + aPopulator[i].bOk = 1; + } + } + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + break; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + break; + } + } + return 1; +} + +static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + +/* +** This function is only called for detail=columns tables. +*/ +static int sqlite3Fts5ExprPhraseCollist( + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, + int *pnCollist +){ + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + int rc = SQLITE_OK; + + assert( iPhrase>=0 && iPhrasenPhrase ); + assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid + && pPhrase->poslist.n>0 + ){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; + rc = fts5ExprSynonymList( + pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist + ); + }else{ + *ppCollist = pPhrase->aTerm[0].pIter->pData; + *pnCollist = pPhrase->aTerm[0].pIter->nData; + } + }else{ + *ppCollist = 0; + *pnCollist = 0; + } + + return rc; +} + +#line 1 "fts5_hash.c" +/* +** 2014 August 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +typedef struct Fts5HashEntry Fts5HashEntry; + +/* +** This file contains the implementation of an in-memory hash table used +** to accumuluate "term -> doclist" content before it is flused to a level-0 +** segment. +*/ + + +struct Fts5Hash { + int eDetail; /* Copy of Fts5Config.eDetail */ + int *pnByte; /* Pointer to bytes counter */ + int nEntry; /* Number of entries currently in hash */ + int nSlot; /* Size of aSlot[] array */ + Fts5HashEntry *pScan; /* Current ordered scan item */ + Fts5HashEntry **aSlot; /* Array of hash slots */ +}; + +/* +** Each entry in the hash table is represented by an object of the +** following type. Each object, its key (a nul-terminated string) and +** its current data are stored in a single memory allocation. The +** key immediately follows the object in memory. The position list +** data immediately follows the key data in memory. +** +** The data that follows the key is in a similar, but not identical format +** to the doclist data stored in the database. It is: +** +** * Rowid, as a varint +** * Position list, without 0x00 terminator. +** * Size of previous position list and rowid, as a 4 byte +** big-endian integer. +** +** iRowidOff: +** Offset of last rowid written to data area. Relative to first byte of +** structure. +** +** nData: +** Bytes of data written since iRowidOff. +*/ +struct Fts5HashEntry { + Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ + Fts5HashEntry *pScanNext; /* Next entry in sorted order */ + + int nAlloc; /* Total size of allocation */ + int iSzPoslist; /* Offset of space for 4-byte poslist size */ + int nData; /* Total bytes of data (incl. structure) */ + int nKey; /* Length of key in bytes */ + u8 bDel; /* Set delete-flag @ iSzPoslist */ + u8 bContent; /* Set content-flag (detail=none mode) */ + i16 iCol; /* Column of last value written */ + int iPos; /* Position of last value written */ + i64 iRowid; /* Rowid of last value written */ +}; + +/* +** Eqivalent to: +** +** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } +*/ +#define fts5EntryKey(p) ( ((char *)(&(p)[1])) ) + + +/* +** Allocate a new hash table. +*/ +static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ + int rc = SQLITE_OK; + Fts5Hash *pNew; + + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_int64 nByte; + memset(pNew, 0, sizeof(Fts5Hash)); + pNew->pnByte = pnByte; + pNew->eDetail = pConfig->eDetail; + + pNew->nSlot = 1024; + nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; + pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64(nByte); + if( pNew->aSlot==0 ){ + sqlite3_free(pNew); + *ppNew = 0; + rc = SQLITE_NOMEM; + }else{ + memset(pNew->aSlot, 0, (size_t)nByte); + } + } + return rc; +} + +/* +** Free a hash table object. +*/ +static void sqlite3Fts5HashFree(Fts5Hash *pHash){ + if( pHash ){ + sqlite3Fts5HashClear(pHash); + sqlite3_free(pHash->aSlot); + sqlite3_free(pHash); + } +} + +/* +** Empty (but do not delete) a hash table. +*/ +static void sqlite3Fts5HashClear(Fts5Hash *pHash){ + int i; + for(i=0; inSlot; i++){ + Fts5HashEntry *pNext; + Fts5HashEntry *pSlot; + for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){ + pNext = pSlot->pHashNext; + sqlite3_free(pSlot); + } + } + memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); + pHash->nEntry = 0; +} + +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + return (h % nSlot); +} + +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + h = (h << 3) ^ h ^ b; + return (h % nSlot); +} + +/* +** Resize the hash table by doubling the number of slots. +*/ +static int fts5HashResize(Fts5Hash *pHash){ + int nNew = pHash->nSlot*2; + int i; + Fts5HashEntry **apNew; + Fts5HashEntry **apOld = pHash->aSlot; + + apNew = (Fts5HashEntry**)sqlite3_malloc64(nNew*sizeof(Fts5HashEntry*)); + if( !apNew ) return SQLITE_NOMEM; + memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); + + for(i=0; inSlot; i++){ + while( apOld[i] ){ + unsigned int iHash; + Fts5HashEntry *p = apOld[i]; + apOld[i] = p->pHashNext; + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), + (int)strlen(fts5EntryKey(p))); + p->pHashNext = apNew[iHash]; + apNew[iHash] = p; + } + } + + sqlite3_free(apOld); + pHash->nSlot = nNew; + pHash->aSlot = apNew; + return SQLITE_OK; +} + +static int fts5HashAddPoslistSize( + Fts5Hash *pHash, + Fts5HashEntry *p, + Fts5HashEntry *p2 +){ + int nRet = 0; + if( p->iSzPoslist ){ + u8 *pPtr = p2 ? (u8*)p2 : (u8*)p; + int nData = p->nData; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + assert( nData==p->iSzPoslist ); + if( p->bDel ){ + pPtr[nData++] = 0x00; + if( p->bContent ){ + pPtr[nData++] = 0x00; + } + } + }else{ + int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */ + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ + + assert( p->bDel==0 || p->bDel==1 ); + if( nPos<=127 ){ + pPtr[p->iSzPoslist] = (u8)nPos; + }else{ + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); + nData += (nByte-1); + } + } + + nRet = nData - p->nData; + if( p2==0 ){ + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; + p->nData = nData; + } + } + return nRet; +} + +/* +** Add an entry to the in-memory hash table. The key is the concatenation +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). +** +** (bByte || pToken) -> (iRowid,iCol,iPos) +** +** Or, if iCol is negative, then the value is a delete marker. +*/ +static int sqlite3Fts5HashWrite( + Fts5Hash *pHash, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, /* First byte of token */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + unsigned int iHash; + Fts5HashEntry *p; + u8 *pPtr; + int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); + + /* Attempt to locate an existing hash entry */ + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + char *zKey = fts5EntryKey(p); + if( zKey[0]==bByte + && p->nKey==nToken + && memcmp(&zKey[1], pToken, nToken)==0 + ){ + break; + } + } + + /* If an existing hash entry cannot be found, create a new one. */ + if( p==0 ){ + /* Figure out how much space to allocate */ + char *zKey; + sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; + if( nByte<128 ) nByte = 128; + + /* Grow the Fts5Hash.aSlot[] array if necessary. */ + if( (pHash->nEntry*2)>=pHash->nSlot ){ + int rc = fts5HashResize(pHash); + if( rc!=SQLITE_OK ) return rc; + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + } + + /* Allocate new Fts5HashEntry and add it to the hash table. */ + p = (Fts5HashEntry*)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, sizeof(Fts5HashEntry)); + p->nAlloc = (int)nByte; + zKey = fts5EntryKey(p); + zKey[0] = bByte; + memcpy(&zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); + p->nKey = nToken; + zKey[nToken+1] = '\0'; + p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); + p->pHashNext = pHash->aSlot[iHash]; + pHash->aSlot[iHash] = p; + pHash->nEntry++; + + /* Add the first rowid field to the hash-entry */ + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); + p->iRowid = iRowid; + + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + } + + }else{ + + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario + ** is: + ** + ** + 9 bytes for a new rowid, + ** + 4 byte reserved for the "poslist size" varint. + ** + 1 byte for a "new column" byte, + ** + 3 bytes for a new column number (16-bit max) as a varint, + ** + 5 bytes for the new position offset (32-bit max). + */ + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ + sqlite3_int64 nNew = p->nAlloc * 2; + Fts5HashEntry *pNew; + Fts5HashEntry **pp; + pNew = (Fts5HashEntry*)sqlite3_realloc64(p, nNew); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->nAlloc = (int)nNew; + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); + *pp = pNew; + p = pNew; + } + nIncr -= p->nData; + } + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); + + pPtr = (u8*)p; + + /* If this is a new rowid, append the 4-byte size field for the previous + ** entry, and the new rowid for this entry. */ + if( iRowid!=p->iRowid ){ + u64 iDiff = (u64)iRowid - (u64)p->iRowid; + fts5HashAddPoslistSize(pHash, p, 0); + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iDiff); + p->iRowid = iRowid; + bNew = 1; + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + p->iPos = 0; + } + } + + if( iCol>=0 ){ + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + p->bContent = 1; + }else{ + /* Append a new column value, if necessary */ + assert( iCol>=p->iCol ); + if( iCol!=p->iCol ){ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = (i16)iCol; + p->iPos = 0; + }else{ + bNew = 1; + p->iCol = (i16)(iPos = iCol); + } + } + + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } + } + }else{ + /* This is a delete. Set the delete flag. */ + p->bDel = 1; + } + + nIncr += p->nData; + *pHash->pnByte += nIncr; + return SQLITE_OK; +} + + +/* +** Arguments pLeft and pRight point to linked-lists of hash-entry objects, +** each sorted in key order. This function merges the two lists into a +** single list and returns a pointer to its first element. +*/ +static Fts5HashEntry *fts5HashEntryMerge( + Fts5HashEntry *pLeft, + Fts5HashEntry *pRight +){ + Fts5HashEntry *p1 = pLeft; + Fts5HashEntry *p2 = pRight; + Fts5HashEntry *pRet = 0; + Fts5HashEntry **ppOut = &pRet; + + while( p1 || p2 ){ + if( p1==0 ){ + *ppOut = p2; + p2 = 0; + }else if( p2==0 ){ + *ppOut = p1; + p1 = 0; + }else{ + int i = 0; + char *zKey1 = fts5EntryKey(p1); + char *zKey2 = fts5EntryKey(p2); + while( zKey1[i]==zKey2[i] ) i++; + + if( ((u8)zKey1[i])>((u8)zKey2[i]) ){ + /* p2 is smaller */ + *ppOut = p2; + ppOut = &p2->pScanNext; + p2 = p2->pScanNext; + }else{ + /* p1 is smaller */ + *ppOut = p1; + ppOut = &p1->pScanNext; + p1 = p1->pScanNext; + } + *ppOut = 0; + } + } + + return pRet; +} + +/* +** Extract all tokens from hash table iHash and link them into a list +** in sorted order. The hash table is cleared before returning. It is +** the responsibility of the caller to free the elements of the returned +** list. +*/ +static int fts5HashEntrySort( + Fts5Hash *pHash, + const char *pTerm, int nTerm, /* Query prefix, if any */ + Fts5HashEntry **ppSorted +){ + const int nMergeSlot = 32; + Fts5HashEntry **ap; + Fts5HashEntry *pList; + int iSlot; + int i; + + *ppSorted = 0; + ap = sqlite3_malloc64(sizeof(Fts5HashEntry*) * nMergeSlot); + if( !ap ) return SQLITE_NOMEM; + memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); + + for(iSlot=0; iSlotnSlot; iSlot++){ + Fts5HashEntry *pIter; + for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ + if( pTerm==0 + || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + ){ + Fts5HashEntry *pEntry = pIter; + pEntry->pScanNext = 0; + for(i=0; ap[i]; i++){ + pEntry = fts5HashEntryMerge(pEntry, ap[i]); + ap[i] = 0; + } + ap[i] = pEntry; + } + } + } + + pList = 0; + for(i=0; inEntry = 0; + sqlite3_free(ap); + *ppSorted = pList; + return SQLITE_OK; +} + +/* +** Query the hash table for a doclist associated with term pTerm/nTerm. +*/ +static int sqlite3Fts5HashQuery( + Fts5Hash *pHash, /* Hash table to query */ + int nPre, + const char *pTerm, int nTerm, /* Query term */ + void **ppOut, /* OUT: Pointer to new object */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +){ + unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); + char *zKey = 0; + Fts5HashEntry *p; + + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + zKey = fts5EntryKey(p); + assert( p->nKey+1==(int)strlen(zKey) ); + if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break; + } + + if( p ){ + int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1; + int nList = p->nData - nHashPre; + u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); + if( pRet ){ + Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre]; + memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList); + nList += fts5HashAddPoslistSize(pHash, p, pFaux); + *pnDoclist = nList; + }else{ + *pnDoclist = 0; + return SQLITE_NOMEM; + } + }else{ + *ppOut = 0; + *pnDoclist = 0; + } + + return SQLITE_OK; +} + +static int sqlite3Fts5HashScanInit( + Fts5Hash *p, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +){ + return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); +} + +static void sqlite3Fts5HashScanNext(Fts5Hash *p){ + assert( !sqlite3Fts5HashScanEof(p) ); + p->pScan = p->pScan->pScanNext; +} + +static int sqlite3Fts5HashScanEof(Fts5Hash *p){ + return (p->pScan==0); +} + +static void sqlite3Fts5HashScanEntry( + Fts5Hash *pHash, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +){ + Fts5HashEntry *p; + if( (p = pHash->pScan) ){ + char *zKey = fts5EntryKey(p); + int nTerm = (int)strlen(zKey); + fts5HashAddPoslistSize(pHash, p, 0); + *pzTerm = zKey; + *ppDoclist = (const u8*)&zKey[nTerm+1]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); + }else{ + *pzTerm = 0; + *ppDoclist = 0; + *pnDoclist = 0; + } +} + +#line 1 "fts5_index.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Low level access to the FTS index stored in the database file. The +** routines in this file file implement all read and write access to the +** %_data table. Other parts of the system access this functionality via +** the interface defined in fts5Int.h. +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +/* +** Overview: +** +** The %_data table contains all the FTS indexes for an FTS5 virtual table. +** As well as the main term index, there may be up to 31 prefix indexes. +** The format is similar to FTS3/4, except that: +** +** * all segment b-tree leaf data is stored in fixed size page records +** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is +** taken to ensure it is possible to iterate in either direction through +** the entries in a doclist, or to seek to a specific entry within a +** doclist, without loading it into memory. +** +** * large doclists that span many pages have associated "doclist index" +** records that contain a copy of the first rowid on each page spanned by +** the doclist. This is used to speed up seek operations, and merges of +** large doclists with very small doclists. +** +** * extra fields in the "structure record" record the state of ongoing +** incremental merge operations. +** +*/ + + +#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ +#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ + +#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ + +#define FTS5_MAIN_PREFIX '0' + +#if FTS5_MAX_PREFIX_INDEXES > 31 +# error "FTS5_MAX_PREFIX_INDEXES is too large" +#endif + +/* +** Details: +** +** The %_data table managed by this module, +** +** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); +** +** , contains the following 5 types of records. See the comments surrounding +** the FTS5_*_ROWID macros below for a description of how %_data rowids are +** assigned to each fo them. +** +** 1. Structure Records: +** +** The set of segments that make up an index - the index structure - are +** recorded in a single record within the %_data table. The record consists +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. If the FTS table features more than one index (because +** there are one or more prefix indexes), it is guaranteed that all share +** the same cookie value. +** +** Immediately following the configuration cookie, the record begins with +** three varints: +** +** + number of levels, +** + total number of segments on all levels, +** + value of write counter. +** +** Then, for each level from 0 to nMax: +** +** + number of input segments in ongoing merge. +** + total number of segments in level. +** + for each segment from oldest to newest: +** + segment id (always > 0) +** + first leaf page number (often 1, always greater than 0) +** + final leaf page number +** +** 2. The Averages Record: +** +** A single record within the %_data table. The data is a list of varints. +** The first value is the number of rows in the index. Then, for each column +** from left to right, the total number of tokens in the column for all +** rows of the table. +** +** 3. Segment leaves: +** +** TERM/DOCLIST FORMAT: +** +** Most of each segment leaf is taken up by term/doclist data. The +** general format of term/doclist, starting with the first term +** on the leaf page, is: +** +** varint : size of first term +** blob: first term data +** doclist: first doclist +** zero-or-more { +** varint: number of bytes in common with previous term +** varint: number of bytes of new term data (nNew) +** blob: nNew bytes of new term data +** doclist: next doclist +** } +** +** doclist format: +** +** varint: first rowid +** poslist: first poslist +** zero-or-more { +** varint: rowid delta (always > 0) +** poslist: next poslist +** } +** +** poslist format: +** +** varint: size of poslist in bytes multiplied by 2, not including +** this field. Plus 1 if this entry carries the "delete" flag. +** collist: collist for column 0 +** zero-or-more { +** 0x01 byte +** varint: column number (I) +** collist: collist for column I +** } +** +** collist format: +** +** varint: first offset + 2 +** zero-or-more { +** varint: offset delta + 2 +** } +** +** PAGE FORMAT +** +** Each leaf page begins with a 4-byte header containing 2 16-bit +** unsigned integer fields in big-endian format. They are: +** +** * The byte offset of the first rowid on the page, if it exists +** and occurs before the first term (otherwise 0). +** +** * The byte offset of the start of the page footer. If the page +** footer is 0 bytes in size, then this field is the same as the +** size of the leaf page in bytes. +** +** The page footer consists of a single varint for each term located +** on the page. Each varint is the byte offset of the current term +** within the page, delta-compressed against the previous value. In +** other words, the first varint in the footer is the byte offset of +** the first term, the second is the byte offset of the second less that +** of the first, and so on. +** +** The term/doclist format described above is accurate if the entire +** term/doclist data fits on a single leaf page. If this is not the case, +** the format is changed in two ways: +** +** + if the first rowid on a page occurs before the first term, it +** is stored as a literal value: +** +** varint: first rowid +** +** + the first term on each page is stored in the same way as the +** very first term of the segment: +** +** varint : size of first term +** blob: first term data +** +** 5. Segment doclist indexes: +** +** Doclist indexes are themselves b-trees, however they usually consist of +** a single leaf record only. The format of each doclist index leaf page +** is: +** +** * Flags byte. Bits are: +** 0x01: Clear if leaf is also the root page, otherwise set. +** +** * Page number of fts index leaf page. As a varint. +** +** * First rowid on page indicated by previous field. As a varint. +** +** * A list of varints, one for each subsequent termless page. A +** positive delta if the termless page contains at least one rowid, +** or an 0x00 byte otherwise. +** +** Internal doclist index nodes are: +** +** * Flags byte. Bits are: +** 0x01: Clear for root page, otherwise set. +** +** * Page number of first child page. As a varint. +** +** * Copy of first rowid on page indicated by previous field. As a varint. +** +** * A list of delta-encoded varints - the first rowid on each subsequent +** child page. +** +*/ + +/* +** Rowids for the averages and structure records in the %_data table. +*/ +#define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ + +/* +** Macros determining the rowids used by segment leaves and dlidx leaves +** and nodes. All nodes and leaves are stored in the %_data table with large +** positive rowids. +** +** Each segment has a unique non-zero 16-bit id. +** +** The rowid for each segment leaf is found by passing the segment id and +** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered +** sequentially starting from 1. +*/ +#define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ +#define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */ +#define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */ +#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ + +#define fts5_dri(segid, dlidx, height, pgno) ( \ + ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \ + ((i64)(dlidx) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ + ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ + ((i64)(pgno)) \ +) + +#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) +#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) + +#ifdef SQLITE_DEBUG +static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } +#endif + + +/* +** Each time a blob is read from the %_data table, it is padded with this +** many zero bytes. This makes it easier to decode the various record formats +** without overreading if the records are corrupt. +*/ +#define FTS5_DATA_ZERO_PADDING 8 +#define FTS5_DATA_PADDING 20 + +typedef struct Fts5Data Fts5Data; +typedef struct Fts5DlidxIter Fts5DlidxIter; +typedef struct Fts5DlidxLvl Fts5DlidxLvl; +typedef struct Fts5DlidxWriter Fts5DlidxWriter; +typedef struct Fts5Iter Fts5Iter; +typedef struct Fts5PageWriter Fts5PageWriter; +typedef struct Fts5SegIter Fts5SegIter; +typedef struct Fts5DoclistIter Fts5DoclistIter; +typedef struct Fts5SegWriter Fts5SegWriter; +typedef struct Fts5Structure Fts5Structure; +typedef struct Fts5StructureLevel Fts5StructureLevel; +typedef struct Fts5StructureSegment Fts5StructureSegment; + +struct Fts5Data { + u8 *p; /* Pointer to buffer containing record */ + int nn; /* Size of record in bytes */ + int szLeaf; /* Size of leaf without page-index */ +}; + +/* +** One object per %_data table. +*/ +struct Fts5Index { + Fts5Config *pConfig; /* Virtual table configuration */ + char *zDataTbl; /* Name of %_data table */ + int nWorkUnit; /* Leaf pages in a "unit" of work */ + + /* + ** Variables related to the accumulation of tokens and doclists within the + ** in-memory hash tables before they are flushed to disk. + */ + Fts5Hash *pHash; /* Hash table for in-memory data */ + int nPendingData; /* Current bytes of pending data */ + i64 iWriteRowid; /* Rowid for current doc being written */ + int bDelete; /* Current write is a delete */ + + /* Error state. */ + int rc; /* Current error code */ + + /* State used by the fts5DataXXX() functions. */ + sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ + sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ + sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxSelect; + int nRead; /* Total number of blocks read */ + + sqlite3_stmt *pDataVersion; + i64 iStructVersion; /* data_version when pStruct read */ + Fts5Structure *pStruct; /* Current db structure (or NULL) */ +}; + +struct Fts5DoclistIter { + u8 *aEof; /* Pointer to 1 byte past end of doclist */ + + /* Output variables. aPoslist==0 at EOF */ + i64 iRowid; + u8 *aPoslist; + int nPoslist; + int nSize; +}; + +/* +** The contents of the "structure" record for each index are represented +** using an Fts5Structure record in memory. Which uses instances of the +** other Fts5StructureXXX types as components. +*/ +struct Fts5StructureSegment { + int iSegid; /* Segment id */ + int pgnoFirst; /* First leaf page number in segment */ + int pgnoLast; /* Last leaf page number in segment */ +}; +struct Fts5StructureLevel { + int nMerge; /* Number of segments in incr-merge */ + int nSeg; /* Total number of segments on level */ + Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ +}; +struct Fts5Structure { + int nRef; /* Object reference count */ + u64 nWriteCounter; /* Total leaves written to level 0 */ + int nSegment; /* Total segments in this structure */ + int nLevel; /* Number of levels in this index */ + Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ +}; + +/* +** An object of type Fts5SegWriter is used to write to segments. +*/ +struct Fts5PageWriter { + int pgno; /* Page number for this page */ + int iPrevPgidx; /* Previous value written into pgidx */ + Fts5Buffer buf; /* Buffer containing leaf data */ + Fts5Buffer pgidx; /* Buffer containing page-index */ + Fts5Buffer term; /* Buffer containing previous term on page */ +}; +struct Fts5DlidxWriter { + int pgno; /* Page number for this page */ + int bPrevValid; /* True if iPrev is valid */ + i64 iPrev; /* Previous rowid value written to page */ + Fts5Buffer buf; /* Buffer containing page data */ +}; +struct Fts5SegWriter { + int iSegid; /* Segid to write to */ + Fts5PageWriter writer; /* PageWriter object */ + i64 iPrevRowid; /* Previous rowid written to current leaf */ + u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ + u8 bFirstRowidInPage; /* True if next rowid is first in page */ + /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */ + u8 bFirstTermInPage; /* True if next term will be first in leaf */ + int nLeafWritten; /* Number of leaf pages written */ + int nEmpty; /* Number of contiguous term-less nodes */ + + int nDlidx; /* Allocated size of aDlidx[] array */ + Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ + + /* Values to insert into the %_idx table */ + Fts5Buffer btterm; /* Next term to insert into %_idx table */ + int iBtPage; /* Page number corresponding to btterm */ +}; + +typedef struct Fts5CResult Fts5CResult; +struct Fts5CResult { + u16 iFirst; /* aSeg[] index of firstest iterator */ + u8 bTermEq; /* True if the terms are equal */ +}; + +/* +** Object for iterating through a single segment, visiting each term/rowid +** pair in the segment. +** +** pSeg: +** The segment to iterate through. +** +** iLeafPgno: +** Current leaf page number within segment. +** +** iLeafOffset: +** Byte offset within the current leaf that is the first byte of the +** position list data (one byte passed the position-list size field). +** rowid field of the current entry. Usually this is the size field of the +** position list data. The exception is if the rowid for the current entry +** is the last thing on the leaf page. +** +** pLeaf: +** Buffer containing current leaf page data. Set to NULL at EOF. +** +** iTermLeafPgno, iTermLeafOffset: +** Leaf page number containing the last term read from the segment. And +** the offset immediately following the term data. +** +** flags: +** Mask of FTS5_SEGITER_XXX values. Interpreted as follows: +** +** FTS5_SEGITER_ONETERM: +** If set, set the iterator to point to EOF after the current doclist +** has been exhausted. Do not proceed to the next term in the segment. +** +** FTS5_SEGITER_REVERSE: +** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If +** it is set, iterate through rowid in descending order instead of the +** default ascending order. +** +** iRowidOffset/nRowidOffset/aRowidOffset: +** These are used if the FTS5_SEGITER_REVERSE flag is set. +** +** For each rowid on the page corresponding to the current term, the +** corresponding aRowidOffset[] entry is set to the byte offset of the +** start of the "position-list-size" field within the page. +** +** iTermIdx: +** Index of current term on iTermLeafPgno. +*/ +struct Fts5SegIter { + Fts5StructureSegment *pSeg; /* Segment to iterate through */ + int flags; /* Mask of configuration flags */ + int iLeafPgno; /* Current leaf page number */ + Fts5Data *pLeaf; /* Current leaf data */ + Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ + int iLeafOffset; /* Byte offset within current leaf */ + + /* Next method */ + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); + + /* The page and offset from which the current term was read. The offset + ** is the offset of the first rowid in the current doclist. */ + int iTermLeafPgno; + int iTermLeafOffset; + + int iPgidxOff; /* Next offset in pgidx */ + int iEndofDoclist; + + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ + int iRowidOffset; /* Current entry in aRowidOffset[] */ + int nRowidOffset; /* Allocated size of aRowidOffset[] array */ + int *aRowidOffset; /* Array of offset to rowid fields */ + + Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ + + /* Variables populated based on current entry. */ + Fts5Buffer term; /* Current term */ + i64 iRowid; /* Current rowid */ + int nPos; /* Number of bytes in current position list */ + u8 bDel; /* True if the delete flag is set */ +}; + +/* +** Argument is a pointer to an Fts5Data structure that contains a +** leaf page. +*/ +#define ASSERT_SZLEAF_OK(x) assert( \ + (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ +) + +#define FTS5_SEGITER_ONETERM 0x01 +#define FTS5_SEGITER_REVERSE 0x02 + +/* +** Argument is a pointer to an Fts5Data structure that contains a leaf +** page. This macro evaluates to true if the leaf contains no terms, or +** false if it contains at least one term. +*/ +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) + +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) + +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) + +/* +** Object for iterating through the merged results of one or more segments, +** visiting each term/rowid pair in the merged data. +** +** nSeg is always a power of two greater than or equal to the number of +** segments that this object is merging data from. Both the aSeg[] and +** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded +** with zeroed objects - these are handled as if they were iterators opened +** on empty segments. +** +** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the +** comparison in this context is the index of the iterator that currently +** points to the smaller term/rowid combination. Iterators at EOF are +** considered to be greater than all other iterators. +** +** aFirst[1] contains the index in aSeg[] of the iterator that points to +** the smallest key overall. aFirst[0] is unused. +** +** poslist: +** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. +** There is no way to tell if this is populated or not. +*/ +struct Fts5Iter { + Fts5IndexIter base; /* Base class containing output vars */ + + Fts5Index *pIndex; /* Index that owns this iterator */ + Fts5Buffer poslist; /* Buffer containing current poslist */ + Fts5Colset *pColset; /* Restrict matches to these columns */ + + /* Invoked to set output variables. */ + void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); + + int nSeg; /* Size of aSeg[] array */ + int bRev; /* True to iterate in reverse order */ + u8 bSkipEmpty; /* True to skip deleted entries */ + + i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ + Fts5CResult *aFirst; /* Current merge state (see above) */ + Fts5SegIter aSeg[1]; /* Array of segment iterators */ +}; + + +/* +** An instance of the following type is used to iterate through the contents +** of a doclist-index record. +** +** pData: +** Record containing the doclist-index data. +** +** bEof: +** Set to true once iterator has reached EOF. +** +** iOff: +** Set to the current offset within record pData. +*/ +struct Fts5DlidxLvl { + Fts5Data *pData; /* Data for current page of this level */ + int iOff; /* Current offset into pData */ + int bEof; /* At EOF already */ + int iFirstOff; /* Used by reverse iterators */ + + /* Output variables */ + int iLeafPgno; /* Page number of current leaf page */ + i64 iRowid; /* First rowid on leaf iLeafPgno */ +}; +struct Fts5DlidxIter { + int nLvl; + int iSegid; + Fts5DlidxLvl aLvl[1]; +}; + +static void fts5PutU16(u8 *aOut, u16 iVal){ + aOut[0] = (iVal>>8); + aOut[1] = (iVal&0xFF); +} + +static u16 fts5GetU16(const u8 *aIn){ + return ((u16)aIn[0] << 8) + aIn[1]; +} + +/* +** Allocate and return a buffer at least nByte bytes in size. +** +** If an OOM error is encountered, return NULL and set the error code in +** the Fts5Index handle passed as the first argument. +*/ +static void *fts5IdxMalloc(Fts5Index *p, sqlite3_int64 nByte){ + return sqlite3Fts5MallocZero(&p->rc, nByte); +} + +/* +** Compare the contents of the pLeft buffer with the pRight/nRight blob. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +#ifdef SQLITE_DEBUG +static int fts5BufferCompareBlob( + Fts5Buffer *pLeft, /* Left hand side of comparison */ + const u8 *pRight, int nRight /* Right hand side of comparison */ +){ + int nCmp = MIN(pLeft->n, nRight); + int res = memcmp(pLeft->p, pRight, nCmp); + return (res==0 ? (pLeft->n - nRight) : res); +} +#endif + +/* +** Compare the contents of the two buffers using memcmp(). If one buffer +** is a prefix of the other, it is considered the lesser. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ + int nCmp = MIN(pLeft->n, pRight->n); + int res = fts5Memcmp(pLeft->p, pRight->p, nCmp); + return (res==0 ? (pLeft->n - pRight->n) : res); +} + +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ + int ret; + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); + return ret; +} + +/* +** Close the read-only blob handle, if it is open. +*/ +static void sqlite3Fts5IndexCloseReader(Fts5Index *p){ + if( p->pReader ){ + sqlite3_blob *pReader = p->pReader; + p->pReader = 0; + sqlite3_blob_close(pReader); + } +} + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = 0; + if( p->rc==SQLITE_OK ){ + int rc = SQLITE_OK; + + if( p->pReader ){ + /* This call may return SQLITE_ABORT if there has been a savepoint + ** rollback since it was last used. In this case a new blob handle + ** is required. */ + sqlite3_blob *pBlob = p->pReader; + p->pReader = 0; + rc = sqlite3_blob_reopen(pBlob, iRowid); + assert( p->pReader==0 ); + p->pReader = pBlob; + if( rc!=SQLITE_OK ){ + sqlite3Fts5IndexCloseReader(p); + } + if( rc==SQLITE_ABORT ) rc = SQLITE_OK; + } + + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ + if( p->pReader==0 && rc==SQLITE_OK ){ + Fts5Config *pConfig = p->pConfig; + rc = sqlite3_blob_open(pConfig->db, + pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader + ); + } + + /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls + ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. + ** All the reasons those functions might return SQLITE_ERROR - missing + ** table, missing row, non-blob/text in block column - indicate + ** backing store corruption. */ + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + + if( rc==SQLITE_OK ){ + u8 *aOut = 0; /* Read blob data into this buffer */ + int nByte = sqlite3_blob_bytes(p->pReader); + sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); + if( pRet ){ + pRet->nn = nByte; + aOut = pRet->p = (u8*)&pRet[1]; + }else{ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pRet); + pRet = 0; + }else{ + /* TODO1: Fix this */ + pRet->p[nByte] = 0x00; + pRet->p[nByte+1] = 0x00; + pRet->szLeaf = fts5GetU16(&pRet->p[2]); + } + } + p->rc = rc; + p->nRead++; + } + + assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + return pRet; +} + +/* +** Release a reference to data record returned by an earlier call to +** fts5DataRead(). +*/ +static void fts5DataRelease(Fts5Data *pData){ + sqlite3_free(pData); +} + +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = fts5DataRead(p, iRowid); + if( pRet ){ + if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ + p->rc = FTS5_CORRUPT; + fts5DataRelease(pRet); + pRet = 0; + } + } + return pRet; +} + +static int fts5IndexPrepareStmt( + Fts5Index *p, + sqlite3_stmt **ppStmt, + char *zSql +){ + if( p->rc==SQLITE_OK ){ + if( zSql ){ + p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB, + ppStmt, 0); + }else{ + p->rc = SQLITE_NOMEM; + } + } + sqlite3_free(zSql); + return p->rc; +} + + +/* +** INSERT OR REPLACE a record into the %_data table. +*/ +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + pConfig->zDb, pConfig->zName + )); + if( p->rc ) return; + } + + sqlite3_bind_int64(p->pWriter, 1, iRowid); + sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); + sqlite3_step(p->pWriter); + p->rc = sqlite3_reset(p->pWriter); + sqlite3_bind_null(p->pWriter, 2); +} + +/* +** Execute the following SQL: +** +** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast +*/ +static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + char *zSql = sqlite3_mprintf( + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + pConfig->zDb, pConfig->zName + ); + if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return; + } + + sqlite3_bind_int64(p->pDeleter, 1, iFirst); + sqlite3_bind_int64(p->pDeleter, 2, iLast); + sqlite3_step(p->pDeleter); + p->rc = sqlite3_reset(p->pDeleter); +} + +/* +** Remove all records associated with segment iSegid. +*/ +static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; + fts5DataDelete(p, iFirst, iLast); + if( p->pIdxDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE segid=?", + pConfig->zDb, pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); + sqlite3_step(p->pIdxDeleter); + p->rc = sqlite3_reset(p->pIdxDeleter); + } +} + +/* +** Release a reference to an Fts5Structure object returned by an earlier +** call to fts5StructureRead() or fts5StructureDecode(). +*/ +static void fts5StructureRelease(Fts5Structure *pStruct){ + if( pStruct && 0>=(--pStruct->nRef) ){ + int i; + assert( pStruct->nRef==0 ); + for(i=0; inLevel; i++){ + sqlite3_free(pStruct->aLevel[i].aSeg); + } + sqlite3_free(pStruct); + } +} + +static void fts5StructureRef(Fts5Structure *pStruct){ + pStruct->nRef++; +} + +/* +** Deserialize and return the structure record currently stored in serialized +** form within buffer pData/nData. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated by one slot. This allows the structure contents +** to be more easily edited. +** +** If an error occurs, *ppOut is set to NULL and an SQLite error code +** returned. Otherwise, *ppOut is set to point to the new object and +** SQLITE_OK returned. +*/ +static int fts5StructureDecode( + const u8 *pData, /* Buffer containing serialized structure */ + int nData, /* Size of buffer pData in bytes */ + int *piCookie, /* Configuration cookie value */ + Fts5Structure **ppOut /* OUT: Deserialized object */ +){ + int rc = SQLITE_OK; + int i = 0; + int iLvl; + int nLevel = 0; + int nSegment = 0; + sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ + Fts5Structure *pRet = 0; /* Structure object to return */ + + /* Grab the cookie value */ + if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); + i = 4; + + /* Read the total number of levels and segments from the start of the + ** structure record. */ + i += fts5GetVarint32(&pData[i], nLevel); + i += fts5GetVarint32(&pData[i], nSegment); + if( nLevel>FTS5_MAX_SEGMENT || nLevel<0 + || nSegment>FTS5_MAX_SEGMENT || nSegment<0 + ){ + return FTS5_CORRUPT; + } + nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ + ); + pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); + + if( pRet ){ + pRet->nRef = 1; + pRet->nLevel = nLevel; + pRet->nSegment = nSegment; + i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); + + for(iLvl=0; rc==SQLITE_OK && iLvlaLevel[iLvl]; + int nTotal = 0; + int iSeg; + + if( i>=nData ){ + rc = FTS5_CORRUPT; + }else{ + i += fts5GetVarint32(&pData[i], pLvl->nMerge); + i += fts5GetVarint32(&pData[i], nTotal); + if( nTotalnMerge ) rc = FTS5_CORRUPT; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, + nTotal * sizeof(Fts5StructureSegment) + ); + nSegment -= nTotal; + } + + if( rc==SQLITE_OK ){ + pLvl->nSeg = nTotal; + for(iSeg=0; iSegaSeg[iSeg]; + if( i>=nData ){ + rc = FTS5_CORRUPT; + break; + } + i += fts5GetVarint32(&pData[i], pSeg->iSegid); + i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); + i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( pSeg->pgnoLastpgnoFirst ){ + rc = FTS5_CORRUPT; + break; + } + } + if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT; + if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT; + } + } + if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + + if( rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + *ppOut = pRet; + return rc; +} + +/* +** +*/ +static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ + if( *pRc==SQLITE_OK ){ + Fts5Structure *pStruct = *ppStruct; + int nLevel = pStruct->nLevel; + sqlite3_int64 nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ + ); + + pStruct = sqlite3_realloc64(pStruct, nByte); + if( pStruct ){ + memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); + pStruct->nLevel++; + *ppStruct = pStruct; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Extend level iLvl so that there is room for at least nExtra more +** segments. +*/ +static void fts5StructureExtendLevel( + int *pRc, + Fts5Structure *pStruct, + int iLvl, + int nExtra, + int bInsert +){ + if( *pRc==SQLITE_OK ){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureSegment *aNew; + sqlite3_int64 nByte; + + nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); + aNew = sqlite3_realloc64(pLvl->aSeg, nByte); + if( aNew ){ + if( bInsert==0 ){ + memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); + }else{ + int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment); + memmove(&aNew[nExtra], aNew, nMove); + memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra); + } + pLvl->aSeg = aNew; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ + Fts5Structure *pRet = 0; + Fts5Config *pConfig = p->pConfig; + int iCookie; /* Configuration cookie */ + Fts5Data *pData; + + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc==SQLITE_OK ){ + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); + if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + fts5DataRelease(pData); + if( p->rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + return pRet; +} + +static i64 fts5IndexDataVersion(Fts5Index *p){ + i64 iVersion = 0; + + if( p->rc==SQLITE_OK ){ + if( p->pDataVersion==0 ){ + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) + ); + if( p->rc ) return 0; + } + + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ + iVersion = sqlite3_column_int64(p->pDataVersion, 0); + } + p->rc = sqlite3_reset(p->pDataVersion); + } + + return iVersion; +} + +/* +** Read, deserialize and return the structure record. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated as described for function fts5StructureDecode() +** above. +** +** If an error occurs, NULL is returned and an error code left in the +** Fts5Index handle. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static Fts5Structure *fts5StructureRead(Fts5Index *p){ + + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } + } + +#if 0 + else{ + Fts5Structure *pTest = fts5StructureReadUncached(p); + if( pTest ){ + int i, j; + assert_nc( p->pStruct->nSegment==pTest->nSegment ); + assert_nc( p->pStruct->nLevel==pTest->nLevel ); + for(i=0; inLevel; i++){ + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); + for(j=0; jaLevel[i].nSeg; j++){ + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; + assert_nc( p1->iSegid==p2->iSegid ); + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); + assert_nc( p1->pgnoLast==p2->pgnoLast ); + } + } + fts5StructureRelease(pTest); + } + } +#endif + + if( p->rc!=SQLITE_OK ) return 0; + assert( p->iStructVersion!=0 ); + assert( p->pStruct!=0 ); + fts5StructureRef(p->pStruct); + return p->pStruct; +} + +static void fts5StructureInvalidate(Fts5Index *p){ + if( p->pStruct ){ + fts5StructureRelease(p->pStruct); + p->pStruct = 0; + } +} + +/* +** Return the total number of segments in index structure pStruct. This +** function is only ever used as part of assert() conditions. +*/ +#ifdef SQLITE_DEBUG +static int fts5StructureCountSegments(Fts5Structure *pStruct){ + int nSegment = 0; /* Total number of segments */ + if( pStruct ){ + int iLvl; /* Used to iterate through levels */ + for(iLvl=0; iLvlnLevel; iLvl++){ + nSegment += pStruct->aLevel[iLvl].nSeg; + } + } + + return nSegment; +} +#endif + +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ +} + + +/* +** Serialize and store the "structure" record. +** +** If an error occurs, leave an error code in the Fts5Index object. If an +** error has already occurred, this function is a no-op. +*/ +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ + if( p->rc==SQLITE_OK ){ + Fts5Buffer buf; /* Buffer to serialize record into */ + int iLvl; /* Used to iterate through levels */ + int iCookie; /* Cookie value to store */ + + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + memset(&buf, 0, sizeof(Fts5Buffer)); + + /* Append the current configuration cookie */ + iCookie = p->pConfig->iCookie; + if( iCookie<0 ) iCookie = 0; + + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + sqlite3Fts5Put32(buf.p, iCookie); + buf.n = 4; + fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); + fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); + fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); + } + + for(iLvl=0; iLvlnLevel; iLvl++){ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg); + assert( pLvl->nMerge<=pLvl->nSeg ); + + for(iSeg=0; iSegnSeg; iSeg++){ + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + } + } + + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); + fts5BufferFree(&buf); + } +} + +#if 0 +static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*); +static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ + int rc = SQLITE_OK; + Fts5Buffer buf; + memset(&buf, 0, sizeof(buf)); + fts5DebugStructure(&rc, &buf, pStruct); + fprintf(stdout, "%s: %s\n", zCaption, buf.p); + fflush(stdout); + fts5BufferFree(&buf); +} +#else +# define fts5PrintStructure(x,y) +#endif + +static int fts5SegmentSize(Fts5StructureSegment *pSeg){ + return 1 + pSeg->pgnoLast - pSeg->pgnoFirst; +} + +/* +** Return a copy of index structure pStruct. Except, promote as many +** segments as possible to level iPromote. If an OOM occurs, NULL is +** returned. +*/ +static void fts5StructurePromoteTo( + Fts5Index *p, + int iPromote, + int szPromote, + Fts5Structure *pStruct +){ + int il, is; + Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote]; + + if( pOut->nMerge==0 ){ + for(il=iPromote+1; ilnLevel; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + if( pLvl->nMerge ) return; + for(is=pLvl->nSeg-1; is>=0; is--){ + int sz = fts5SegmentSize(&pLvl->aSeg[is]); + if( sz>szPromote ) return; + fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1); + if( p->rc ) return; + memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment)); + pOut->nSeg++; + pLvl->nSeg--; + } + } + } +} + +/* +** A new segment has just been written to level iLvl of index structure +** pStruct. This function determines if any segments should be promoted +** as a result. Segments are promoted in two scenarios: +** +** a) If the segment just written is smaller than one or more segments +** within the previous populated level, it is promoted to the previous +** populated level. +** +** b) If the segment just written is larger than the newest segment on +** the next populated level, then that segment, and any other adjacent +** segments that are also smaller than the one just written, are +** promoted. +** +** If one or more segments are promoted, the structure object is updated +** to reflect this. +*/ +static void fts5StructurePromote( + Fts5Index *p, /* FTS5 backend object */ + int iLvl, /* Index level just updated */ + Fts5Structure *pStruct /* Index structure */ +){ + if( p->rc==SQLITE_OK ){ + int iTst; + int iPromote = -1; + int szPromote = 0; /* Promote anything this size or smaller */ + Fts5StructureSegment *pSeg; /* Segment just written */ + int szSeg; /* Size of segment just written */ + int nSeg = pStruct->aLevel[iLvl].nSeg; + + if( nSeg==0 ) return; + pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; + szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); + + /* Check for condition (a) */ + for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); + if( iTst>=0 ){ + int i; + int szMax = 0; + Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; + assert( pTst->nMerge==0 ); + for(i=0; inSeg; i++){ + int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; + if( sz>szMax ) szMax = sz; + } + if( szMax>=szSeg ){ + /* Condition (a) is true. Promote the newest segment on level + ** iLvl to level iTst. */ + iPromote = iTst; + szPromote = szMax; + } + } + + /* If condition (a) is not met, assume (b) is true. StructurePromoteTo() + ** is a no-op if it is not. */ + if( iPromote<0 ){ + iPromote = iLvl; + szPromote = szSeg; + } + fts5StructurePromoteTo(p, iPromote, szPromote, pStruct); + } +} + + +/* +** Advance the iterator passed as the only argument. If the end of the +** doclist-index page is reached, return non-zero. +*/ +static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ + Fts5Data *pData = pLvl->pData; + + if( pLvl->iOff==0 ){ + assert( pLvl->bEof==0 ); + pLvl->iOff = 1; + pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno); + pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid); + pLvl->iFirstOff = pLvl->iOff; + }else{ + int iOff; + for(iOff=pLvl->iOff; iOffnn; iOff++){ + if( pData->p[iOff] ) break; + } + + if( iOffnn ){ + i64 iVal; + pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; + iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + pLvl->iRowid += iVal; + pLvl->iOff = iOff; + }else{ + pLvl->bEof = 1; + } + } + + return pLvl->bEof; +} + +/* +** Advance the iterator passed as the only argument. +*/ +static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvlnLvl ); + if( fts5DlidxLvlNext(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterNextR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ) fts5DlidxLvlNext(pLvl); + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterNextR(p, pIter, 0); +} + +/* +** The iterator passed as the first argument has the following fields set +** as follows. This function sets up the rest of the iterator so that it +** points to the first rowid in the doclist-index. +** +** pData: +** pointer to doclist-index record, +** +** When this function is called pIter->iLeafPgno is the page number the +** doclist is associated with (the one featuring the term). +*/ +static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){ + int i; + for(i=0; inLvl; i++){ + fts5DlidxLvlNext(&pIter->aLvl[i]); + } + return pIter->aLvl[0].bEof; +} + + +static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ + return p->rc!=SQLITE_OK || pIter->aLvl[0].bEof; +} + +static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){ + int i; + + /* Advance each level to the last entry on the last page */ + for(i=pIter->nLvl-1; p->rc==SQLITE_OK && i>=0; i--){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[i]; + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + + if( i>0 ){ + Fts5DlidxLvl *pChild = &pLvl[-1]; + fts5DataRelease(pChild->pData); + memset(pChild, 0, sizeof(Fts5DlidxLvl)); + pChild->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno) + ); + } + } +} + +/* +** Move the iterator passed as the only argument to the previous entry. +*/ +static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ + int iOff = pLvl->iOff; + + assert( pLvl->bEof==0 ); + if( iOff<=pLvl->iFirstOff ){ + pLvl->bEof = 1; + }else{ + u8 *a = pLvl->pData->p; + i64 iVal; + int iLimit; + int ii; + int nZero = 0; + + /* Currently iOff points to the first byte of a varint. This block + ** decrements iOff until it points to the first byte of the previous + ** varint. Taking care not to read any memory locations that occur + ** before the buffer in memory. */ + iLimit = (iOff>9 ? iOff-9 : 0); + for(iOff--; iOff>iLimit; iOff--){ + if( (a[iOff-1] & 0x80)==0 ) break; + } + + fts5GetVarint(&a[iOff], (u64*)&iVal); + pLvl->iRowid -= iVal; + pLvl->iLeafPgno--; + + /* Skip backwards past any 0x00 varints. */ + for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ + nZero++; + } + if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ + /* The byte immediately before the last 0x00 byte has the 0x80 bit + ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 + ** bytes before a[ii]. */ + int bZero = 0; /* True if last 0x00 counts */ + if( (ii-8)>=pLvl->iFirstOff ){ + int j; + for(j=1; j<=8 && (a[ii-j] & 0x80); j++); + bZero = (j>8); + } + if( bZero==0 ) nZero--; + } + pLvl->iLeafPgno -= nZero; + pLvl->iOff = iOff - nZero; + } + + return pLvl->bEof; +} + +static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvlnLvl ); + if( fts5DlidxLvlPrev(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterPrevR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ){ + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + } + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterPrevR(p, pIter, 0); +} + +/* +** Free a doclist-index iterator object allocated by fts5DlidxIterInit(). +*/ +static void fts5DlidxIterFree(Fts5DlidxIter *pIter){ + if( pIter ){ + int i; + for(i=0; inLvl; i++){ + fts5DataRelease(pIter->aLvl[i].pData); + } + sqlite3_free(pIter); + } +} + +static Fts5DlidxIter *fts5DlidxIterInit( + Fts5Index *p, /* Fts5 Backend to iterate within */ + int bRev, /* True for ORDER BY ASC */ + int iSegid, /* Segment id */ + int iLeafPg /* Leaf page number to load dlidx for */ +){ + Fts5DlidxIter *pIter = 0; + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + Fts5DlidxIter *pNew; + + pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg); + Fts5DlidxLvl *pLvl = &pNew->aLvl[i]; + pIter = pNew; + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, iRowid); + if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){ + bDone = 1; + } + pIter->nLvl = i+1; + } + } + + if( p->rc==SQLITE_OK ){ + pIter->iSegid = iSegid; + if( bRev==0 ){ + fts5DlidxIterFirst(pIter); + }else{ + fts5DlidxIterLast(p, pIter); + } + } + + if( p->rc!=SQLITE_OK ){ + fts5DlidxIterFree(pIter); + pIter = 0; + } + + return pIter; +} + +static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iRowid; +} +static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iLeafPgno; +} + +/* +** Load the next leaf page into the segment iterator. +*/ +static void fts5SegIterNextPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter /* Iterator to advance to next page */ +){ + Fts5Data *pLeaf; + Fts5StructureSegment *pSeg = pIter->pSeg; + fts5DataRelease(pIter->pLeaf); + pIter->iLeafPgno++; + if( pIter->pNextLeaf ){ + pIter->pLeaf = pIter->pNextLeaf; + pIter->pNextLeaf = 0; + }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ + pIter->pLeaf = fts5LeafRead(p, + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) + ); + }else{ + pIter->pLeaf = 0; + } + pLeaf = pIter->pLeaf; + + if( pLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf; + if( fts5LeafIsTermless(pLeaf) ){ + pIter->iEndofDoclist = pLeaf->nn+1; + }else{ + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], + pIter->iEndofDoclist + ); + } + } +} + +/* +** Argument p points to a buffer containing a varint to be interpreted as a +** position list size field. Read the varint and return the number of bytes +** read. Before returning, set *pnSz to the number of bytes in the position +** list, and *pbDel to true if the delete flag is set, or false otherwise. +*/ +static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ + int nSz; + int n = 0; + fts5FastGetVarint32(p, n, nSz); + assert_nc( nSz>=0 ); + *pnSz = nSz/2; + *pbDel = nSz & 0x0001; + return n; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of a +** position-list size field. Read the value of the field and store it +** in the following variables: +** +** Fts5SegIter.nPos +** Fts5SegIter.bDel +** +** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the +** position list content (if any). +*/ +static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ + if( p->rc==SQLITE_OK ){ + int iOff = pIter->iLeafOffset; /* Offset to read at */ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); + pIter->bDel = 0; + pIter->nPos = 1; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->bDel = 1; + iOff++; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->nPos = 1; + iOff++; + }else{ + pIter->nPos = 0; + } + } + }else{ + int nSz; + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + pIter->iLeafOffset = iOff; + } +} + +static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + int iOff = pIter->iLeafOffset; + + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + return; + } + iOff = 4; + a = pIter->pLeaf->p; + } + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of the +** "nSuffix" field of a term. Function parameter nKeep contains the value +** of the "nPrefix" field (if there was one - it is passed 0 if this is +** the first term in the segment). +** +** This function populates: +** +** Fts5SegIter.term +** Fts5SegIter.rowid +** +** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of +** the first position list. The position list belonging to document +** (Fts5SegIter.iRowid). +*/ +static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + int iOff = pIter->iLeafOffset; /* Offset to read at */ + int nNew; /* Bytes of new data */ + + iOff += fts5GetVarint32(&a[iOff], nNew); + if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ + p->rc = FTS5_CORRUPT; + return; + } + pIter->term.n = nKeep; + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + assert( pIter->term.n<=pIter->term.nSpace ); + iOff += nNew; + pIter->iTermLeafOffset = iOff; + pIter->iTermLeafPgno = pIter->iLeafPgno; + pIter->iLeafOffset = iOff; + + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); + pIter->iEndofDoclist += nExtra; + } + + fts5SegIterLoadRowid(p, pIter); +} + +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); + +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + pIter->xNext = fts5SegIterNext_Reverse; + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xNext = fts5SegIterNext_None; + }else{ + pIter->xNext = fts5SegIterNext; + } +} + +/* +** Initialize the iterator object pIter to iterate through the entries in +** segment pSeg. The iterator is left pointing to the first entry when +** this function returns. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterInit( + Fts5Index *p, /* FTS index object */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + if( pSeg->pgnoFirst==0 ){ + /* This happens if the segment is being used as an input to an incremental + ** merge and all data has already been "trimmed". See function + ** fts5TrimSegments() for details. In this case leave the iterator empty. + ** The caller will see the (pIter->pLeaf==0) and assume the iterator is + ** at EOF already. */ + assert( pIter->pLeaf==0 ); + return; + } + + if( p->rc==SQLITE_OK ){ + memset(pIter, 0, sizeof(*pIter)); + fts5SegIterSetNext(p, pIter); + pIter->pSeg = pSeg; + pIter->iLeafPgno = pSeg->pgnoFirst-1; + fts5SegIterNextPage(p, pIter); + } + + if( p->rc==SQLITE_OK ){ + pIter->iLeafOffset = 4; + assert_nc( pIter->pLeaf->nn>4 ); + assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + } +} + +/* +** This function is only ever called on iterators created by calls to +** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set. +** +** The iterator is in an unusual state when this function is called: the +** Fts5SegIter.iLeafOffset variable is set to the offset of the start of +** the position-list size field for the first relevant rowid on the page. +** Fts5SegIter.rowid is set, but nPos and bDel are not. +** +** This function advances the iterator so that it points to the last +** relevant rowid on the page and, if necessary, initializes the +** aRowidOffset[] and iRowidOffset variables. At this point the iterator +** is in its regular state - Fts5SegIter.iLeafOffset points to the first +** byte of the position list content associated with said rowid. +*/ +static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; + int n = pIter->pLeaf->szLeaf; + int i = pIter->iLeafOffset; + u8 *a = pIter->pLeaf->p; + int iRowidOffset = 0; + + if( n>pIter->iEndofDoclist ){ + n = pIter->iEndofDoclist; + } + + ASSERT_SZLEAF_OK(pIter->pLeaf); + while( 1 ){ + u64 iDelta = 0; + + if( eDetail==FTS5_DETAIL_NONE ){ + /* todo */ + if( i=n ) break; + i += fts5GetVarint(&a[i], &iDelta); + pIter->iRowid += iDelta; + + /* If necessary, grow the pIter->aRowidOffset[] array. */ + if( iRowidOffset>=pIter->nRowidOffset ){ + int nNew = pIter->nRowidOffset + 8; + int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + pIter->aRowidOffset = aNew; + pIter->nRowidOffset = nNew; + } + + pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset; + pIter->iLeafOffset = i; + } + pIter->iRowidOffset = iRowidOffset; + fts5SegIterLoadNPos(p, pIter); +} + +/* +** +*/ +static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ + Fts5Data *pNew; + pIter->iLeafPgno--; + pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( + pIter->pSeg->iSegid, pIter->iLeafPgno + )); + if( pNew ){ + /* iTermLeafOffset may be equal to szLeaf if the term is the last + ** thing on the page - i.e. the first rowid is on the following page. + ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ + assert( pIter->pLeaf==0 ); + if( pIter->iTermLeafOffsetszLeaf ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; + } + }else{ + int iRowidOff; + iRowidOff = fts5LeafFirstRowidOff(pNew); + if( iRowidOff ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = iRowidOff; + } + } + + if( pIter->pLeaf ){ + u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; + pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid); + break; + }else{ + fts5DataRelease(pNew); + } + } + } + + if( pIter->pLeaf ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + fts5SegIterReverseInitPage(p, pIter); + } +} + +/* +** Return true if the iterator passed as the second argument currently +** points to a delete marker. A delete marker is an entry with a 0 byte +** position-list. +*/ +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used by reverse iterators. +*/ +static void fts5SegIterNext_Reverse( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbUnused /* Unused */ +){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->pNextLeaf==0 ); + UNUSED_PARAM(pbUnused); + + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + u64 iDelta; + + pIter->iRowidOffset--; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += pIter->nPos; + } + fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid -= iDelta; + }else{ + fts5SegIterReverseNewPage(p, pIter); + } +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used if detail=none and the +** iterator is not a reverse direction iterator. +*/ +static void fts5SegIterNext_None( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + int iOff; + + assert( p->rc==SQLITE_OK ); + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); + + ASSERT_SZLEAF_OK(pIter->pLeaf); + iOff = pIter->iLeafOffset; + + /* Next entry is on the next page */ + if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( p->rc || pIter->pLeaf==0 ) return; + pIter->iRowid = 0; + iOff = 4; + } + + if( iOffiEndofDoclist ){ + /* Next entry is on the current page */ + i64 iDelta; + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); + pIter->iLeafOffset = iOff; + pIter->iRowid += iDelta; + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ + if( pIter->pSeg ){ + int nKeep = 0; + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); + } + pIter->iLeafOffset = iOff; + fts5SegIterLoadTerm(p, pIter, nKeep); + }else{ + const u8 *pList = 0; + const char *zTerm = 0; + int nList; + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + if( pList==0 ) goto next_none_eof; + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList; + sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + } + + if( pbNewTerm ) *pbNewTerm = 1; + }else{ + goto next_none_eof; + } + + fts5SegIterLoadNPos(p, pIter); + + return; + next_none_eof: + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + + +/* +** Advance iterator pIter to the next entry. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. It +** is not considered an error if the iterator reaches EOF. If an error has +** already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterNext( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; + u8 *a; + int n; + + assert( pbNewTerm==0 || *pbNewTerm==0 ); + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + /* Search for the end of the position list within the current page. */ + a = pLeaf->p; + n = pLeaf->szLeaf; + + ASSERT_SZLEAF_OK(pLeaf); + iOff = pIter->iLeafOffset + pIter->nPos; + + if( iOffiEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ + bNewTerm = 1; + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); + } + }else{ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); + } + pIter->iLeafOffset = iOff; + + }else if( pIter->pSeg==0 ){ + const u8 *pList = 0; + const char *zTerm = 0; + int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), + (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; + } + }else{ + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; + ASSERT_SZLEAF_OK(pLeaf); + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + ); + } + } + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff + ); + pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; + bNewTerm = 1; + } + assert_nc( iOffszLeaf ); + if( iOff>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + } + } + + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf ){ + if( bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); + fts5SegIterLoadNPos(p, pIter); + if( pbNewTerm ) *pbNewTerm = 1; + } + }else{ + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. */ + int nSz; + assert( p->rc==SQLITE_OK ); + assert_nc( pIter->iLeafOffset<=pIter->pLeaf->nn ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + } +} + +#define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } + +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOffpDlidx; + Fts5Data *pLast = 0; + int pgnoLast = 0; + + if( pDlidx ){ + int iSegid = pIter->pSeg->iSegid; + pgnoLast = fts5DlidxIterPgno(pDlidx); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); + }else{ + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of + ** position-list content for the current rowid. Back it up so that it + ** points to the start of the position-list size field. */ + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + pIter->iLeafOffset = iPoslist; + + /* If this condition is true then the largest rowid for the current + ** term may not be stored on the current page. So search forward to + ** see where said rowid really is. */ + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ + int pgno; + Fts5StructureSegment *pSeg = pIter->pSeg; + + /* The last rowid in the doclist may not be on the current page. Search + ** forward to find the page containing the last rowid. */ + for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + Fts5Data *pNew = fts5DataRead(p, iAbs); + if( pNew ){ + int iRowid, bTermless; + iRowid = fts5LeafFirstRowidOff(pNew); + bTermless = fts5LeafIsTermless(pNew); + if( iRowid ){ + SWAPVAL(Fts5Data*, pNew, pLast); + pgnoLast = pgno; + } + fts5DataRelease(pNew); + if( bTermless==0 ) break; + } + } + } + } + + /* If pLast is NULL at this point, then the last rowid for this doclist + ** lies on the page currently indicated by the iterator. In this case + ** pIter->iLeafOffset is already set to point to the position-list size + ** field associated with the first relevant rowid on the page. + ** + ** Or, if pLast is non-NULL, then it is the page that contains the last + ** rowid. In this case configure the iterator so that it points to the + ** first rowid on this page. + */ + if( pLast ){ + int iOff; + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = pLast; + pIter->iLeafPgno = pgnoLast; + iOff = fts5LeafFirstRowidOff(pLast); + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } + + } + + fts5SegIterReverseInitPage(p, pIter); +} + +/* +** Iterator pIter currently points to the first rowid of a doclist. +** There is a doclist-index associated with the final term on the current +** page. If the current term is the last term on the page, load the +** doclist-index from disk and initialize an iterator at (pIter->pDlidx). +*/ +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ + int iSeg = pIter->pSeg->iSegid; + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx==0 ); + + /* Check if the current doclist ends on this page. If it does, return + ** early without loading the doclist-index (as it belongs to a different + ** term. */ + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclistszLeaf + ){ + return; + } + + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); +} + +/* +** The iterator object passed as the second argument currently contains +** no valid values except for the Fts5SegIter.pLeaf member variable. This +** function searches the leaf page for a term matching (pTerm/nTerm). +** +** If the specified term is found on the page, then the iterator is left +** pointing to it. If argument bGe is zero and the term is not found, +** the iterator is left pointing at EOF. +** +** If bGe is non-zero and the specified term is not found, then the +** iterator is left pointing to the smallest term in the segment that +** is larger than the specified term, even if this term is not on the +** current page. +*/ +static void fts5LeafSeek( + Fts5Index *p, /* Leave any error code here */ + int bGe, /* True for a >= search */ + Fts5SegIter *pIter, /* Iterator to seek */ + const u8 *pTerm, int nTerm /* Term to search for */ +){ + int iOff; + const u8 *a = pIter->pLeaf->p; + int szLeaf = pIter->pLeaf->szLeaf; + int n = pIter->pLeaf->nn; + + u32 nMatch = 0; + u32 nKeep = 0; + u32 nNew = 0; + u32 iTermOff; + int iPgidx; /* Current offset in pgidx */ + int bEndOfPage = 0; + + assert( p->rc==SQLITE_OK ); + + iPgidx = szLeaf; + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); + iOff = iTermOff; + if( iOff>n ){ + p->rc = FTS5_CORRUPT; + return; + } + + while( 1 ){ + + /* Figure out how many new bytes are in this term */ + fts5FastGetVarint32(a, iOff, nNew); + if( nKeep=nMatch ); + if( nKeep==nMatch ){ + u32 nCmp; + u32 i; + nCmp = (u32)MIN(nNew, nTerm-nMatch); + for(i=0; ipTerm[nMatch] ){ + goto search_failed; + } + } + + if( iPgidx>=n ){ + bEndOfPage = 1; + break; + } + + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); + iTermOff += nKeep; + iOff = iTermOff; + + if( iOff>=n ){ + p->rc = FTS5_CORRUPT; + return; + } + + /* Read the nKeep field of the next term. */ + fts5FastGetVarint32(a, iOff, nKeep); + } + + search_failed: + if( bGe==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + return; + }else if( bEndOfPage ){ + do { + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) return; + a = pIter->pLeaf->p; + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ + iPgidx = pIter->pLeaf->szLeaf; + iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); + if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + }else{ + nKeep = 0; + iTermOff = iOff; + n = pIter->pLeaf->nn; + iOff += fts5GetVarint32(&a[iOff], nNew); + break; + } + } + }while( 1 ); + } + + search_success: + if( (i64)iOff+nNew>n || nNew<1 ){ + p->rc = FTS5_CORRUPT; + return; + } + pIter->iLeafOffset = iOff + nNew; + pIter->iTermLeafOffset = pIter->iLeafOffset; + pIter->iTermLeafPgno = pIter->iLeafPgno; + + fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + + if( iPgidx>=n ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); + pIter->iEndofDoclist = iTermOff + nExtra; + } + pIter->iPgidxOff = iPgidx; + + fts5SegIterLoadRowid(p, pIter); + fts5SegIterLoadNPos(p, pIter); +} + +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + } + return p->pIdxSelect; +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within segment +** pSeg. If there is no such term in the index, the iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterSeekInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = 1; + int bGe = (flags & FTS5INDEX_QUERY_SCAN); + int bDlidx = 0; /* True if there is a doclist-index */ + sqlite3_stmt *pIdxSelect = 0; + + assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); + assert( pTerm && nTerm ); + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + + /* This block sets stack variable iPg to the leaf page number that may + ** contain term (pTerm/nTerm), if it is present in the segment. */ + pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc ) return; + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ + i64 val = sqlite3_column_int(pIdxSelect, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); + + if( iPgpgnoFirst ){ + iPg = pSeg->pgnoFirst; + bDlidx = 0; + } + + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + + if( pIter->pLeaf ){ + fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); + } + + if( p->rc==SQLITE_OK && bGe==0 ){ + pIter->flags |= FTS5_SEGITER_ONETERM; + if( pIter->pLeaf ){ + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + } + if( bDlidx ){ + fts5SegIterLoadDlidx(p, pIter); + } + if( flags & FTS5INDEX_QUERY_DESC ){ + fts5SegIterReverse(p, pIter); + } + } + } + + fts5SegIterSetNext(p, pIter); + + /* Either: + ** + ** 1) an error has occurred, or + ** 2) the iterator points to EOF, or + ** 3) the iterator points to an entry with term (pTerm/nTerm), or + ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points + ** to an entry with a term greater than or equal to (pTerm/nTerm). + */ + assert_nc( p->rc!=SQLITE_OK /* 1 */ + || pIter->pLeaf==0 /* 2 */ + || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */ + || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */ + ); +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within the +** in-memory hash table. If there is no such term in the hash-table, the +** iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterHashInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5SegIter *pIter /* Object to populate */ +){ + int nList = 0; + const u8 *z = 0; + int n = 0; + Fts5Data *pLeaf = 0; + + assert( p->pHash ); + assert( p->rc==SQLITE_OK ); + + if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ + const u8 *pList = 0; + + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); + n = (z ? (int)strlen((const char*)z) : 0); + if( pList ){ + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf ){ + pLeaf->p = (u8*)pList; + } + } + }else{ + p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), + (const char*)pTerm, nTerm, (void**)&pLeaf, &nList + ); + if( pLeaf ){ + pLeaf->p = (u8*)&pLeaf[1]; + } + z = pTerm; + n = nTerm; + pIter->flags |= FTS5_SEGITER_ONETERM; + } + + if( pLeaf ){ + sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); + pLeaf->nn = pLeaf->szLeaf = nList; + pIter->pLeaf = pLeaf; + pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pLeaf->nn; + + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + } + + fts5SegIterSetNext(p, pIter); +} + +/* +** Zero the iterator passed as the only argument. +*/ +static void fts5SegIterClear(Fts5SegIter *pIter){ + fts5BufferFree(&pIter->term); + fts5DataRelease(pIter->pLeaf); + fts5DataRelease(pIter->pNextLeaf); + fts5DlidxIterFree(pIter->pDlidx); + sqlite3_free(pIter->aRowidOffset); + memset(pIter, 0, sizeof(Fts5SegIter)); +} + +#ifdef SQLITE_DEBUG + +/* +** This function is used as part of the big assert() procedure implemented by +** fts5AssertMultiIterSetup(). It ensures that the result currently stored +** in *pRes is the correct result of comparing the current positions of the +** two iterators. +*/ +static void fts5AssertComparisonResult( + Fts5Iter *pIter, + Fts5SegIter *p1, + Fts5SegIter *p2, + Fts5CResult *pRes +){ + int i1 = p1 - pIter->aSeg; + int i2 = p2 - pIter->aSeg; + + if( p1->pLeaf || p2->pLeaf ){ + if( p1->pLeaf==0 ){ + assert( pRes->iFirst==i2 ); + }else if( p2->pLeaf==0 ){ + assert( pRes->iFirst==i1 ); + }else{ + int nMin = MIN(p1->term.n, p2->term.n); + int res = fts5Memcmp(p1->term.p, p2->term.p, nMin); + if( res==0 ) res = p1->term.n - p2->term.n; + + if( res==0 ){ + assert( pRes->bTermEq==1 ); + assert( p1->iRowid!=p2->iRowid ); + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1; + }else{ + assert( pRes->bTermEq==0 ); + } + + if( res<0 ){ + assert( pRes->iFirst==i1 ); + }else{ + assert( pRes->iFirst==i2 ); + } + } + } +} + +/* +** This function is a no-op unless SQLITE_DEBUG is defined when this module +** is compiled. In that case, this function is essentially an assert() +** statement used to verify that the contents of the pIter->aFirst[] array +** are correct. +*/ +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int i; + + assert( (pFirst->pLeaf==0)==pIter->base.bEof ); + + /* Check that pIter->iSwitchRowid is set correctly. */ + for(i=0; inSeg; i++){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + assert( p1==pFirst + || p1->pLeaf==0 + || fts5BufferCompare(&pFirst->term, &p1->term) + || p1->iRowid==pIter->iSwitchRowid + || (p1->iRowidiSwitchRowid)==pIter->bRev + ); + } + + for(i=0; inSeg; i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + Fts5SegIter *p2 = &pIter->aSeg[i+1]; + Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + + for(i=1; i<(pIter->nSeg / 2); i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ]; + Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ]; + Fts5CResult *pRes = &pIter->aFirst[i]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + } +} +#else +# define fts5AssertMultiIterSetup(x,y) +#endif + +/* +** Do the comparison necessary to populate pIter->aFirst[iOut]. +** +** If the returned value is non-zero, then it is the index of an entry +** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing +** to a key that is a duplicate of another, higher priority, +** segment-iterator in the pSeg->aSeg[] array. +*/ +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ + int i1; /* Index of left-hand Fts5SegIter */ + int i2; /* Index of right-hand Fts5SegIter */ + int iRes; + Fts5SegIter *p1; /* Left-hand Fts5SegIter */ + Fts5SegIter *p2; /* Right-hand Fts5SegIter */ + Fts5CResult *pRes = &pIter->aFirst[iOut]; + + assert( iOutnSeg && iOut>0 ); + assert( pIter->bRev==0 || pIter->bRev==1 ); + + if( iOut>=(pIter->nSeg/2) ){ + i1 = (iOut - pIter->nSeg/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pIter->aFirst[iOut*2].iFirst; + i2 = pIter->aFirst[iOut*2+1].iFirst; + } + p1 = &pIter->aSeg[i1]; + p2 = &pIter->aSeg[i2]; + + pRes->bTermEq = 0; + if( p1->pLeaf==0 ){ /* If p1 is at EOF */ + iRes = i2; + }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */ + iRes = i1; + }else{ + int res = fts5BufferCompare(&p1->term, &p2->term); + if( res==0 ){ + assert_nc( i2>i1 ); + assert_nc( i2!=0 ); + pRes->bTermEq = 1; + if( p1->iRowid==p2->iRowid ){ + p1->bDel = p2->bDel; + return i2; + } + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; + } + assert( res!=0 ); + if( res<0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pRes->iFirst = (u16)iRes; + return 0; +} + +/* +** Move the seg-iter so that it points to the first rowid on page iLeafPgno. +** It is an error if leaf iLeafPgno does not exist or contains no rowids. +*/ +static void fts5SegIterGotoPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int iLeafPgno +){ + assert( iLeafPgno>pIter->iLeafPgno ); + + if( iLeafPgno>pIter->pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5DataRelease(pIter->pNextLeaf); + pIter->pNextLeaf = 0; + pIter->iLeafPgno = iLeafPgno-1; + fts5SegIterNextPage(p, pIter); + assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); + + if( p->rc==SQLITE_OK ){ + int iOff; + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + + iOff = fts5LeafFirstRowidOff(pIter->pLeaf); + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + } + } +} + +/* +** Advance the iterator passed as the second argument until it is at or +** past rowid iFrom. Regardless of the value of iFrom, the iterator is +** always advanced at least once. +*/ +static void fts5SegIterNextFrom( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + i64 iMatch /* Advance iterator at least this far */ +){ + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5DlidxIter *pDlidx = pIter->pDlidx; + int iLeafPgno = pIter->iLeafPgno; + int bMove = 1; + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx ); + assert( pIter->pLeaf ); + + if( bRev==0 ){ + while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){ + iLeafPgno = fts5DlidxIterPgno(pDlidx); + fts5DlidxIterNext(p, pDlidx); + } + assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc ); + if( iLeafPgno>pIter->iLeafPgno ){ + fts5SegIterGotoPage(p, pIter, iLeafPgno); + bMove = 0; + } + }else{ + assert( pIter->pNextLeaf==0 ); + assert( iMatchiRowid ); + while( !fts5DlidxIterEof(p, pDlidx) && iMatchiLeafPgno ); + + if( iLeafPgnoiLeafPgno ){ + pIter->iLeafPgno = iLeafPgno+1; + fts5SegIterReverseNewPage(p, pIter); + bMove = 0; + } + } + + do{ + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); + if( pIter->pLeaf==0 ) break; + if( bRev==0 && pIter->iRowid>=iMatch ) break; + if( bRev!=0 && pIter->iRowid<=iMatch ) break; + bMove = 1; + }while( p->rc==SQLITE_OK ); +} + + +/* +** Free the iterator object passed as the second argument. +*/ +static void fts5MultiIterFree(Fts5Iter *pIter){ + if( pIter ){ + int i; + for(i=0; inSeg; i++){ + fts5SegIterClear(&pIter->aSeg[i]); + } + fts5BufferFree(&pIter->poslist); + sqlite3_free(pIter); + } +} + +static void fts5MultiIterAdvanced( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + int iMinset /* Minimum entry in aFirst[] to set */ +){ + int i; + for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, 0); + i = pIter->nSeg + iEq; + } + } +} + +/* +** Sub-iterator iChanged of iterator pIter has just been advanced. It still +** points to the same term though - just a different rowid. This function +** attempts to update the contents of the pIter->aFirst[] accordingly. +** If it does so successfully, 0 is returned. Otherwise 1. +** +** If non-zero is returned, the caller should call fts5MultiIterAdvanced() +** on the iterator instead. That function does the same as this one, except +** that it deals with more complicated cases as well. +*/ +static int fts5MultiIterAdvanceRowid( + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + Fts5SegIter **ppFirst +){ + Fts5SegIter *pNew = &pIter->aSeg[iChanged]; + + if( pNew->iRowid==pIter->iSwitchRowid + || (pNew->iRowidiSwitchRowid)==pIter->bRev + ){ + int i; + Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001]; + pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64; + for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){ + Fts5CResult *pRes = &pIter->aFirst[i]; + + assert( pNew->pLeaf ); + assert( pRes->bTermEq==0 || pOther->pLeaf ); + + if( pRes->bTermEq ){ + if( pNew->iRowid==pOther->iRowid ){ + return 1; + }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + pNew = pOther; + }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + } + } + pRes->iFirst = (u16)(pNew - pIter->aSeg); + if( i==1 ) break; + + pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; + } + } + + *ppFirst = pNew; + return 0; +} + +/* +** Set the pIter->bEof variable based on the state of the sub-iterators. +*/ +static void fts5MultiIterSetEof(Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + pIter->base.bEof = pSeg->pLeaf==0; + pIter->iSwitchRowid = pSeg->iRowid; +} + +/* +** Move the iterator to the next entry. +** +** If an error occurs, an error code is left in Fts5Index.rc. It is not +** considered an error if the iterator reaches EOF, or if it is already at +** EOF when this function is called. +*/ +static void fts5MultiIterNext( + Fts5Index *p, + Fts5Iter *pIter, + int bFrom, /* True if argument iFrom is valid */ + i64 iFrom /* Advance at least as far as this */ +){ + int bUseFrom = bFrom; + assert( pIter->base.bEof==0 ); + while( p->rc==SQLITE_OK ){ + int iFirst = pIter->aFirst[1].iFirst; + int bNewTerm = 0; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + assert( p->rc==SQLITE_OK ); + if( bUseFrom && pSeg->pDlidx ){ + fts5SegIterNextFrom(p, pSeg, iFrom); + }else{ + pSeg->xNext(p, pSeg, &bNewTerm); + } + + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + if( pSeg->pLeaf==0 ) return; + } + + fts5AssertMultiIterSetup(p, pIter); + assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); + if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + pIter->xSetOutputs(pIter, pSeg); + return; + } + bUseFrom = 0; + } +} + +static void fts5MultiIterNext2( + Fts5Index *p, + Fts5Iter *pIter, + int *pbNewTerm /* OUT: True if *might* be new term */ +){ + assert( pIter->bSkipEmpty ); + if( p->rc==SQLITE_OK ){ + *pbNewTerm = 0; + do{ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + int bNewTerm = 0; + + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, &bNewTerm); + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + *pbNewTerm = 1; + } + fts5AssertMultiIterSetup(p, pIter); + + }while( fts5MultiIterIsEmpty(p, pIter) ); + } +} + +static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ + UNUSED_PARAM2(pUnused1, pUnused2); +} + +static Fts5Iter *fts5MultiIterAlloc( + Fts5Index *p, /* FTS5 backend to iterate within */ + int nSeg +){ + Fts5Iter *pNew; + int nSlot; /* Power of two >= nSeg */ + + for(nSlot=2; nSlotaSeg[] */ + sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ + ); + if( pNew ){ + pNew->nSeg = nSlot; + pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; + pNew->pIndex = p; + pNew->xSetOutputs = fts5IterSetOutputs_Noop; + } + return pNew; +} + +static void fts5PoslistCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} + +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; + +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static void fts5PoslistOffsetsCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } + } + } +} + +static void fts5PoslistFilterCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; + + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); + }else{ + pCtx->eState = 0; + } + } + + do { + while( ieState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i=nChunk ){ + pCtx->eState = 2; + }else{ + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } + } + } + }while( inPos; /* Number of bytes still to come */ + Fts5Data *pData = 0; + u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); + int pgno = pSeg->iLeafPgno; + int pgnoSave = 0; + + /* This function does not work with detail=none databases. */ + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ + pgnoSave = pgno+1; + } + + while( 1 ){ + xChunk(p, pCtx, pChunk, nChunk); + nRem -= nChunk; + fts5DataRelease(pData); + if( nRem<=0 ){ + break; + }else if( pSeg->pSeg==0 ){ + p->rc = FTS5_CORRUPT; + return; + }else{ + pgno++; + pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); + if( pData==0 ) break; + pChunk = &pData->p[4]; + nChunk = MIN(nRem, pData->szLeaf - 4); + if( pgno==pgnoSave ){ + assert( pSeg->pNextLeaf==0 ); + pSeg->pNextLeaf = pData; + pData = 0; + } + } + } +} + +/* +** Iterator pIter currently points to a valid entry (not EOF). This +** function appends the position list data for the current entry to +** buffer pBuf. It does not make a copy of the position-list size +** field. +*/ +static void fts5SegiterPoslist( + Fts5Index *p, + Fts5SegIter *pSeg, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+FTS5_DATA_ZERO_PADDING) ){ + memset(&pBuf->p[pBuf->n+pSeg->nPos], 0, FTS5_DATA_ZERO_PADDING); + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } + } + } +} + +/* +** Parameter pPos points to a buffer containing a position list, size nPos. +** This function filters it according to pColset (which must be non-NULL) +** and sets pIter->base.pData/nData to point to the new position list. +** If memory is required for the new position list, use buffer pIter->poslist. +** Or, if the new position list is a contiguous subset of the input, set +** pIter->base.pData/nData to point directly to it. +** +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. If an OOM error is encountered, *pRc is set to SQLITE_NOMEM +** before returning. +*/ +static void fts5IndexExtractColset( + int *pRc, + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Iter *pIter +){ + if( *pRc==SQLITE_OK ){ + const u8 *p = pPos; + const u8 *aCopy = p; + const u8 *pEnd = &p[nPos]; /* One byte past end of position list */ + int i = 0; + int iCurrent = 0; + + if( pColset->nCol>1 && sqlite3Fts5BufferSize(pRc, &pIter->poslist, nPos) ){ + return; + } + + while( 1 ){ + while( pColset->aiCol[i]nCol ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + } + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( paiCol[i]==iCurrent ){ + if( pColset->nCol==1 ){ + pIter->base.pData = aCopy; + pIter->base.nData = p-aCopy; + return; + } + fts5BufferSafeAppendBlob(&pIter->poslist, aCopy, p-aCopy); + } + if( p==pEnd ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + aCopy = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } + } + } + +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + UNUSED_PARAM(pSeg); + pIter->base.nData = 0; +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. +*/ +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pIter->pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; + + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( abase.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int *pRc = &pIter->pIndex->rc; + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, pIter); + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + if( *pRc==SQLITE_OK ){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } + } + } +} + + +/* +** Allocate a new Fts5Iter object. +** +** The new object will be used to iterate through data in structure pStruct. +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel +** is zero or greater, data from the first nSegment segments on level iLevel +** is merged. +** +** The iterator initially points to the first term/rowid entry in the +** iterated data. +*/ +static void fts5MultiIterNew( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Structure *pStruct, /* Structure of specific index */ + int flags, /* FTS5INDEX_QUERY_XXX flags */ + Fts5Colset *pColset, /* Colset to filter on (or NULL) */ + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ + int iLevel, /* Level to iterate (-1 for all) */ + int nSegment, /* Number of segments to merge (iLevel>=0) */ + Fts5Iter **ppOut /* New object */ +){ + int nSeg = 0; /* Number of segment-iters in use */ + int iIter = 0; /* */ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl; + Fts5Iter *pNew; + + assert( (pTerm==0 && nTerm==0) || iLevel<0 ); + + /* Allocate space for the new multi-seg-iterator. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->pHash ? 1 : 0); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + } + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); + if( pNew==0 ) return; + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pColset = pColset; + if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ + fts5IterSetOutputCb(&p->rc, pNew); + } + + /* Initialize each of the component segment iterators. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->pHash ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); + } + for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + if( pTerm==0 ){ + fts5SegIterInit(p, pSeg, pIter); + }else{ + fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); + } + } + } + }else{ + pLvl = &pStruct->aLevel[iLevel]; + for(iSeg=nSeg-1; iSeg>=0; iSeg--){ + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + } + } + assert( iIter==nSeg ); + } + + /* If the above was successful, each component iterators now points + ** to the first entry in its segment. In this case initialize the + ** aFirst[] array. Or, if an error has occurred, free the iterator + ** object and set the output variable to NULL. */ + if( p->rc==SQLITE_OK ){ + for(iIter=pNew->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pNew, iEq, iIter); + } + } + fts5MultiIterSetEof(pNew); + fts5AssertMultiIterSetup(p, pNew); + + if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + fts5MultiIterNext(p, pNew, 0, 0); + }else if( pNew->base.bEof==0 ){ + Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; + pNew->xSetOutputs(pNew, pSeg); + } + + }else{ + fts5MultiIterFree(pNew); + *ppOut = 0; + } +} + +/* +** Create an Fts5Iter that iterates through the doclist provided +** as the second argument. +*/ +static void fts5MultiIterNew2( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Data *pData, /* Doclist to iterate through */ + int bDesc, /* True for descending rowid order */ + Fts5Iter **ppOut /* New object */ +){ + Fts5Iter *pNew; + pNew = fts5MultiIterAlloc(p, 2); + if( pNew ){ + Fts5SegIter *pIter = &pNew->aSeg[1]; + + pIter->flags = FTS5_SEGITER_ONETERM; + if( pData->szLeaf>0 ){ + pIter->pLeaf = pData; + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; + pNew->aFirst[1].iFirst = 1; + if( bDesc ){ + pNew->bRev = 1; + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + pData = 0; + }else{ + pNew->base.bEof = 1; + } + fts5SegIterSetNext(p, pIter); + + *ppOut = pNew; + } + + fts5DataRelease(pData); +} + +/* +** Return true if the iterator is at EOF or if an error has occurred. +** False otherwise. +*/ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ + assert( p->rc + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + ); + return (p->rc || pIter->base.bEof); +} + +/* +** Return the rowid of the entry that the iterator currently points +** to. If the iterator points to EOF when this function is called the +** results are undefined. +*/ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; +} + +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5Iter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter, 1, iMatch); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid>=iMatch ) break; + if( pIter->bRev!=0 && iRowid<=iMatch ) break; + } +} + +/* +** Return a pointer to a buffer containing the term associated with the +** entry that the iterator currently points to. +*/ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + *pn = p->term.n; + return p->term.p; +} + +/* +** Allocate a new segment-id for the structure pStruct. The new segment +** id must be between 1 and 65335 inclusive, and must not be used by +** any currently existing segment. If a free segment id cannot be found, +** SQLITE_FULL is returned. +** +** If an error has already occurred, this function is a no-op. 0 is +** returned in this case. +*/ +static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ + int iSegid = 0; + + if( p->rc==SQLITE_OK ){ + if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ + p->rc = SQLITE_FULL; + }else{ + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT && iId>0 ){ + aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32); + } + } + } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + + { + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc==SQLITE_OK ){ + u8 aBlob[2] = {0xff, 0xff}; + sqlite3_bind_int(pIdxSelect, 1, iSegid); + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); + assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); + } + } +#endif + } + } + + return iSegid; +} + +/* +** Discard all data currently cached in the hash-tables. +*/ +static void fts5IndexDiscardData(Fts5Index *p){ + assert( p->pHash || p->nPendingData==0 ); + if( p->pHash ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + } +} + +/* +** Return the size of the prefix, in bytes, that buffer +** (pNew/) shares with buffer (pOld/nOld). +** +** Buffer (pNew/) is guaranteed to be greater +** than buffer (pOld/nOld). +*/ +static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ + int i; + for(i=0; inDlidx>0 && pWriter->aDlidx[0].buf.n>0) ); + for(i=0; inDlidx; i++){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + if( pDlidx->buf.n==0 ) break; + if( bFlush ){ + assert( pDlidx->pgno!=0 ); + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + } + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + } +} + +/* +** Grow the pWriter->aDlidx[] array to at least nLvl elements in size. +** Any new array elements are zeroed before returning. +*/ +static int fts5WriteDlidxGrow( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nLvl +){ + if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){ + Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc64( + pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl + ); + if( aDlidx==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + memset(&aDlidx[pWriter->nDlidx], 0, nByte); + pWriter->aDlidx = aDlidx; + pWriter->nDlidx = nLvl; + } + } + return p->rc; +} + +/* +** If the current doclist-index accumulating in pWriter->aDlidx[] is large +** enough, flush it to disk and return 1. Otherwise discard it and return +** zero. +*/ +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag = 0; + + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written + ** to the database, also write the doclist-index to disk. */ + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ + bFlag = 1; + } + fts5WriteDlidxClear(p, pWriter, bFlag); + pWriter->nEmpty = 0; + return bFlag; +} + +/* +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. +** +** The doclist-index for that term is currently stored in-memory within the +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function +** writes it out to disk. Or, if it is too small to bother with, discards +** it. +** +** Fts5SegWriter.btterm currently contains the first term on page iBtPage. +*/ +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag; + + assert( pWriter->iBtPage || pWriter->nEmpty==0 ); + if( pWriter->iBtPage==0 ) return; + bFlag = fts5WriteFlushDlidx(p, pWriter); + + if( p->rc==SQLITE_OK ){ + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); + /* The following was already done in fts5WriteInit(): */ + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); + sqlite3_step(p->pIdxWriter); + p->rc = sqlite3_reset(p->pIdxWriter); + sqlite3_bind_null(p->pIdxWriter, 2); + } + pWriter->iBtPage = 0; +} + +/* +** This is called once for each leaf page except the first that contains +** at least one term. Argument (nTerm/pTerm) is the split-key - a term that +** is larger than all terms written to earlier leaves, and equal to or +** smaller than the first term on the new leaf. +** +** If an error occurs, an error code is left in Fts5Index.rc. If an error +** has already occurred when this function is called, it is a no-op. +*/ +static void fts5WriteBtreeTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter, /* Writer object */ + int nTerm, const u8 *pTerm /* First term on new page */ +){ + fts5WriteFlushBtree(p, pWriter); + if( p->rc==SQLITE_OK ){ + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; + } +} + +/* +** This function is called when flushing a leaf page that contains no +** terms at all to disk. +*/ +static void fts5WriteBtreeNoTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter /* Writer object */ +){ + /* If there were no rowids on the leaf page either and the doclist-index + ** has already been started, append an 0x00 byte to it. */ + if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0]; + assert( pDlidx->bPrevValid ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0); + } + + /* Increment the "number of sequential leaves without a term" counter. */ + pWriter->nEmpty++; +} + +static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){ + i64 iRowid; + int iOff; + + iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid); + fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid); + return iRowid; +} + +/* +** Rowid iRowid has just been appended to the current leaf page. It is the +** first on the page. This function appends an appropriate entry to the current +** doclist-index. +*/ +static void fts5WriteDlidxAppend( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + i64 iVal; + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + + if( pDlidx->buf.n>=p->pConfig->pgsz ){ + /* The current doclist-index page is full. Write it to disk and push + ** a copy of iRowid (which will become the first rowid on the next + ** doclist-index leaf page) up into the next level of the b-tree + ** hierarchy. If the node being flushed is currently the root node, + ** also push its first rowid upwards. */ + pDlidx->buf.p[0] = 0x01; /* Not the root node */ + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + fts5WriteDlidxGrow(p, pWriter, i+2); + pDlidx = &pWriter->aDlidx[i]; + if( p->rc==SQLITE_OK && pDlidx[1].buf.n==0 ){ + i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf); + + /* This was the root node. Push its first rowid up to the new root. */ + pDlidx[1].pgno = pDlidx->pgno; + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst); + pDlidx[1].bPrevValid = 1; + pDlidx[1].iPrev = iFirst; + } + + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + pDlidx->pgno++; + }else{ + bDone = 1; + } + + if( pDlidx->bPrevValid ){ + iVal = iRowid - pDlidx->iPrev; + }else{ + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); + assert( pDlidx->buf.n==0 ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); + iVal = iRowid; + } + + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal); + pDlidx->bPrevValid = 1; + pDlidx->iPrev = iRowid; + } +} + +static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ + static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; + Fts5PageWriter *pPage = &pWriter->writer; + i64 iRowid; + + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); + + /* Set the szLeaf header field. */ + assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); + + if( pWriter->bFirstTermInPage ){ + /* No term was written to this page. */ + assert( pPage->pgidx.n==0 ); + fts5WriteBtreeNoTerm(p, pWriter); + }else{ + /* Append the pgidx to the page buffer. Set the szLeaf header field. */ + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); + } + + /* Write the page out to disk */ + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); + fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); + + /* Initialize the next page. */ + fts5BufferZero(&pPage->buf); + fts5BufferZero(&pPage->pgidx); + fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); + pPage->iPrevPgidx = 0; + pPage->pgno++; + + /* Increase the leaves written counter */ + pWriter->nLeafWritten++; + + /* The new leaf holds no terms or rowids */ + pWriter->bFirstTermInPage = 1; + pWriter->bFirstRowidInPage = 1; +} + +/* +** Append term pTerm/nTerm to the segment being written by the writer passed +** as the second argument. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5WriteAppendTerm( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nTerm, const u8 *pTerm +){ + int nPrefix; /* Bytes of prefix compression for term */ + Fts5PageWriter *pPage = &pWriter->writer; + Fts5Buffer *pPgidx = &pWriter->writer.pgidx; + int nMin = MIN(pPage->term.n, nTerm); + + assert( p->rc==SQLITE_OK ); + assert( pPage->buf.n>=4 ); + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); + + /* If the current leaf page is full, flush it to disk. */ + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ + if( pPage->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + if( p->rc!=SQLITE_OK ) return; + } + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); + } + + /* TODO1: Updating pgidx here. */ + pPgidx->n += sqlite3Fts5PutVarint( + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx + ); + pPage->iPrevPgidx = pPage->buf.n; +#if 0 + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); + pPgidx->n += 2; +#endif + + if( pWriter->bFirstTermInPage ){ + nPrefix = 0; + if( pPage->pgno!=1 ){ + /* This is the first term on a leaf that is not the leftmost leaf in + ** the segment b-tree. In this case it is necessary to add a term to + ** the b-tree hierarchy that is (a) larger than the largest term + ** already written to the segment and (b) smaller than or equal to + ** this term. In other words, a prefix of (pTerm/nTerm) that is one + ** byte longer than the longest prefix (pTerm/nTerm) shares with the + ** previous term. + ** + ** Usually, the previous term is available in pPage->term. The exception + ** is if this is the first term written in an incremental-merge step. + ** In this case the previous term is not available, so just write a + ** copy of (pTerm/nTerm) into the parent node. This is slightly + ** inefficient, but still correct. */ + int n = nTerm; + if( pPage->term.n ){ + n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm); + } + fts5WriteBtreeTerm(p, pWriter, n, pTerm); + if( p->rc!=SQLITE_OK ) return; + pPage = &pWriter->writer; + } + }else{ + nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm); + fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); + } + + /* Append the number of bytes of new data, then the term data itself + ** to the page. */ + fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); + fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); + + /* Update the Fts5PageWriter.term field. */ + fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); + pWriter->bFirstTermInPage = 0; + + pWriter->bFirstRowidInPage = 0; + pWriter->bFirstRowidInDoclist = 1; + + assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); + pWriter->aDlidx[0].pgno = pPage->pgno; +} + +/* +** Append a rowid and position-list size field to the writers output. +*/ +static void fts5WriteAppendRowid( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + if( p->rc==SQLITE_OK ){ + Fts5PageWriter *pPage = &pWriter->writer; + + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ + fts5WriteFlushLeaf(p, pWriter); + } + + /* If this is to be the first rowid written to the page, set the + ** rowid-pointer in the page-header. Also append a value to the dlidx + ** buffer, in case a doclist-index is required. */ + if( pWriter->bFirstRowidInPage ){ + fts5PutU16(pPage->buf.p, (u16)pPage->buf.n); + fts5WriteDlidxAppend(p, pWriter, iRowid); + } + + /* Write the rowid. */ + if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); + }else{ + assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + } + pWriter->iPrevRowid = iRowid; + pWriter->bFirstRowidInDoclist = 0; + pWriter->bFirstRowidInPage = 0; + } +} + +static void fts5WriteAppendPoslistData( + Fts5Index *p, + Fts5SegWriter *pWriter, + const u8 *aData, + int nData +){ + Fts5PageWriter *pPage = &pWriter->writer; + const u8 *a = aData; + int n = nData; + + assert( p->pConfig->pgsz>0 ); + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + ){ + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; + int nCopy = 0; + while( nCopyrc, &pPage->buf, nCopy, a); + a += nCopy; + n -= nCopy; + fts5WriteFlushLeaf(p, pWriter); + } + if( n>0 ){ + fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); + } +} + +/* +** Flush any data cached by the writer object to the database. Free any +** allocations associated with the writer. +*/ +static void fts5WriteFinish( + Fts5Index *p, + Fts5SegWriter *pWriter, /* Writer object */ + int *pnLeaf /* OUT: Number of leaf pages in b-tree */ +){ + int i; + Fts5PageWriter *pLeaf = &pWriter->writer; + if( p->rc==SQLITE_OK ){ + assert( pLeaf->pgno>=1 ); + if( pLeaf->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + *pnLeaf = pLeaf->pgno-1; + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } + } + fts5BufferFree(&pLeaf->term); + fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pLeaf->pgidx); + fts5BufferFree(&pWriter->btterm); + + for(i=0; inDlidx; i++){ + sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); + } + sqlite3_free(pWriter->aDlidx); +} + +static void fts5WriteInit( + Fts5Index *p, + Fts5SegWriter *pWriter, + int iSegid +){ + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; + + memset(pWriter, 0, sizeof(Fts5SegWriter)); + pWriter->iSegid = iSegid; + + fts5WriteDlidxGrow(p, pWriter, 1); + pWriter->writer.pgno = 1; + pWriter->bFirstTermInPage = 1; + pWriter->iBtPage = 1; + + assert( pWriter->writer.buf.n==0 ); + assert( pWriter->writer.pgidx.n==0 ); + + /* Grow the two buffers to pgsz + padding bytes in size. */ + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer); + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer); + + if( p->pIdxWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + pConfig->zDb, pConfig->zName + )); + } + + if( p->rc==SQLITE_OK ){ + /* Initialize the 4-byte leaf-page header to 0x00. */ + memset(pWriter->writer.buf.p, 0, 4); + pWriter->writer.buf.n = 4; + + /* Bind the current output segment id to the index-writer. This is an + ** optimization over binding the same value over and over as rows are + ** inserted into %_idx by the current writer. */ + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); + } +} + +/* +** Iterator pIter was used to iterate through the input segments of on an +** incremental merge operation. This function is called if the incremental +** merge step has finished but the input has not been completely exhausted. +*/ +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ + int i; + Fts5Buffer buf; + memset(&buf, 0, sizeof(Fts5Buffer)); + for(i=0; inSeg && p->rc==SQLITE_OK; i++){ + Fts5SegIter *pSeg = &pIter->aSeg[i]; + if( pSeg->pSeg==0 ){ + /* no-op */ + }else if( pSeg->pLeaf==0 ){ + /* All keys from this input segment have been transfered to the output. + ** Set both the first and last page-numbers to 0 to indicate that the + ** segment is now empty. */ + pSeg->pSeg->pgnoLast = 0; + pSeg->pSeg->pgnoFirst = 0; + }else{ + int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */ + i64 iLeafRowid; + Fts5Data *pData; + int iId = pSeg->pSeg->iSegid; + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; + + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); + pData = fts5LeafRead(p, iLeafRowid); + if( pData ){ + if( iOff>pData->szLeaf ){ + /* This can occur if the pages that the segments occupy overlap - if + ** a single page has been assigned to more than one segment. In + ** this case a prior iteration of this loop may have corrupted the + ** segment currently being trimmed. */ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); + fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); + fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); + fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], (u16)buf.n); + } + + /* Set up the new page-index array */ + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclistszLeaf + && pSeg->iPgidxOff<=pData->nn + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } + + pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); + fts5DataWrite(p, iLeafRowid, buf.p, buf.n); + } + fts5DataRelease(pData); + } + } + } + fts5BufferFree(&buf); +} + +static void fts5MergeChunkCallback( + Fts5Index *p, + void *pCtx, + const u8 *pChunk, int nChunk +){ + Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; + fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); +} + +/* +** +*/ +static void fts5IndexMergeLevel( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ + int iLvl, /* Level to read input from */ + int *pnRem /* Write up to this many output leaves */ +){ + Fts5Structure *pStruct = *ppStruct; + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureLevel *pLvlOut; + Fts5Iter *pIter = 0; /* Iterator to read input data */ + int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ + int nInput; /* Number of input segments */ + Fts5SegWriter writer; /* Writer object */ + Fts5StructureSegment *pSeg; /* Output segment */ + Fts5Buffer term; + int bOldest; /* True if the output segment is the oldest */ + int eDetail = p->pConfig->eDetail; + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + int bTermWritten = 0; /* True if current term already output */ + + assert( iLvlnLevel ); + assert( pLvl->nMerge<=pLvl->nSeg ); + + memset(&writer, 0, sizeof(Fts5SegWriter)); + memset(&term, 0, sizeof(Fts5Buffer)); + if( pLvl->nMerge ){ + pLvlOut = &pStruct->aLevel[iLvl+1]; + assert( pLvlOut->nSeg>0 ); + nInput = pLvl->nMerge; + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; + + fts5WriteInit(p, &writer, pSeg->iSegid); + writer.writer.pgno = pSeg->pgnoLast+1; + writer.iBtPage = 0; + }else{ + int iSegid = fts5AllocateSegid(p, pStruct); + + /* Extend the Fts5Structure object as required to ensure the output + ** segment exists. */ + if( iLvl==pStruct->nLevel-1 ){ + fts5StructureAddLevel(&p->rc, ppStruct); + pStruct = *ppStruct; + } + fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); + if( p->rc ) return; + pLvl = &pStruct->aLevel[iLvl]; + pLvlOut = &pStruct->aLevel[iLvl+1]; + + fts5WriteInit(p, &writer, iSegid); + + /* Add the new segment to the output level */ + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; + pLvlOut->nSeg++; + pSeg->pgnoFirst = 1; + pSeg->iSegid = iSegid; + pStruct->nSegment++; + + /* Read input from all segments in the input level */ + nInput = pLvl->nSeg; + } + bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); + + assert( iLvl>=0 ); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int nPos; /* position-list size field value */ + int nTerm; + const u8 *pTerm; + + pTerm = fts5MultiIterTerm(pIter, &nTerm); + if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm) ){ + if( pnRem && writer.nLeafWritten>nRem ){ + break; + } + fts5BufferSet(&p->rc, &term, nTerm, pTerm); + bTermWritten =0; + } + + /* Check for key annihilation. */ + if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + /* This is a new term. Append a term to the output segment. */ + fts5WriteAppendTerm(p, &writer, nTerm, pTerm); + bTermWritten = 1; + } + + /* Append the rowid to the output */ + /* WRITEPOSLISTSIZE */ + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( pSegIter->bDel ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + if( pSegIter->nPos>0 ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + } + } + }else{ + /* Append the position-list data to the output */ + nPos = pSegIter->nPos*2 + pSegIter->bDel; + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + } + } + + /* Flush the last leaf page to disk. Set the output segment b-tree height + ** and last leaf page number at the same time. */ + fts5WriteFinish(p, &writer, &pSeg->pgnoLast); + + if( fts5MultiIterEof(p, pIter) ){ + int i; + + /* Remove the redundant segments from the %_data table */ + for(i=0; iaSeg[i].iSegid); + } + + /* Remove the redundant segments from the input level */ + if( pLvl->nSeg!=nInput ){ + int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); + memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); + } + pStruct->nSegment -= nInput; + pLvl->nSeg -= nInput; + pLvl->nMerge = 0; + if( pSeg->pgnoLast==0 ){ + pLvlOut->nSeg--; + pStruct->nSegment--; + } + }else{ + assert( pSeg->pgnoLast>0 ); + fts5TrimSegments(p, pIter); + pLvl->nMerge = nInput; + } + + fts5MultiIterFree(pIter); + fts5BufferFree(&term); + if( pnRem ) *pnRem -= writer.nLeafWritten; +} + +/* +** Do up to nPg pages of automerge work on the index. +** +** Return true if any changes were actually made, or false otherwise. +*/ +static int fts5IndexMerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nPg, /* Pages of work to do */ + int nMin /* Minimum number of segments to merge */ +){ + int nRem = nPg; + int bRet = 0; + Fts5Structure *pStruct = *ppStruct; + while( nRem>0 && p->rc==SQLITE_OK ){ + int iLvl; /* To iterate through levels */ + int iBestLvl = 0; /* Level offering the most input segments */ + int nBest = 0; /* Number of input segments on best level */ + + /* Set iBestLvl to the level to read input segments from. */ + assert( pStruct->nLevel>0 ); + for(iLvl=0; iLvlnLevel; iLvl++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + if( pLvl->nMerge ){ + if( pLvl->nMerge>nBest ){ + iBestLvl = iLvl; + nBest = pLvl->nMerge; + } + break; + } + if( pLvl->nSeg>nBest ){ + nBest = pLvl->nSeg; + iBestLvl = iLvl; + } + } + + /* If nBest is still 0, then the index must be empty. */ +#ifdef SQLITE_DEBUG + for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ + assert( pStruct->aLevel[iLvl].nSeg==0 ); + } +#endif + + if( nBestaLevel[iBestLvl].nMerge==0 ){ + break; + } + bRet = 1; + fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); + if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ + fts5StructurePromote(p, iBestLvl+1, pStruct); + } + } + *ppStruct = pStruct; + return bRet; +} + +/* +** A total of nLeaf leaf pages of data has just been flushed to a level-0 +** segment. This function updates the write-counter accordingly and, if +** necessary, performs incremental merge work. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5IndexAutomerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nLeaf /* Number of output leaves just written */ +){ + if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ + Fts5Structure *pStruct = *ppStruct; + u64 nWrite; /* Initial value of write-counter */ + int nWork; /* Number of work-quanta to perform */ + int nRem; /* Number of leaf pages left to write */ + + /* Update the write-counter. While doing so, set nWork. */ + nWrite = pStruct->nWriteCounter; + nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); + pStruct->nWriteCounter += nLeaf; + nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); + + fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); + } +} + +static void fts5IndexCrisismerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ +){ + const int nCrisis = p->pConfig->nCrisisMerge; + Fts5Structure *pStruct = *ppStruct; + int iLvl = 0; + + assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; +} + +static int fts5IndexReturn(Fts5Index *p){ + int rc = p->rc; + p->rc = SQLITE_OK; + return rc; +} + +typedef struct Fts5FlushCtx Fts5FlushCtx; +struct Fts5FlushCtx { + Fts5Index *pIdx; + Fts5SegWriter writer; +}; + +/* +** Buffer aBuf[] contains a list of varints, all small enough to fit +** in a 32-bit integer. Return the size of the largest prefix of this +** list nMax bytes or less in size. +*/ +static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ + int ret; + u32 dummy; + ret = fts5GetVarint32(aBuf, dummy); + if( ret nMax ) break; + ret += i; + } + } + return ret; +} + +/* +** Flush the contents of in-memory hash table iHash to a new level-0 +** segment on disk. Also update the corresponding structure record. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5FlushOneHash(Fts5Index *p){ + Fts5Hash *pHash = p->pHash; + Fts5Structure *pStruct; + int iSegid; + int pgnoLast = 0; /* Last leaf page number in segment */ + + /* Obtain a reference to the index structure and allocate a new segment-id + ** for the new level-0 segment. */ + pStruct = fts5StructureRead(p); + iSegid = fts5AllocateSegid(p, pStruct); + fts5StructureInvalidate(p); + + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Write the term for this entry to disk. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); + fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + + assert( writer.bFirstRowidInPage==0 ); + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + i64 iRowid = 0; + i64 iDelta = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffp[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + if( p->rc!=SQLITE_OK ) break; + }else{ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + } + assert( pBuf->n<=pBuf->nSpace ); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffp[pBuf->n++] = 0; + iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; + } + } + } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); + } + sqlite3Fts5HashClear(pHash); + fts5WriteFinish(p, &writer, &pgnoLast); + + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); + } + + fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexCrisismerge(p, &pStruct); + fts5StructureWrite(p, pStruct); + fts5StructureRelease(pStruct); +} + +/* +** Flush any data stored in the in-memory hash tables to the database. +*/ +static void fts5IndexFlush(Fts5Index *p){ + /* Unless it is empty, flush the hash table to disk */ + if( p->nPendingData ){ + assert( p->pHash ); + p->nPendingData = 0; + fts5FlushOneHash(p); + } +} + +static Fts5Structure *fts5IndexOptimizeStruct( + Fts5Index *p, + Fts5Structure *pStruct +){ + Fts5Structure *pNew = 0; + sqlite3_int64 nByte = sizeof(Fts5Structure); + int nSeg = pStruct->nSegment; + int i; + + /* Figure out if this structure requires optimization. A structure does + ** not require optimization if either: + ** + ** + it consists of fewer than two segments, or + ** + all segments are on the same level, or + ** + all segments except one are currently inputs to a merge operation. + ** + ** In the first case, return NULL. In the second, increment the ref-count + ** on *pStruct and return a copy of the pointer to it. + */ + if( nSeg<2 ) return 0; + for(i=0; inLevel; i++){ + int nThis = pStruct->aLevel[i].nSeg; + if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + fts5StructureRef(pStruct); + return pStruct; + } + assert( pStruct->aLevel[i].nMerge<=nThis ); + } + + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + + if( pNew ){ + Fts5StructureLevel *pLvl; + nByte = nSeg * sizeof(Fts5StructureSegment); + pNew->nLevel = pStruct->nLevel+1; + pNew->nRef = 1; + pNew->nWriteCounter = pStruct->nWriteCounter; + pLvl = &pNew->aLevel[pStruct->nLevel]; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pLvl->aSeg ){ + int iLvl, iSeg; + int iSegOut = 0; + /* Iterate through all segments, from oldest to newest. Add them to + ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest + ** segment in the data structure. */ + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; + iSegOut++; + } + } + pNew->nSegment = pLvl->nSeg = nSeg; + }else{ + sqlite3_free(pNew); + pNew = 0; + } + } + + return pNew; +} + +static int sqlite3Fts5IndexOptimize(Fts5Index *p){ + Fts5Structure *pStruct; + Fts5Structure *pNew = 0; + + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + fts5StructureInvalidate(p); + + if( pStruct ){ + pNew = fts5IndexOptimizeStruct(p, pStruct); + } + fts5StructureRelease(pStruct); + + assert( pNew==0 || pNew->nSegment>0 ); + if( pNew ){ + int iLvl; + for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ + int nRem = FTS5_OPT_WORK_UNIT; + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); + } + + fts5StructureWrite(p, pNew); + fts5StructureRelease(pNew); + } + + return fts5IndexReturn(p); +} + +/* +** This is called to implement the special "VALUES('merge', $nMerge)" +** INSERT command. +*/ +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ + Fts5Structure *pStruct = fts5StructureRead(p); + if( pStruct ){ + int nMin = p->pConfig->nUsermerge; + fts5StructureInvalidate(p); + if( nMerge<0 ){ + Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); + fts5StructureRelease(pStruct); + pStruct = pNew; + nMin = 2; + nMerge = nMerge*-1; + } + if( pStruct && pStruct->nLevel ){ + if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ + fts5StructureWrite(p, pStruct); + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +static void fts5AppendRowid( + Fts5Index *p, + i64 iDelta, + Fts5Iter *pUnused, + Fts5Buffer *pBuf +){ + UNUSED_PARAM(pUnused); + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); +} + +static void fts5AppendPoslist( + Fts5Index *p, + i64 iDelta, + Fts5Iter *pMulti, + Fts5Buffer *pBuf +){ + int nData = pMulti->base.nData; + int nByte = nData + 9 + 9 + FTS5_DATA_ZERO_PADDING; + assert( nData>0 ); + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nByte) ){ + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nData*2); + fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); + memset(&pBuf->p[pBuf->n], 0, FTS5_DATA_ZERO_PADDING); + } +} + + +static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ + u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; + + assert( pIter->aPoslist || (p==0 && pIter->aPoslist==0) ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + pIter->nSize = fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); + }else{ + pIter->nPoslist = ((int)(p[0])) >> 1; + pIter->nSize = 1; + } + + pIter->aPoslist = p; + if( &pIter->aPoslist[pIter->nPoslist]>pIter->aEof ){ + pIter->aPoslist = 0; + } + } +} + +static void fts5DoclistIterInit( + Fts5Buffer *pBuf, + Fts5DoclistIter *pIter +){ + memset(pIter, 0, sizeof(*pIter)); + if( pBuf->n>0 ){ + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; + fts5DoclistIterNext(pIter); + } +} + +#if 0 +/* +** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. +*/ +static void fts5MergeAppendDocid( + Fts5Buffer *pBuf, /* Buffer to write to */ + i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ + i64 iRowid /* Rowid to append */ +){ + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); + *piLastRowid = iRowid; +} +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} + +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ + Fts5Buffer tmp = *p1; + *p1 = *p2; + *p2 = tmp; +} + +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ + int i = *piOff; + if( i>=pBuf->n ){ + *piOff = -1; + }else{ + u64 iVal; + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); + *piRowid += iVal; + } +} + +/* +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. +** In this case the buffers consist of a delta-encoded list of rowids only. +*/ +static void fts5MergeRowidLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + int nBuf, /* Number of entries in apBuf[] */ + Fts5Buffer *aBuf /* Array of other lists to merge into p1 */ +){ + int i1 = 0; + int i2 = 0; + i64 iRowid1 = 0; + i64 iRowid2 = 0; + i64 iOut = 0; + Fts5Buffer *p2 = &aBuf[0]; + Fts5Buffer out; + + (void)nBuf; + memset(&out, 0, sizeof(out)); + assert( nBuf==1 ); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); + if( p->rc ) return; + + fts5NextRowid(p1, &i1, &iRowid1); + fts5NextRowid(p2, &i2, &iRowid2); + while( i1>=0 || i2>=0 ){ + if( i1>=0 && (i2<0 || iRowid1iOut ); + fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); + iOut = iRowid1; + fts5NextRowid(p1, &i1, &iRowid1); + }else{ + assert( iOut==0 || iRowid2>iOut ); + fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); + iOut = iRowid2; + if( i1>=0 && iRowid1==iRowid2 ){ + fts5NextRowid(p1, &i1, &iRowid1); + } + fts5NextRowid(p2, &i2, &iRowid2); + } + } + + fts5BufferSwap(&out, p1); + fts5BufferFree(&out); +} + +typedef struct PrefixMerger PrefixMerger; +struct PrefixMerger { + Fts5DoclistIter iter; /* Doclist iterator */ + i64 iPos; /* For iterating through a position list */ + int iOff; + u8 *aPos; + PrefixMerger *pNext; /* Next in docid/poslist order */ +}; + +static void fts5PrefixMergerInsertByRowid( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iter.aPoslist ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iter.iRowid>(*pp)->iter.iRowid ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + +static void fts5PrefixMergerInsertByPosition( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iPos>=0 ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iPos>(*pp)->iPos ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + + +/* +** Array aBuf[] contains nBuf doclists. These are all merged in with the +** doclist in buffer p1. +*/ +static void fts5MergePrefixLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + int nBuf, /* Number of buffers in array aBuf[] */ + Fts5Buffer *aBuf /* Other lists to merge in */ +){ +#define fts5PrefixMergerNextPosition(p) \ + sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos); +#define FTS5_MERGE_NLIST 16 + PrefixMerger aMerger[FTS5_MERGE_NLIST]; + PrefixMerger *pHead = 0; + int i; + int nOut = 0; + Fts5Buffer out = {0, 0, 0}; + Fts5Buffer tmp = {0, 0, 0}; + i64 iLastRowid = 0; + + /* Initialize a doclist-iterator for each input buffer. Arrange them in + ** a linked-list starting at pHead in ascending order of rowid. Avoid + ** linking any iterators already at EOF into the linked list at all. */ + assert( nBuf+1<=sizeof(aMerger)/sizeof(aMerger[0]) ); + memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); + pHead = &aMerger[nBuf]; + fts5DoclistIterInit(p1, &pHead->iter); + for(i=0; in + 9 + 10*nBuf; + + /* The maximum size of the output is equal to the sum of the + ** input sizes + 1 varint (9 bytes). The extra varint is because if the + ** first rowid in one input is a large negative number, and the first in + ** the other a non-negative number, the delta for the non-negative + ** number will be larger on disk than the literal integer value + ** was. + ** + ** Or, if the input position-lists are corrupt, then the output might + ** include up to (nBuf+1) extra 10-byte positions created by interpreting -1 + ** (the value PoslistNext64() uses for EOF) as a position and appending + ** it to the output. This can happen at most once for each input + ** position-list, hence (nBuf+1) 10 byte paddings. */ + if( sqlite3Fts5BufferSize(&p->rc, &out, nOut) ) return; + + while( pHead ){ + fts5MergeAppendDocid(&out, iLastRowid, pHead->iter.iRowid); + + if( pHead->pNext && iLastRowid==pHead->pNext->iter.iRowid ){ + /* Merge data from two or more poslists */ + i64 iPrev = 0; + int nTmp = FTS5_DATA_ZERO_PADDING; + int nMerge = 0; + PrefixMerger *pSave = pHead; + PrefixMerger *pThis = 0; + int nTail = 0; + + pHead = 0; + while( pSave && pSave->iter.iRowid==iLastRowid ){ + PrefixMerger *pNext = pSave->pNext; + pSave->iOff = 0; + pSave->iPos = 0; + pSave->aPos = &pSave->iter.aPoslist[pSave->iter.nSize]; + fts5PrefixMergerNextPosition(pSave); + nTmp += pSave->iter.nPoslist + 10; + nMerge++; + fts5PrefixMergerInsertByPosition(&pHead, pSave); + pSave = pNext; + } + + if( pHead==0 || pHead->pNext==0 ){ + p->rc = FTS5_CORRUPT; + break; + } + + /* See the earlier comment in this function for an explanation of why + ** corrupt input position lists might cause the output to consume + ** at most nMerge*10 bytes of unexpected space. */ + if( sqlite3Fts5BufferSize(&p->rc, &tmp, nTmp+nMerge*10) ){ + break; + } + fts5BufferZero(&tmp); + + pThis = pHead; + pHead = pThis->pNext; + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + fts5PrefixMergerNextPosition(pThis); + fts5PrefixMergerInsertByPosition(&pHead, pThis); + + while( pHead->pNext ){ + pThis = pHead; + if( pThis->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + } + fts5PrefixMergerNextPosition(pThis); + pHead = pThis->pNext; + fts5PrefixMergerInsertByPosition(&pHead, pThis); + } + + if( pHead->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pHead->iPos); + } + nTail = pHead->iter.nPoslist - pHead->iOff; + + /* WRITEPOSLISTSIZE */ + assert( tmp.n+nTail<=nTmp ); + if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + break; + } + fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + if( nTail>0 ){ + fts5BufferSafeAppendBlob(&out, &pHead->aPos[pHead->iOff], nTail); + } + + pHead = pSave; + for(i=0; iiter.aPoslist && pX->iter.iRowid==iLastRowid ){ + fts5DoclistIterNext(&pX->iter); + fts5PrefixMergerInsertByRowid(&pHead, pX); + } + } + + }else{ + /* Copy poslist from pHead to output */ + PrefixMerger *pThis = pHead; + Fts5DoclistIter *pI = &pThis->iter; + fts5BufferSafeAppendBlob(&out, pI->aPoslist, pI->nPoslist+pI->nSize); + fts5DoclistIterNext(pI); + pHead = pThis->pNext; + fts5PrefixMergerInsertByRowid(&pHead, pThis); + } + } + + fts5BufferFree(p1); + fts5BufferFree(&tmp); + memset(&out.p[out.n], 0, FTS5_DATA_ZERO_PADDING); + *p1 = out; +} + +static void fts5SetupPrefixIter( + Fts5Index *p, /* Index to read from */ + int bDesc, /* True for "ORDER BY rowid DESC" */ + int iIdx, /* Index to scan for data */ + u8 *pToken, /* Buffer containing prefix to match */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset, /* Restrict matches to these columns */ + Fts5Iter **ppIter /* OUT: New iterator */ +){ + Fts5Structure *pStruct; + Fts5Buffer *aBuf; + int nBuf = 32; + int nMerge = 1; + + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + xMerge = fts5MergeRowidLists; + xAppend = fts5AppendRowid; + }else{ + nMerge = FTS5_MERGE_NLIST-1; + nBuf = nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ + xMerge = fts5MergePrefixLists; + xAppend = fts5AppendPoslist; + } + + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); + pStruct = fts5StructureRead(p); + + if( aBuf && pStruct ){ + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + int i; + i64 iLastRowid = 0; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + Fts5Data *pData; + Fts5Buffer doclist; + int bNewTerm = 1; + + memset(&doclist, 0, sizeof(doclist)); + if( iIdx!=0 ){ + int dummy = 0; + const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; + pToken[0] = FTS5_MAIN_PREFIX; + fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for(; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &dummy) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + p1->xSetOutputs(p1, pSeg); + if( p1->base.nData ){ + xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; + } + } + fts5MultiIterFree(p1); + } + + pToken[0] = FTS5_MAIN_PREFIX + iIdx; + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); + + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); + if( bNewTerm ){ + if( nTermbase.nData==0 ) continue; + + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ + for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=nBuf ); + for(iStore=i1; iStorebase.iRowid-iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; + } + + assert( (nBuf%nMerge)==0 ); + for(i=0; irc==SQLITE_OK ){ + xMerge(p, &doclist, nMerge, &aBuf[i]); + } + for(iFree=i; iFreep = (u8*)&pData[1]; + pData->nn = pData->szLeaf = doclist.n; + if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + fts5MultiIterNew2(p, pData, bDesc, ppIter); + } + fts5BufferFree(&doclist); + } + + fts5StructureRelease(pStruct); + sqlite3_free(aBuf); +} + + +/* +** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain +** to the document with rowid iRowid. +*/ +static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ + assert( p->rc==SQLITE_OK ); + + /* Allocate the hash table if it has not already been allocated */ + if( p->pHash==0 ){ + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); + } + + /* Flush the hash table to disk if required */ + if( iRowidiWriteRowid + || (iRowid==p->iWriteRowid && p->bDelete==0) + || (p->nPendingData > p->pConfig->nHashSize) + ){ + fts5IndexFlush(p); + } + + p->iWriteRowid = iRowid; + p->bDelete = bDelete; + return fts5IndexReturn(p); +} + +/* +** Commit data to disk. +*/ +static int sqlite3Fts5IndexSync(Fts5Index *p){ + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + sqlite3Fts5IndexCloseReader(p); + return fts5IndexReturn(p); +} + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +static int sqlite3Fts5IndexRollback(Fts5Index *p){ + sqlite3Fts5IndexCloseReader(p); + fts5IndexDiscardData(p); + fts5StructureInvalidate(p); + /* assert( p->rc==SQLITE_OK ); */ + return SQLITE_OK; +} + +/* +** The %_data table is completely empty when this function is called. This +** function populates it with the initial structure objects for each index, +** and the initial version of the "averages" record (a zero-byte blob). +*/ +static int sqlite3Fts5IndexReinit(Fts5Index *p){ + Fts5Structure s; + fts5StructureInvalidate(p); + fts5IndexDiscardData(p); + memset(&s, 0, sizeof(Fts5Structure)); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); + fts5StructureWrite(p, &s); + return fts5IndexReturn(p); +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying %_data table. +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +static int sqlite3Fts5IndexOpen( + Fts5Config *pConfig, + int bCreate, + Fts5Index **pp, + char **pzErr +){ + int rc = SQLITE_OK; + Fts5Index *p; /* New object */ + + *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); + if( rc==SQLITE_OK ){ + p->pConfig = pConfig; + p->nWorkUnit = FTS5_WORK_UNIT; + p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); + if( p->zDataTbl && bCreate ){ + rc = sqlite3Fts5CreateTable( + pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", + 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p); + } + } + } + + assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); + if( rc ){ + sqlite3Fts5IndexClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). +*/ +static int sqlite3Fts5IndexClose(Fts5Index *p){ + int rc = SQLITE_OK; + if( p ){ + assert( p->pReader==0 ); + fts5StructureInvalidate(p); + sqlite3_finalize(p->pWriter); + sqlite3_finalize(p->pDeleter); + sqlite3_finalize(p->pIdxWriter); + sqlite3_finalize(p->pIdxDeleter); + sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pDataVersion); + sqlite3Fts5HashFree(p->pHash); + sqlite3_free(p->zDataTbl); + sqlite3_free(p); + } + return rc; +} + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +static int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +){ + int n = 0; + int i; + for(i=0; i=nByte ) return 0; /* Input contains fewer than nChar chars */ + if( (unsigned char)p[n++]>=0xc0 ){ + if( n>=nByte ) return 0; + while( (p[n] & 0xc0)==0x80 ){ + n++; + if( n>=nByte ){ + if( i+1==nChar ) break; + return 0; + } + } + } + } + return n; +} + +/* +** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of +** unicode characters in the string. +*/ +static int fts5IndexCharlen(const char *pIn, int nIn){ + int nChar = 0; + int i = 0; + while( i=0xc0 ){ + while( i delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + int i; /* Used to iterate through indexes */ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = p->pConfig; + + assert( p->rc==SQLITE_OK ); + assert( (iCol<0)==p->bDelete ); + + /* Add the entry to the main terms index. */ + rc = sqlite3Fts5HashWrite( + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken + ); + + for(i=0; inPrefix && rc==SQLITE_OK; i++){ + const int nChar = pConfig->aPrefix[i]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5HashWrite(p->pHash, + p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, + nByte + ); + } + } + + return rc; +} + +/* +** Open a new iterator to iterate though all rowid that match the +** specified token or token prefix. +*/ +static int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +){ + Fts5Config *pConfig = p->pConfig; + Fts5Iter *pRet = 0; + Fts5Buffer buf = {0, 0, 0}; + + /* If the QUERY_SCAN flag is set, all other flags must be clear. */ + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); + + if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ + int iIdx = 0; /* Index to search */ + int iPrefixIdx = 0; /* +1 prefix index */ + if( nToken ) memcpy(&buf.p[1], pToken, nToken); + + /* Figure out which index to search and set iIdx accordingly. If this + ** is a prefix query for which there is no prefix index, set iIdx to + ** greater than pConfig->nPrefix to indicate that the query will be + ** satisfied by scanning multiple terms in the main index. + ** + ** If the QUERY_TEST_NOIDX flag was specified, then this must be a + ** prefix-query. Instead of using a prefix-index (if one exists), + ** evaluate the prefix query using the main FTS index. This is used + ** for internal sanity checking by the integrity-check in debug + ** mode only. */ +#ifdef SQLITE_DEBUG + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ + assert( flags & FTS5INDEX_QUERY_PREFIX ); + iIdx = 1+pConfig->nPrefix; + }else +#endif + if( flags & FTS5INDEX_QUERY_PREFIX ){ + int nChar = fts5IndexCharlen(pToken, nToken); + for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ + int nIdxChar = pConfig->aPrefix[iIdx-1]; + if( nIdxChar==nChar ) break; + if( nIdxChar==nChar+1 ) iPrefixIdx = iIdx; + } + } + + if( iIdx<=pConfig->nPrefix ){ + /* Straight index lookup */ + Fts5Structure *pStruct = fts5StructureRead(p); + buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); + if( pStruct ){ + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + pColset, buf.p, nToken+1, -1, 0, &pRet + ); + fts5StructureRelease(pStruct); + } + }else{ + /* Scan multiple terms in the main index */ + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; + fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); + assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } + } + + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pRet); + pRet = 0; + sqlite3Fts5IndexCloseReader(p); + } + + *ppIter = (Fts5IndexIter*)pRet; + sqlite3Fts5BufferFree(&buf); + } + return fts5IndexReturn(p); +} + +/* +** Return true if the iterator passed as the only argument is at EOF. +*/ +/* +** Move to the next matching rowid. +*/ +static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + assert( pIter->pIndex->rc==SQLITE_OK ); + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching term/rowid. Used by the fts5vocab module. +*/ +static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *p = pIter->pIndex; + + assert( pIter->pIndex->rc==SQLITE_OK ); + + fts5MultiIterNext(p, pIter, 0, 0); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; + pIter->base.bEof = 1; + } + } + + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching rowid that occurs at or after iMatch. The +** definition of "at or after" depends on whether this iterator iterates +** in ascending or descending rowid order. +*/ +static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Return the current term. +*/ +static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ + int n; + const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); + assert_nc( z || n<=1 ); + *pn = n-1; + return (z ? &z[1] : 0); +} + +/* +** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). +*/ +static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *pIndex = pIter->pIndex; + fts5MultiIterFree(pIter); + sqlite3Fts5IndexCloseReader(pIndex); + } +} + +/* +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. +*/ +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->nn ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; inn && iColp[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); + return fts5IndexReturn(p); +} + +/* +** Replace the current "averages" record with the contents of the buffer +** supplied as the second argument. +*/ +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ + assert( p->rc==SQLITE_OK ); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); + return fts5IndexReturn(p); +} + +/* +** Return the total number of blocks this module has read from the %_data +** table since it was created. +*/ +static int sqlite3Fts5IndexReads(Fts5Index *p){ + return p->nRead; +} + +/* +** Set the 32-bit cookie value stored at the start of all structure +** records to the value passed as the second argument. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ + int rc; /* Return code */ + Fts5Config *pConfig = p->pConfig; /* Configuration object */ + u8 aCookie[4]; /* Binary representation of iNew */ + sqlite3_blob *pBlob = 0; + + assert( p->rc==SQLITE_OK ); + sqlite3Fts5Put32(aCookie, iNew); + + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob + ); + if( rc==SQLITE_OK ){ + sqlite3_blob_write(pBlob, aCookie, 4, 0); + rc = sqlite3_blob_close(pBlob); + } + + return rc; +} + +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + fts5StructureRelease(pStruct); + return fts5IndexReturn(p); +} + + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the integrity-check +** functionality. +*/ + +/* +** Return a simple checksum value based on the arguments. +*/ +static u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +){ + int i; + u64 ret = iRowid; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); + for(i=0; iiLeaf ); + cksum1 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterPrev(p, pDlidx) + ){ + i64 iRowid = fts5DlidxIterRowid(pDlidx); + int pgno = fts5DlidxIterPgno(pDlidx); + assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); + cksum2 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; +} + +static int fts5QueryCksum( + Fts5Index *p, /* Fts5 index object */ + int iIdx, + const char *z, /* Index key to query for */ + int n, /* Size of index key in bytes */ + int flags, /* Flags for Fts5IndexQuery */ + u64 *pCksum /* IN/OUT: Checksum value */ +){ + int eDetail = p->pConfig->eDetail; + u64 cksum = *pCksum; + Fts5IndexIter *pIter = 0; + int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + + while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ + i64 rowid = pIter->iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); + }else{ + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNext(pIter); + } + } + sqlite3Fts5IterClose(pIter); + + *pCksum = cksum; + return rc; +} + +/* +** Check if buffer z[], size n bytes, contains as series of valid utf-8 +** encoded codepoints. If so, return 0. Otherwise, if the buffer does not +** contain valid utf-8, return non-zero. +*/ +static int fts5TestUtf8(const char *z, int n){ + int i = 0; + assert_nc( n>0 ); + while( i=n || (z[i+1] & 0xC0)!=0x80 ) return 1; + i += 2; + }else + if( (z[i] & 0xF0)==0xE0 ){ + if( i+2>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else + if( (z[i] & 0xF8)==0xF0 ){ + if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + if( (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else{ + return 1; + } + } + + return 0; +} + +/* +** This function is also purely an internal test. It does not contribute to +** FTS functionality, or even the integrity-check, in any way. +*/ +static void fts5TestTerm( + Fts5Index *p, + Fts5Buffer *pPrev, /* Previous term */ + const char *z, int n, /* Possibly new term to test */ + u64 expected, + u64 *pCksum +){ + int rc = p->rc; + if( pPrev->n==0 ){ + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + }else + if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + u64 cksum3 = *pCksum; + const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ + int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ + int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX); + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); + u64 ck1 = 0; + u64 ck2 = 0; + + /* Check that the results returned for ASC and DESC queries are + ** the same. If not, call this corruption. */ + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); + if( rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_DESC; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + } + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + + /* If this is a prefix query, check that the results returned if the + ** the index is disabled are the same. In both ASC and DESC order. + ** + ** This check may only be performed if the hash table is empty. This + ** is because the hash table only supports a single scan query at + ** a time, and the multi-iter loop from which this function is called + ** is already performing such a scan. + ** + ** Also only do this if buffer zTerm contains nTerm bytes of valid + ** utf-8. Otherwise, the last part of the buffer contents might contain + ** a non-utf-8 sequence that happens to be a prefix of a valid utf-8 + ** character stored in the main fts index, which will cause the + ** test to fail. */ + if( p->nPendingData==0 && 0==fts5TestUtf8(zTerm, nTerm) ){ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + } + + cksum3 ^= ck1; + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + + if( rc==SQLITE_OK && cksum3!=expected ){ + rc = FTS5_CORRUPT; + } + *pCksum = cksum3; + } + p->rc = rc; +} + +#else +# define fts5TestDlidxReverse(x,y,z) +# define fts5TestTerm(u,v,w,x,y,z) +#endif + +/* +** Check that: +** +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and +** contain zero terms. +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and +** contain zero rowids. +*/ +static void fts5IndexIntegrityCheckEmpty( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ + int iFirst, + int iNoRowid, + int iLast +){ + int i; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + if( pLeaf ){ + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + } +} + +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ + int iTermOff = 0; + int ii; + + Fts5Buffer buf1 = {0,0,0}; + Fts5Buffer buf2 = {0,0,0}; + + ii = pLeaf->szLeaf; + while( iinn && p->rc==SQLITE_OK ){ + int res; + int iOff; + int nIncr; + + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); + iTermOff += nIncr; + iOff = iTermOff; + + if( iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( iTermOff==nIncr ){ + int nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + }else{ + int nKeep, nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + buf1.n = nKeep; + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + + if( p->rc==SQLITE_OK ){ + res = fts5BufferCompare(&buf1, &buf2); + if( res<=0 ) p->rc = FTS5_CORRUPT; + } + } + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); + } + + fts5BufferFree(&buf1); + fts5BufferFree(&buf2); +} + +static void fts5IndexIntegrityCheckSegment( + Fts5Index *p, /* FTS5 backend object */ + Fts5StructureSegment *pSeg /* Segment to check internal consistency */ +){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + int rc2; + int iIdxPrevLeaf = pSeg->pgnoFirst-1; + int iDlidxPrevLeaf = pSeg->pgnoLast; + + if( pSeg->pgnoFirst==0 ) return; + + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d " + "ORDER BY 1, 2", + pConfig->zDb, pConfig->zName, pSeg->iSegid + )); + + /* Iterate through the b-tree hierarchy. */ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iRow; /* Rowid for this leaf */ + Fts5Data *pLeaf; /* Data for this leaf */ + + const char *zIdxTerm = (const char*)sqlite3_column_blob(pStmt, 1); + int nIdxTerm = sqlite3_column_bytes(pStmt, 1); + int iIdxLeaf = sqlite3_column_int(pStmt, 2); + int bIdxDlidx = sqlite3_column_int(pStmt, 3); + + /* If the leaf in question has already been trimmed from the segment, + ** ignore this b-tree entry. Otherwise, load it into memory. */ + if( iIdxLeafpgnoFirst ) continue; + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); + pLeaf = fts5LeafRead(p, iRow); + if( pLeaf==0 ) break; + + /* Check that the leaf contains at least one term, and that it is equal + ** to or larger than the split-key in zIdxTerm. Also check that if there + ** is also a rowid pointer within the leaf page header, it points to a + ** location before the term. */ + if( pLeaf->nn<=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + int iOff; /* Offset of first term on leaf */ + int iRowidOff; /* Offset of first rowid on leaf */ + int nTerm; /* Size of term on leaf in bytes */ + int res; /* Comparison of term and split-key */ + + iOff = fts5LeafFirstTermOff(pLeaf); + iRowidOff = fts5LeafFirstRowidOff(pLeaf); + if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); + res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + if( res==0 ) res = nTerm - nIdxTerm; + if( res<0 ) p->rc = FTS5_CORRUPT; + } + + fts5IntegrityCheckPgidx(p, pLeaf); + } + fts5DataRelease(pLeaf); + if( p->rc ) break; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + fts5IndexIntegrityCheckEmpty( + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 + ); + if( p->rc ) break; + + /* If there is a doclist-index, check that it looks right. */ + if( bIdxDlidx ){ + Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ + int iPrevLeaf = iIdxLeaf; + int iSegid = pSeg->iSegid; + int iPg = 0; + i64 iKey; + + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterNext(p, pDlidx) + ){ + + /* Check any rowid-less pages that occur before the current leaf. */ + for(iPg=iPrevLeaf+1; iPgrc = FTS5_CORRUPT; + fts5DataRelease(pLeaf); + } + } + iPrevLeaf = fts5DlidxIterPgno(pDlidx); + + /* Check that the leaf page indicated by the iterator really does + ** contain the rowid suggested by the same. */ + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); + pLeaf = fts5DataRead(p, iKey); + if( pLeaf ){ + i64 iRowid; + int iRowidOff = fts5LeafFirstRowidOff(pLeaf); + ASSERT_SZLEAF_OK(pLeaf); + if( iRowidOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); + if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + } + } + + iDlidxPrevLeaf = iPg; + fts5DlidxIterFree(pDlidx); + fts5TestDlidxReverse(p, iSegid, iIdxLeaf); + }else{ + iDlidxPrevLeaf = pSeg->pgnoLast; + /* TODO: Check there is no doclist index */ + } + + iIdxPrevLeaf = iIdxLeaf; + } + + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ +#if 0 + if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + } +#endif +} + + +/* +** Run internal checks to ensure that the FTS index (a) is internally +** consistent and (b) contains entries for which the XOR of the checksums +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. +** +** Return SQLITE_CORRUPT if any of the internal checks fail, or if the +** checksum does not match. Return SQLITE_OK if all checks pass without +** error, or some other SQLite error code if another error (e.g. OOM) +** occurs. +*/ +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ + int eDetail = p->pConfig->eDetail; + u64 cksum2 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ + Fts5Iter *pIter; /* Used to iterate through entire index */ + Fts5Structure *pStruct; /* Index structure */ + +#ifdef SQLITE_DEBUG + /* Used by extra internal tests only run if NDEBUG is not defined */ + u64 cksum3 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ +#endif + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + + /* Load the FTS index structure */ + pStruct = fts5StructureRead(p); + + /* Check that the internal nodes of each segment match the leaves */ + if( pStruct ){ + int iLvl, iSeg; + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5IndexIntegrityCheckSegment(p, pSeg); + } + } + } + + /* The cksum argument passed to this function is a checksum calculated + ** based on all expected entries in the FTS index (including prefix index + ** entries). This block checks that a checksum calculated based on the + ** actual contents of FTS index is identical. + ** + ** Two versions of the same checksum are calculated. The first (stack + ** variable cksum2) based on entries extracted from the full-text index + ** while doing a linear scan of each individual index in turn. + ** + ** As each term visited by the linear scans, a separate query for the + ** same term is performed. cksum3 is calculated based on the entries + ** extracted by these queries. + */ + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + int n; /* Size of term in bytes */ + i64 iPos = 0; /* Position read from poslist */ + int iOff = 0; /* Offset within poslist */ + i64 iRowid = fts5MultiIterRowid(pIter); + char *z = (char*)fts5MultiIterTerm(pIter, &n); + + /* If this is a new term, query for it. Update cksum3 with the results. */ + fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } + }else{ + poslist.n = 0; + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + fts5BufferAppendBlob(&p->rc, &poslist, 4, (const u8*)"\0\0\0\0"); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } + } + } + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + + fts5MultiIterFree(pIter); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; + + fts5StructureRelease(pStruct); +#ifdef SQLITE_DEBUG + fts5BufferFree(&term); +#endif + fts5BufferFree(&poslist); + return fts5IndexReturn(p); +} + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the fts5_decode() scalar +** function only. +*/ + +/* +** Decode a segment-data rowid from the %_data table. This function is +** the opposite of macro FTS5_SEGMENT_ROWID(). +*/ +static void fts5DecodeRowid( + i64 iRowid, /* Rowid from %_data table */ + int *piSegid, /* OUT: Segment id */ + int *pbDlidx, /* OUT: Dlidx flag */ + int *piHeight, /* OUT: Height */ + int *piPgno /* OUT: Page number */ +){ + *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); + iRowid >>= FTS5_DATA_PAGE_B; + + *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); + iRowid >>= FTS5_DATA_HEIGHT_B; + + *pbDlidx = (int)(iRowid & 0x0001); + iRowid >>= FTS5_DATA_DLI_B; + + *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); +} + +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ + int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ + fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + + if( iSegid==0 ){ + if( iKey==FTS5_AVERAGES_ROWID ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); + }else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); + } + } + else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + ); + } +} + +static void fts5DebugStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + Fts5Structure *p +){ + int iLvl, iSeg; /* Iterate through levels, segments */ + + for(iLvl=0; iLvlnLevel; iLvl++){ + Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg + ); + for(iSeg=0; iSegnSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); + } +} + +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This +** function appends a human-readable representation of the same object +** to the buffer passed as the second argument. +*/ +static void fts5DecodeStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int rc; /* Return code */ + Fts5Structure *p = 0; /* Decoded structure object */ + + rc = fts5StructureDecode(pBlob, nBlob, 0, &p); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + + fts5DebugStructure(pRc, pBuf, p); + fts5StructureRelease(p); +} + +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain an "averages" record. This function +** appends a human-readable representation of record to the buffer passed +** as the second argument. +*/ +static void fts5DecodeAverages( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int i = 0; + const char *zSpace = ""; + + while( i0 ){ + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } + while( iOff0 ) memcpy(a, aBlob, n); + + fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + + fts5DebugRowid(&rc, &s, iRowid); + if( bDlidx ){ + Fts5Data dlidx; + Fts5DlidxLvl lvl; + + dlidx.p = a; + dlidx.nn = n; + + memset(&lvl, 0, sizeof(Fts5DlidxLvl)); + lvl.pData = &dlidx; + lvl.iLeafPgno = iPgno; + + for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, + " %d(%lld)", lvl.iLeafPgno, lvl.iRowid + ); + } + }else if( iSegid==0 ){ + if( iRowid==FTS5_AVERAGES_ROWID ){ + fts5DecodeAverages(&rc, &s, a, n); + }else{ + fts5DecodeStructure(&rc, &s, a, n); + } + }else if( eDetailNone ){ + Fts5Buffer term; /* Current term read from page */ + int szLeaf; + int iPgidxOff = szLeaf = fts5GetU16(&a[2]); + int iTermOff; + int nKeep = 0; + int iOff; + + memset(&term, 0, sizeof(Fts5Buffer)); + + /* Decode any entries that occur before the first term. */ + if( szLeafn ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + } + + /* Decode the position list tail at the start of the page */ + if( iRowidOff!=0 ){ + iOff = iRowidOff; + }else if( iTermOff!=0 ){ + iOff = iTermOff; + }else{ + iOff = szLeaf; + } + if( iOff>n ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + fts5DecodePoslist(&rc, &s, &a[4], iOff-4); + + /* Decode any more doclist data that appears on the page before the + ** first term. */ + nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff; + if( nDoclist+iOff>n ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist); + + while( iPgidxOffszLeaf ){ + rc = FTS5_CORRUPT; + break; + } + + if( bFirst==0 ){ + iOff += fts5GetVarint32(&a[iOff], nByte); + if( nByte>term.n ){ + rc = FTS5_CORRUPT; + break; + } + term.n = nByte; + } + iOff += fts5GetVarint32(&a[iOff], nByte); + if( iOff+nByte>n ){ + rc = FTS5_CORRUPT; + break; + } + fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); + iOff += nByte; + + sqlite3Fts5BufferAppendPrintf( + &rc, &s, " term=%.*s", term.n, (const char*)term.p + ); + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff); + } + + fts5BufferFree(&term); + } + + decode_out: + sqlite3_free(a); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + fts5BufferFree(&s); +} + +/* +** The implementation of user-defined scalar function fts5_rowid(). +*/ +static void fts5RowidFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args (always 2) */ + sqlite3_value **apVal /* Function arguments */ +){ + const char *zArg; + if( nArg==0 ){ + sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); + }else{ + zArg = (const char*)sqlite3_value_text(apVal[0]); + if( 0==sqlite3_stricmp(zArg, "segment") ){ + i64 iRowid; + int segid, pgno; + if( nArg!=3 ){ + sqlite3_result_error(pCtx, + "should be: fts5_rowid('segment', segid, pgno))", -1 + ); + }else{ + segid = sqlite3_value_int(apVal[1]); + pgno = sqlite3_value_int(apVal[2]); + iRowid = FTS5_SEGMENT_ROWID(segid, pgno); + sqlite3_result_int64(pCtx, iRowid); + } + }else{ + sqlite3_result_error(pCtx, + "first arg to fts5_rowid() must be 'segment'" , -1 + ); + } + } +} + +/* +** This is called as part of registering the FTS5 module with database +** connection db. It registers several user-defined scalar functions useful +** with FTS5. +** +** If successful, SQLITE_OK is returned. If an error occurs, some other +** SQLite error code is returned instead. +*/ +static int sqlite3Fts5IndexInit(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 + ); + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_decode_none", 2, + SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 + ); + } + return rc; +} + + +static int sqlite3Fts5IndexReset(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} + +#line 1 "fts5_main.c" +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts5 code are activated - conditions that are +** only true if it is guaranteed that the fts5 database is not corrupt. +*/ +int sqlite3_fts5_may_be_corrupt = 1; + + +typedef struct Fts5Auxdata Fts5Auxdata; +typedef struct Fts5Auxiliary Fts5Auxiliary; +typedef struct Fts5Cursor Fts5Cursor; +typedef struct Fts5FullTable Fts5FullTable; +typedef struct Fts5Sorter Fts5Sorter; +typedef struct Fts5TokenizerModule Fts5TokenizerModule; + +/* +** NOTES ON TRANSACTIONS: +** +** SQLite invokes the following virtual table methods as transactions are +** opened and closed by the user: +** +** xBegin(): Start of a new transaction. +** xSync(): Initial part of two-phase commit. +** xCommit(): Final part of two-phase commit. +** xRollback(): Rollback the transaction. +** +** Anything that is required as part of a commit that may fail is performed +** in the xSync() callback. Current versions of SQLite ignore any errors +** returned by xCommit(). +** +** And as sub-transactions are opened/closed: +** +** xSavepoint(int S): Open savepoint S. +** xRelease(int S): Commit and close savepoint S. +** xRollbackTo(int S): Rollback to start of savepoint S. +** +** During a write-transaction the fts5_index.c module may cache some data +** in-memory. It is flushed to disk whenever xSync(), xRelease() or +** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() +** is called. +** +** Additionally, if SQLITE_DEBUG is defined, an instance of the following +** structure is used to record the current transaction state. This information +** is not required, but it is used in the assert() statements executed by +** function fts5CheckTransactionState() (see below). +*/ +struct Fts5TransactionState { + int eState; /* 0==closed, 1==open, 2==synced */ + int iSavepoint; /* Number of open savepoints (0 -> none) */ +}; + +/* +** A single object of this type is allocated when the FTS5 module is +** registered with a database handle. It is used to store pointers to +** all registered FTS5 extensions - tokenizers and auxiliary functions. +*/ +struct Fts5Global { + fts5_api api; /* User visible part of object (see fts5.h) */ + sqlite3 *db; /* Associated database connection */ + i64 iNextId; /* Used to allocate unique cursor ids */ + Fts5Auxiliary *pAux; /* First in list of all aux. functions */ + Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ + Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ + Fts5Cursor *pCsr; /* First in list of all open cursors */ +}; + +/* +** Each auxiliary function registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pAux list. +*/ +struct Fts5Auxiliary { + Fts5Global *pGlobal; /* Global context for this function */ + char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc; /* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5Auxiliary *pNext; /* Next registered auxiliary function */ +}; + +/* +** Each tokenizer module registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pTok list. +*/ +struct Fts5TokenizerModule { + char *zName; /* Name of tokenizer */ + void *pUserData; /* User pointer passed to xCreate() */ + fts5_tokenizer x; /* Tokenizer functions */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ +}; + +struct Fts5FullTable { + Fts5Table p; /* Public class members from fts5Int.h */ + Fts5Storage *pStorage; /* Document store */ + Fts5Global *pGlobal; /* Global (connection wide) data */ + Fts5Cursor *pSortCsr; /* Sort data from this cursor */ +#ifdef SQLITE_DEBUG + struct Fts5TransactionState ts; +#endif +}; + +struct Fts5MatchPhrase { + Fts5Buffer *pPoslist; /* Pointer to current poslist */ + int nTerm; /* Size of phrase in terms */ +}; + +/* +** pStmt: +** SELECT rowid, FROM ORDER BY +rank; +** +** aIdx[]: +** There is one entry in the aIdx[] array for each phrase in the query, +** the value of which is the offset within aPoslist[] following the last +** byte of the position list for the corresponding phrase. +*/ +struct Fts5Sorter { + sqlite3_stmt *pStmt; + i64 iRowid; /* Current rowid */ + const u8 *aPoslist; /* Position lists for current row */ + int nIdx; /* Number of entries in aIdx[] */ + int aIdx[1]; /* Offsets into aPoslist for current row */ +}; + + +/* +** Virtual-table cursor object. +** +** iSpecial: +** If this is a 'special' query (refer to function fts5SpecialMatch()), +** then this variable contains the result of the query. +** +** iFirstRowid, iLastRowid: +** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the +** cursor iterates in ascending order of rowids, iFirstRowid is the lower +** limit of rowids to return, and iLastRowid the upper. In other words, the +** WHERE clause in the user's query might have been: +** +** MATCH AND rowid BETWEEN $iFirstRowid AND $iLastRowid +** +** If the cursor iterates in descending order of rowid, iFirstRowid +** is the upper limit (i.e. the "first" rowid visited) and iLastRowid +** the lower. +*/ +struct Fts5Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ + int *aColumnSize; /* Values for xColumnSize() */ + i64 iCsrId; /* Cursor id */ + + /* Zero from this point onwards on cursor reset */ + int ePlan; /* FTS5_PLAN_XXX value */ + int bDesc; /* True for "ORDER BY rowid DESC" queries */ + i64 iFirstRowid; /* Return no rowids earlier than this */ + i64 iLastRowid; /* Return no rowids later than this */ + sqlite3_stmt *pStmt; /* Statement used to read %_content */ + Fts5Expr *pExpr; /* Expression for MATCH queries */ + Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ + int csrflags; /* Mask of cursor flags (see below) */ + i64 iSpecial; /* Result of special query */ + + /* "rank" function. Populated on demand from vtab.xColumn(). */ + char *zRank; /* Custom rank function */ + char *zRankArgs; /* Custom rank function args */ + Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ + int nRankArg; /* Number of trailing arguments for rank() */ + sqlite3_value **apRankArg; /* Array of trailing arguments */ + sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ + + /* Auxiliary data storage */ + Fts5Auxiliary *pAux; /* Currently executing extension function */ + Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ + + /* Cache used by auxiliary functions xInst() and xInstCount() */ + Fts5PoslistReader *aInstIter; /* One for each phrase */ + int nInstAlloc; /* Size of aInst[] array (entries / 3) */ + int nInstCount; /* Number of phrase instances */ + int *aInst; /* 3 integers per phrase instance */ +}; + +/* +** Bits that make up the "idxNum" parameter passed indirectly by +** xBestIndex() to xFilter(). +*/ +#define FTS5_BI_MATCH 0x0001 /* MATCH ? */ +#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */ +#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */ +#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */ +#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */ + +#define FTS5_BI_ORDER_RANK 0x0020 +#define FTS5_BI_ORDER_ROWID 0x0040 +#define FTS5_BI_ORDER_DESC 0x0080 + +/* +** Values for Fts5Cursor.csrflags +*/ +#define FTS5CSR_EOF 0x01 +#define FTS5CSR_REQUIRE_CONTENT 0x02 +#define FTS5CSR_REQUIRE_DOCSIZE 0x04 +#define FTS5CSR_REQUIRE_INST 0x08 +#define FTS5CSR_FREE_ZRANK 0x10 +#define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 + +#define BitFlagAllTest(x,y) (((x) & (y))==(y)) +#define BitFlagTest(x,y) (((x) & (y))!=0) + + +/* +** Macros to Set(), Clear() and Test() cursor flags. +*/ +#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag)) +#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag)) +#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag)) + +struct Fts5Auxdata { + Fts5Auxiliary *pAux; /* Extension to which this belongs */ + void *pPtr; /* Pointer value */ + void(*xDelete)(void*); /* Destructor */ + Fts5Auxdata *pNext; /* Next object in linked list */ +}; + +#ifdef SQLITE_DEBUG +#define FTS5_BEGIN 1 +#define FTS5_SYNC 2 +#define FTS5_COMMIT 3 +#define FTS5_ROLLBACK 4 +#define FTS5_SAVEPOINT 5 +#define FTS5_RELEASE 6 +#define FTS5_ROLLBACKTO 7 +static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ + switch( op ){ + case FTS5_BEGIN: + assert( p->ts.eState==0 ); + p->ts.eState = 1; + p->ts.iSavepoint = -1; + break; + + case FTS5_SYNC: + assert( p->ts.eState==1 ); + p->ts.eState = 2; + break; + + case FTS5_COMMIT: + assert( p->ts.eState==2 ); + p->ts.eState = 0; + break; + + case FTS5_ROLLBACK: + assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 ); + p->ts.eState = 0; + break; + + case FTS5_SAVEPOINT: + assert( p->ts.eState==1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint>=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint; + break; + + case FTS5_RELEASE: + assert( p->ts.eState==1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint<=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint-1; + break; + + case FTS5_ROLLBACKTO: + assert( p->ts.eState==1 ); + assert( iSavepoint>=-1 ); + /* The following assert() can fail if another vtab strikes an error + ** within an xSavepoint() call then SQLite calls xRollbackTo() - without + ** having called xSavepoint() on this vtab. */ + /* assert( iSavepoint<=p->ts.iSavepoint ); */ + p->ts.iSavepoint = iSavepoint; + break; + } +} +#else +# define fts5CheckTransactionState(x,y,z) +#endif + +/* +** Return true if pTab is a contentless table. +*/ +static int fts5IsContentless(Fts5FullTable *pTab){ + return pTab->p.pConfig->eContent==FTS5_CONTENT_NONE; +} + +/* +** Delete a virtual table handle allocated by fts5InitVtab(). +*/ +static void fts5FreeVtab(Fts5FullTable *pTab){ + if( pTab ){ + sqlite3Fts5IndexClose(pTab->p.pIndex); + sqlite3Fts5StorageClose(pTab->pStorage); + sqlite3Fts5ConfigFree(pTab->p.pConfig); + sqlite3_free(pTab); + } +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5DisconnectMethod(sqlite3_vtab *pVtab){ + fts5FreeVtab((Fts5FullTable*)pVtab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5DestroyMethod(sqlite3_vtab *pVtab){ + Fts5Table *pTab = (Fts5Table*)pVtab; + int rc = sqlite3Fts5DropAll(pTab->pConfig); + if( rc==SQLITE_OK ){ + fts5FreeVtab((Fts5FullTable*)pVtab); + } + return rc; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts5InitVtab( + int bCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts5Global *pGlobal = (Fts5Global*)pAux; + const char **azConfig = (const char**)argv; + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = 0; /* Results of parsing argc/argv */ + Fts5FullTable *pTab = 0; /* New virtual table object */ + + /* Allocate the new vtab object and parse the configuration */ + pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable)); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); + assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); + } + if( rc==SQLITE_OK ){ + pTab->p.pConfig = pConfig; + pTab->pGlobal = pGlobal; + } + + /* Open the index sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr); + } + + /* Open the storage sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageOpen( + pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr + ); + } + + /* Call sqlite3_declare_vtab() */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigDeclareVtab(pConfig); + } + + /* Load the initial configuration */ + if( rc==SQLITE_OK ){ + assert( pConfig->pzErrmsg==0 ); + pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + sqlite3Fts5IndexRollback(pTab->p.pIndex); + pConfig->pzErrmsg = 0; + } + + if( rc!=SQLITE_OK ){ + fts5FreeVtab(pTab); + pTab = 0; + }else if( bCreate ){ + fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); + } + *ppVTab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5InitVtab(). +*/ +static int fts5ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** The different query plans. +*/ +#define FTS5_PLAN_MATCH 1 /* ( MATCH ?) */ +#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */ +#define FTS5_PLAN_SPECIAL 3 /* An internal query */ +#define FTS5_PLAN_SORTED_MATCH 4 /* ( MATCH ? ORDER BY rank) */ +#define FTS5_PLAN_SCAN 5 /* No usable constraint */ +#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008012 ) +#endif + { + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +static int fts5UsePatternMatch( + Fts5Config *pConfig, + struct sqlite3_index_constraint *p +){ + assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); + assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); + if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + return 1; + } + if( pConfig->ePattern==FTS5_PATTERN_LIKE + && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) + ){ + return 1; + } + return 0; +} + +/* +** Implementation of the xBestIndex method for FTS5 tables. Within the +** WHERE constraint, it searches for the following: +** +** 1. A MATCH constraint against the table column. +** 2. A MATCH constraint against the "rank" column. +** 3. A MATCH constraint against some other column. +** 4. An == constraint against the rowid column. +** 5. A < or <= constraint against the rowid column. +** 6. A > or >= constraint against the rowid column. +** +** Within the ORDER BY, the following are supported: +** +** 5. ORDER BY rank [ASC|DESC] +** 6. ORDER BY rowid [ASC|DESC] +** +** Information for the xFilter call is passed via both the idxNum and +** idxStr variables. Specifically, idxNum is a bitmask of the following +** flags used to encode the ORDER BY clause: +** +** FTS5_BI_ORDER_RANK +** FTS5_BI_ORDER_ROWID +** FTS5_BI_ORDER_DESC +** +** idxStr is used to encode data from the WHERE clause. For each argument +** passed to the xFilter method, the following is appended to idxStr: +** +** Match against table column: "m" +** Match against rank column: "r" +** Match against other column: "M" +** LIKE against other column: "L" +** GLOB against other column: "G" +** Equality constraint against the rowid: "=" +** A < or <= against the rowid: "<" +** A > or >= against the rowid: ">" +** +** This function ensures that there is at most one "r" or "=". And that if +** there exists an "=" then there is no "<" or ">". +** +** Costs are assigned as follows: +** +** a) If an unusable MATCH operator is present in the WHERE clause, the +** cost is unconditionally set to 1e50 (a really big number). +** +** a) If a MATCH operator is present, the cost depends on the other +** constraints also present. As follows: +** +** * No other constraints: cost=1000.0 +** * One rowid range constraint: cost=750.0 +** * Both rowid range constraints: cost=500.0 +** * An == rowid constraint: cost=100.0 +** +** b) Otherwise, if there is no MATCH: +** +** * No other constraints: cost=1000000.0 +** * One rowid range constraint: cost=750000.0 +** * Both rowid range constraints: cost=250000.0 +** * An == rowid constraint: cost=10.0 +** +** Costs are not modified by the ORDER BY clause. +*/ +static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts5Table *pTab = (Fts5Table*)pVTab; + Fts5Config *pConfig = pTab->pConfig; + const int nCol = pConfig->nCol; + int idxFlags = 0; /* Parameter passed through to xFilter() */ + int i; + + char *idxStr; + int iIdxStr = 0; + int iCons = 0; + + int bSeenEq = 0; + int bSeenGt = 0; + int bSeenLt = 0; + int bSeenMatch = 0; + int bSeenRank = 0; + + + assert( SQLITE_INDEX_CONSTRAINT_EQbLock ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "recursively defined fts5 content table" + ); + return SQLITE_ERROR; + } + + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + if( idxStr==0 ) return SQLITE_NOMEM; + pInfo->idxStr = idxStr; + pInfo->needToFreeIdxStr = 1; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + int iCol = p->iColumn; + if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol) + ){ + /* A MATCH operator or equivalent */ + if( p->usable==0 || iCol<0 ){ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Set a prohibitively high cost. */ + pInfo->estimatedCost = 1e50; + assert( iIdxStr < pInfo->nConstraint*6 + 1 ); + idxStr[iIdxStr] = 0; + return SQLITE_OK; + }else{ + if( iCol==nCol+1 ){ + if( bSeenRank ) continue; + idxStr[iIdxStr++] = 'r'; + bSeenRank = 1; + }else if( iCol>=0 ){ + bSeenMatch = 1; + idxStr[iIdxStr++] = 'M'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + assert( idxStr[iIdxStr]=='\0' ); + } + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; + } + }else if( p->usable ){ + if( iCol>=0 && iColop==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB ); + idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + assert( idxStr[iIdxStr]=='\0' ); + }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ + idxStr[iIdxStr++] = '='; + bSeenEq = 1; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + } + } + } + + if( bSeenEq==0 ){ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->iColumn<0 && p->usable ){ + int op = p->op; + if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){ + if( bSeenLt ) continue; + idxStr[iIdxStr++] = '<'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenLt = 1; + }else + if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){ + if( bSeenGt ) continue; + idxStr[iIdxStr++] = '>'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenGt = 1; + } + } + } + } + idxStr[iIdxStr] = '\0'; + + /* Set idxFlags flags for the ORDER BY clause */ + if( pInfo->nOrderBy==1 ){ + int iSort = pInfo->aOrderBy[0].iColumn; + if( iSort==(pConfig->nCol+1) && bSeenMatch ){ + idxFlags |= FTS5_BI_ORDER_RANK; + }else if( iSort==-1 ){ + idxFlags |= FTS5_BI_ORDER_ROWID; + } + if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ + pInfo->orderByConsumed = 1; + if( pInfo->aOrderBy[0].desc ){ + idxFlags |= FTS5_BI_ORDER_DESC; + } + } + } + + /* Calculate the estimated cost based on the flags set in idxFlags. */ + if( bSeenEq ){ + pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; + if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); + }else if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; + }else{ + pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; + } + + pInfo->idxNum = idxFlags; + return SQLITE_OK; +} + +static int fts5NewTransaction(Fts5FullTable *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; + } + return sqlite3Fts5StorageReset(pTab->pStorage); +} + +/* +** Implementation of xOpen method. +*/ +static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)pVTab; + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = 0; /* New cursor object */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + int rc; /* Return code */ + + rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); + pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte); + if( pCsr ){ + Fts5Global *pGlobal = pTab->pGlobal; + memset(pCsr, 0, (size_t)nByte); + pCsr->aColumnSize = (int*)&pCsr[1]; + pCsr->pNext = pGlobal->pCsr; + pGlobal->pCsr = pCsr; + pCsr->iCsrId = ++pGlobal->iNextId; + }else{ + rc = SQLITE_NOMEM; + } + } + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static int fts5StmtType(Fts5Cursor *pCsr){ + if( pCsr->ePlan==FTS5_PLAN_SCAN ){ + return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; + } + return FTS5_STMT_LOOKUP; +} + +/* +** This function is called after the cursor passed as the only argument +** is moved to point at a different row. It clears all cached data +** specific to the previous row stored by the cursor object. +*/ +static void fts5CsrNewrow(Fts5Cursor *pCsr){ + CsrFlagSet(pCsr, + FTS5CSR_REQUIRE_CONTENT + | FTS5CSR_REQUIRE_DOCSIZE + | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST + ); +} + +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Auxdata *pData; + Fts5Auxdata *pNext; + + sqlite3_free(pCsr->aInstIter); + sqlite3_free(pCsr->aInst); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } + + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); +} + + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ + if( pCursor ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + Fts5Cursor **pp; + + fts5FreeCursorComponents(pCsr); + /* Remove the cursor from the Fts5Global.pCsr list */ + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); + *pp = pCsr->pNext; + + sqlite3_free(pCsr); + } + return SQLITE_OK; +} + +static int fts5SorterNext(Fts5Cursor *pCsr){ + Fts5Sorter *pSorter = pCsr->pSorter; + int rc; + + rc = sqlite3_step(pSorter->pStmt); + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_EOF); + }else if( rc==SQLITE_ROW ){ + const u8 *a; + const u8 *aBlob; + int nBlob; + int i; + int iOff = 0; + rc = SQLITE_OK; + + pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); + nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); + aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); + + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; + } + + fts5CsrNewrow(pCsr); + } + + return rc; +} + + +/* +** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors +** open on table pTab. +*/ +static void fts5TripCursors(Fts5FullTable *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && pCsr->base.pVtab==(sqlite3_vtab*)pTab + ){ + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); + } + } +} + +/* +** If the REQUIRE_RESEEK flag is set on the cursor passed as the first +** argument, close and reopen all Fts5IndexIter iterators that the cursor +** is using. Then attempt to move the cursor to a rowid equal to or laster +** (in the cursors sort order - ASC or DESC) than the current rowid. +** +** If the new rowid is not equal to the old, set output parameter *pbSkip +** to 1 before returning. Otherwise, leave it unchanged. +** +** Return SQLITE_OK if successful or if no reseek was required, or an +** error code if an error occurred. +*/ +static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ + int rc = SQLITE_OK; + assert( *pbSkip==0 ); + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int bDesc = pCsr->bDesc; + i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); + + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ + *pbSkip = 1; + } + + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); + fts5CsrNewrow(pCsr); + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + *pbSkip = 1; + } + } + return rc; +} + + +/* +** Advance the cursor to the next row in the table that matches the +** search criteria. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts5EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. +*/ +static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc; + + assert( (pCsr->ePlan<3)== + (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) + ); + assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + + if( pCsr->ePlan<3 ){ + int bSkip = 0; + if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; + rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); + CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr)); + fts5CsrNewrow(pCsr); + }else{ + switch( pCsr->ePlan ){ + case FTS5_PLAN_SPECIAL: { + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = SQLITE_OK; + break; + } + + case FTS5_PLAN_SORTED_MATCH: { + rc = fts5SorterNext(pCsr); + break; + } + + default: { + Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig; + pConfig->bLock++; + rc = sqlite3_step(pCsr->pStmt); + pConfig->bLock--; + if( rc!=SQLITE_ROW ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = sqlite3_reset(pCsr->pStmt); + if( rc!=SQLITE_OK ){ + pCursor->pVtab->zErrMsg = sqlite3_mprintf( + "%s", sqlite3_errmsg(pConfig->db) + ); + } + }else{ + rc = SQLITE_OK; + } + break; + } + } + } + + return rc; +} + + +static int fts5PrepareStatement( + sqlite3_stmt **ppStmt, + Fts5Config *pConfig, + const char *zFmt, + ... +){ + sqlite3_stmt *pRet = 0; + int rc; + char *zSql; + va_list ap; + + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pRet, 0); + if( rc!=SQLITE_OK ){ + *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + } + sqlite3_free(zSql); + } + + va_end(ap); + *ppStmt = pRet; + return rc; +} + +static int fts5CursorFirstSorted( + Fts5FullTable *pTab, + Fts5Cursor *pCsr, + int bDesc +){ + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Sorter *pSorter; + int nPhrase; + sqlite3_int64 nByte; + int rc; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); + if( pSorter==0 ) return SQLITE_NOMEM; + memset(pSorter, 0, (size_t)nByte); + pSorter->nIdx = nPhrase; + + /* TODO: It would be better to have some system for reusing statement + ** handles here, rather than preparing a new one for each query. But that + ** is not possible as SQLite reference counts the virtual table objects. + ** And since the statement required here reads from this very virtual + ** table, saving it creates a circular reference. + ** + ** If SQLite a built-in statement cache, this wouldn't be a problem. */ + rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s", + pConfig->zDb, pConfig->zName, zRank, pConfig->zName, + (zRankArgs ? ", " : ""), + (zRankArgs ? zRankArgs : ""), + bDesc ? "DESC" : "ASC" + ); + + pCsr->pSorter = pSorter; + if( rc==SQLITE_OK ){ + assert( pTab->pSortCsr==0 ); + pTab->pSortCsr = pCsr; + rc = fts5SorterNext(pCsr); + pTab->pSortCsr = 0; + } + + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + pCsr->pSorter = 0; + } + + return rc; +} + +static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ + int rc; + Fts5Expr *pExpr = pCsr->pExpr; + rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + if( sqlite3Fts5ExprEof(pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + } + fts5CsrNewrow(pCsr); + return rc; +} + +/* +** Process a "special" query. A special query is identified as one with a +** MATCH expression that begins with a '*' character. The remainder of +** the text passed to the MATCH operator are used as the special query +** parameters. +*/ +static int fts5SpecialMatch( + Fts5FullTable *pTab, + Fts5Cursor *pCsr, + const char *zQuery +){ + int rc = SQLITE_OK; /* Return code */ + const char *z = zQuery; /* Special query text */ + int n; /* Number of bytes in text at z */ + + while( z[0]==' ' ) z++; + for(n=0; z[n] && z[n]!=' '; n++); + + assert( pTab->p.base.zErrMsg==0 ); + pCsr->ePlan = FTS5_PLAN_SPECIAL; + + if( n==5 && 0==sqlite3_strnicmp("reads", z, n) ){ + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex); + } + else if( n==2 && 0==sqlite3_strnicmp("id", z, n) ){ + pCsr->iSpecial = pCsr->iCsrId; + } + else{ + /* An unrecognized directive. Return an error message. */ + pTab->p.base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Search for an auxiliary function named zName that can be used with table +** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary +** structure. Otherwise, if no such function exists, return NULL. +*/ +static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){ + Fts5Auxiliary *pAux; + + for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){ + if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + + +static int fts5FindRankFunction(Fts5Cursor *pCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + Fts5Auxiliary *pAux = 0; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + if( zRankArgs ){ + char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pStmt, 0); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_int64 nByte; + pCsr->nRankArg = sqlite3_column_count(pStmt); + nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; + pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte); + if( rc==SQLITE_OK ){ + int i; + for(i=0; inRankArg; i++){ + pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i); + } + } + pCsr->pRankArgStmt = pStmt; + }else{ + rc = sqlite3_finalize(pStmt); + assert( rc!=SQLITE_OK ); + } + } + } + } + + if( rc==SQLITE_OK ){ + pAux = fts5FindAuxiliary(pTab, zRank); + if( pAux==0 ){ + assert( pTab->p.base.zErrMsg==0 ); + pTab->p.base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); + rc = SQLITE_ERROR; + } + } + + pCsr->pRank = pAux; + return rc; +} + + +static int fts5CursorParseRank( + Fts5Config *pConfig, + Fts5Cursor *pCsr, + sqlite3_value *pRank +){ + int rc = SQLITE_OK; + if( pRank ){ + const char *z = (const char*)sqlite3_value_text(pRank); + char *zRank = 0; + char *zRankArgs = 0; + + if( z==0 ){ + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + } + if( rc==SQLITE_OK ){ + pCsr->zRank = zRank; + pCsr->zRankArgs = zRankArgs; + CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK); + }else if( rc==SQLITE_ERROR ){ + pCsr->base.pVtab->zErrMsg = sqlite3_mprintf( + "parse error in rank function: %s", z + ); + } + }else{ + if( pConfig->zRank ){ + pCsr->zRank = (char*)pConfig->zRank; + pCsr->zRankArgs = (char*)pConfig->zRankArgs; + }else{ + pCsr->zRank = (char*)FTS5_DEFAULT_RANK; + pCsr->zRankArgs = 0; + } + } + return rc; +} + +static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** There are three possible query strategies: +** +** 1. Full-text search using a MATCH operator. +** 2. A by-rowid lookup. +** 3. A full-table scan. +*/ +static int fts5FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; /* Error code */ + int bDesc; /* True if ORDER BY [rank|rowid] DESC */ + int bOrderByRank; /* True if ORDER BY rank */ + sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ + sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ + sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ + sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ + int iCol; /* Column on LHS of MATCH operator */ + char **pzErrmsg = pConfig->pzErrmsg; + int i; + int iIdxStr = 0; + Fts5Expr *pExpr = 0; + + if( pConfig->bLock ){ + pTab->p.base.zErrMsg = sqlite3_mprintf( + "recursively defined fts5 content table" + ); + return SQLITE_ERROR; + } + + if( pCsr->ePlan ){ + fts5FreeCursorComponents(pCsr); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); + } + + assert( pCsr->pStmt==0 ); + assert( pCsr->pExpr==0 ); + assert( pCsr->csrflags==0 ); + assert( pCsr->pRank==0 ); + assert( pCsr->zRank==0 ); + assert( pCsr->zRankArgs==0 ); + assert( pTab->pSortCsr==0 || nVal==0 ); + + assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg ); + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + + /* Decode the arguments passed through to this function. */ + for(i=0; i='0' && idxStr[iIdxStr]<='9' ); + + if( zText[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); + goto filter_out; + }else{ + char **pzErr = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + if( rc!=SQLITE_OK ) goto filter_out; + } + + break; + } + case 'L': + case 'G': { + int bGlob = (idxStr[iIdxStr-1]=='G'); + const char *zText = (const char*)sqlite3_value_text(apVal[i]); + iCol = 0; + do{ + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); + iIdxStr++; + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); + if( zText ){ + rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + if( rc!=SQLITE_OK ) goto filter_out; + break; + } + case '=': + pRowidEq = apVal[i]; + break; + case '<': + pRowidLe = apVal[i]; + break; + default: assert( idxStr[iIdxStr-1]=='>' ); + pRowidGe = apVal[i]; + break; + } + } + bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); + pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); + + /* Set the cursor upper and lower rowid limits. Only some strategies + ** actually use them. This is ok, as the xBestIndex() method leaves the + ** sqlite3_index_constraint.omit flag clear for range constraints + ** on the rowid field. */ + if( pRowidEq ){ + pRowidLe = pRowidGe = pRowidEq; + } + if( bDesc ){ + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + }else{ + pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + } + + if( pTab->pSortCsr ){ + /* If pSortCsr is non-NULL, then this call is being made as part of + ** processing for a "... MATCH ORDER BY rank" query (ePlan is + ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will + ** return results to the user for this query. The current cursor + ** (pCursor) is used to execute the query issued by function + ** fts5CursorFirstSorted() above. */ + assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); + assert( nVal==0 && bOrderByRank==0 && bDesc==0 ); + assert( pCsr->iLastRowid==LARGEST_INT64 ); + assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + if( pTab->pSortCsr->bDesc ){ + pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid; + }else{ + pCsr->iLastRowid = pTab->pSortCsr->iLastRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid; + } + pCsr->ePlan = FTS5_PLAN_SOURCE; + pCsr->pExpr = pTab->pSortCsr->pExpr; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + }else if( pCsr->pExpr ){ + rc = fts5CursorParseRank(pConfig, pCsr, pRank); + if( rc==SQLITE_OK ){ + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); + }else{ + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + } + } + }else if( pConfig->zContent==0 ){ + *pConfig->pzErrmsg = sqlite3_mprintf( + "%s: table does not support scanning", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup + ** by rowid (ePlan==FTS5_PLAN_ROWID). */ + pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg + ); + if( rc==SQLITE_OK ){ + if( pCsr->ePlan==FTS5_PLAN_ROWID ){ + sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq); + }else{ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); + sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); + } + rc = fts5NextMethod(pCursor); + } + } + + filter_out: + sqlite3Fts5ExprFree(pExpr); + pConfig->pzErrmsg = pzErrmsg; + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); +} + +/* +** Return the rowid that the cursor currently points to. +*/ +static i64 fts5CursorRowid(Fts5Cursor *pCsr){ + assert( pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + || pCsr->ePlan==FTS5_PLAN_SOURCE + ); + if( pCsr->pSorter ){ + return pCsr->pSorter->iRowid; + }else{ + return sqlite3Fts5ExprRowid(pCsr->pExpr); + } +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts5 +** exposes %_content.rowid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = pCsr->ePlan; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + switch( ePlan ){ + case FTS5_PLAN_SPECIAL: + *pRowid = 0; + break; + + case FTS5_PLAN_SOURCE: + case FTS5_PLAN_MATCH: + case FTS5_PLAN_SORTED_MATCH: + *pRowid = fts5CursorRowid(pCsr); + break; + + default: + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + break; + } + + return SQLITE_OK; +} + +/* +** If the cursor requires seeking (bSeekRequired flag is set), seek it. +** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. +** +** If argument bErrormsg is true and an error occurs, an error message may +** be left in sqlite3_vtab.zErrMsg. +*/ +static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ + int rc = SQLITE_OK; + + /* If the cursor does not yet have a statement handle, obtain one now. */ + if( pCsr->pStmt==0 ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int eStmt = fts5StmtType(pCsr); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0) + ); + assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 ); + assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); + } + + if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + assert( pCsr->pExpr ); + sqlite3_reset(pCsr->pStmt); + sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); + pTab->pConfig->bLock++; + rc = sqlite3_step(pCsr->pStmt); + pTab->pConfig->bLock--; + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + }else if( pTab->pConfig->pzErrmsg ){ + *pTab->pConfig->pzErrmsg = sqlite3_mprintf( + "%s", sqlite3_errmsg(pTab->pConfig->db) + ); + } + } + } + return rc; +} + +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + assert( p->p.base.zErrMsg==0 ); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** This function is called to handle an FTS INSERT command. In other words, +** an INSERT statement of the form: +** +** INSERT INTO fts(fts) VALUES($pCmd) +** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) +** +** Argument pVal is the value assigned to column "fts" by the INSERT +** statement. This function returns SQLITE_OK if successful, or an SQLite +** error code if an error occurs. +** +** The commands implemented by this function are documented in the "Special +** INSERT Directives" section of the documentation. It should be updated if +** more commands are added to this function. +*/ +static int fts5SpecialInsert( + Fts5FullTable *pTab, /* Fts5 table object */ + const char *zCmd, /* Text inserted into table-name column */ + sqlite3_value *pVal /* Value inserted into rank column */ +){ + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + int bError = 0; + + if( 0==sqlite3_stricmp("delete-all", zCmd) ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5SetVtabError(pTab, + "'delete-all' may only be used with a " + "contentless or external content fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ + if( pConfig->eContent==FTS5_CONTENT_NONE ){ + fts5SetVtabError(pTab, + "'rebuild' may not be used with a contentless fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageRebuild(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ + rc = sqlite3Fts5StorageOptimize(pTab->pStorage); + }else if( 0==sqlite3_stricmp("merge", zCmd) ){ + int nMerge = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); + }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ + int iArg = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg); +#ifdef SQLITE_DEBUG + }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ + pConfig->bPrefixIndex = sqlite3_value_int(pVal); +#endif + }else{ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); + } + if( rc==SQLITE_OK ){ + if( bError ){ + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0); + } + } + } + return rc; +} + +static int fts5SpecialDelete( + Fts5FullTable *pTab, + sqlite3_value **apVal +){ + int rc = SQLITE_OK; + int eType1 = sqlite3_value_type(apVal[1]); + if( eType1==SQLITE_INTEGER ){ + sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + } + return rc; +} + +static void fts5StorageInsert( + int *pRc, + Fts5FullTable *pTab, + sqlite3_value **apVal, + i64 *piRowid +){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); + } + *pRc = rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +** +** A delete specifies a single argument - the rowid of the row to remove. +** +** Update and insert operations pass: +** +** 1. The "old" rowid, or NULL. +** 2. The "new" rowid. +** 3. Values for each of the nCol matchable columns. +** 4. Values for the two hidden columns ( and "rank"). +*/ +static int fts5UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Config *pConfig = pTab->p.pConfig; + int eType0; /* value_type() of apVal[0] */ + int rc = SQLITE_OK; /* Return code */ + + /* A transaction must be open when this is called. */ + assert( pTab->ts.eState==1 ); + + assert( pVtab->zErrMsg==0 ); + assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER + || sqlite3_value_type(apVal[0])==SQLITE_NULL + ); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + + /* Put any active cursors into REQUIRE_SEEK state. */ + fts5TripCursors(pTab); + + eType0 = sqlite3_value_type(apVal[0]); + if( eType0==SQLITE_NULL + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL + ){ + /* A "special" INSERT op. These are handled separately. */ + const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && 0==sqlite3_stricmp("delete", z) + ){ + rc = fts5SpecialDelete(pTab, apVal); + }else{ + rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); + } + }else{ + /* A regular INSERT, UPDATE or DELETE statement. The trick here is that + ** any conflict on the rowid value must be detected before any + ** modifications are made to the database file. There are 4 cases: + ** + ** 1) DELETE + ** 2) UPDATE (rowid not modified) + ** 3) UPDATE (rowid modified) + ** 4) INSERT + ** + ** Cases 3 and 4 may violate the rowid constraint. + */ + int eConflict = SQLITE_ABORT; + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + eConflict = sqlite3_vtab_on_conflict(pConfig->db); + } + + assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); + assert( nArg!=1 || eType0==SQLITE_INTEGER ); + + /* Filter out attempts to run UPDATE or DELETE on contentless tables. + ** This is not suported. */ + if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ + pTab->p.base.zErrMsg = sqlite3_mprintf( + "cannot %s contentless fts5 table: %s", + (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName + ); + rc = SQLITE_ERROR; + } + + /* DELETE */ + else if( nArg==1 ){ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + } + + /* INSERT or UPDATE */ + else{ + int eType1 = sqlite3_value_numeric_type(apVal[1]); + + if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ + rc = SQLITE_MISMATCH; + } + + else if( eType0!=SQLITE_INTEGER ){ + /* If this is a REPLACE, first remove the current entry (if any) */ + if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + + /* UPDATE */ + else{ + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ + if( eType1==SQLITE_INTEGER && iOld!=iNew ){ + if( eConflict==SQLITE_REPLACE ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + }else{ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal,*pRowid); + } + } + }else{ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + } + } + } + + pTab->p.pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xSync() method. +*/ +static int fts5SyncMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + fts5CheckTransactionState(pTab, FTS5_SYNC, 0); + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + fts5TripCursors(pTab); + rc = sqlite3Fts5StorageSync(pTab->pStorage); + pTab->p.pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts5BeginMethod(sqlite3_vtab *pVtab){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + fts5NewTransaction((Fts5FullTable*)pVtab); + return SQLITE_OK; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts5SyncMethod(). +*/ +static int fts5CommitMethod(sqlite3_vtab *pVtab){ + UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts5RollbackMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + return rc; +} + +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); + +static void *fts5ApiUserData(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return pCsr->pAux->pUserData; +} + +static int fts5ApiColumnCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; +} + +static int fts5ApiColumnTotalSize( + Fts5Context *pCtx, + int iCol, + sqlite3_int64 *pnToken +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); +} + +static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); +} + +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); +} + +static int fts5ApiPhraseCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); +} + +static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); +} + +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) + || pCsr->ePlan==FTS5_PLAN_SPECIAL + ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + } + } + return rc; +} + +static int fts5CsrPoslist( + Fts5Cursor *pCsr, + int iPhrase, + const u8 **pa, + int *pn +){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + int rc = SQLITE_OK; + int bLive = (pCsr->pSorter==0); + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistPopulator *aPopulator; + int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); + if( aPopulator==0 ) rc = SQLITE_NOMEM; + for(i=0; inCol && rc==SQLITE_OK; i++){ + int n; const char *z; + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aPopulator, i, z, n + ); + } + } + sqlite3_free(aPopulator); + + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } + + return rc; +} + +/* +** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated +** correctly for the current view. Return SQLITE_OK if successful, or an +** SQLite error code otherwise. +*/ +static int fts5CacheInstArray(Fts5Cursor *pCsr){ + int rc = SQLITE_OK; + Fts5PoslistReader *aIter; /* One iterator for each phrase */ + int nIter; /* Number of iterators/phrases */ + int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol; + + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + if( pCsr->aInstIter==0 ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter; + pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); + } + aIter = pCsr->aInstIter; + + if( aIter ){ + int nInst = 0; /* Number instances seen so far */ + int i; + + /* Initialize all iterators */ + for(i=0; i=pCsr->nInstAlloc ){ + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc64( + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + }else{ + rc = SQLITE_NOMEM; + break; + } + } + + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + if( aInst[1]<0 || aInst[1]>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } + } + + pCsr->nInstCount = nInst; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); + } + return rc; +} + +static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ + *pnInst = pCsr->nInstCount; + } + return rc; +} + +static int fts5ApiInst( + Fts5Context *pCtx, + int iIdx, + int *piPhrase, + int *piCol, + int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; +#if 0 + }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 2]; + *piOff = -1; +#endif + }else{ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 1]; + *piOff = pCsr->aInst[iIdx*3 + 2]; + } + } + return rc; +} + +static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ + return fts5CursorRowid((Fts5Cursor*)pCtx); +} + +static int fts5ColumnSizeCb( + void *pContext, /* Pointer to int */ + int tflags, + const char *pUnused, /* Buffer containing token */ + int nUnused, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int *pCnt = (int*)pContext; + UNUSED_PARAM2(pUnused, nUnused); + UNUSED_PARAM2(iUnused1, iUnused2); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } + return SQLITE_OK; +} + +static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ + if( pConfig->bColumnsize ){ + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); + }else if( pConfig->zContent==0 ){ + int i; + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + pCsr->aColumnSize[i] = -1; + } + } + }else{ + int i; + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + const char *z; int n; + void *p = (void*)(&pCsr->aColumnSize[i]); + pCsr->aColumnSize[i] = 0; + rc = fts5ApiColumnText(pCtx, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize( + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + ); + } + } + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); + } + if( iCol<0 ){ + int i; + *pnToken = 0; + for(i=0; inCol; i++){ + *pnToken += pCsr->aColumnSize[i]; + } + }else if( iColnCol ){ + *pnToken = pCsr->aColumnSize[iCol]; + }else{ + *pnToken = 0; + rc = SQLITE_RANGE; + } + return rc; +} + +/* +** Implementation of the xSetAuxdata() method. +*/ +static int fts5ApiSetAuxdata( + Fts5Context *pCtx, /* Fts5 context */ + void *pPtr, /* Pointer to save as auxdata */ + void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + + /* Search through the cursors list of Fts5Auxdata objects for one that + ** corresponds to the currently executing auxiliary function. */ + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + if( pData->xDelete ){ + pData->xDelete(pData->pPtr); + } + }else{ + int rc = SQLITE_OK; + pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata)); + if( pData==0 ){ + if( xDelete ) xDelete(pPtr); + return rc; + } + pData->pAux = pCsr->pAux; + pData->pNext = pCsr->pAuxdata; + pCsr->pAuxdata = pData; + } + + pData->xDelete = xDelete; + pData->pPtr = pPtr; + return SQLITE_OK; +} + +static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + void *pRet = 0; + + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + pRet = pData->pPtr; + if( bClear ){ + pData->pPtr = 0; + pData->xDelete = 0; + } + } + + return pRet; +} + +static void fts5ApiPhraseNext( + Fts5Context *pUnused, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + UNUSED_PARAM(pUnused); + if( pIter->a>=pIter->b ){ + *piCol = -1; + *piOff = -1; + }else{ + int iVal; + pIter->a += fts5GetVarint32(pIter->a, iVal); + if( iVal==1 ){ + pIter->a += fts5GetVarint32(pIter->a, iVal); + *piCol = iVal; + *piOff = 0; + pIter->a += fts5GetVarint32(pIter->a, iVal); + } + *piOff += (iVal-2); + } +} + +static int fts5ApiPhraseFirst( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int n; + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + } + return rc; +} + +static void fts5ApiPhraseNextColumn( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + }else{ + int iIncr; + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); + *piCol += (iIncr-2); + } + }else{ + while( 1 ){ + int dummy; + if( pIter->a>=pIter->b ){ + *piCol = -1; + return; + } + if( pIter->a[0]==0x01 ) break; + pIter->a += fts5GetVarint32(pIter->a, dummy); + } + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + } +} + +static int fts5ApiPhraseFirstColumn( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int n; + if( pSorter ){ + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + n = pSorter->aIdx[iPhrase] - i1; + pIter->a = &pSorter->aPoslist[i1]; + }else{ + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + } + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + *piCol = 0; + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); + } + }else{ + int n; + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } + } + } + + return rc; +} + + +static int fts5ApiQueryPhrase(Fts5Context*, int, void*, + int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) +); + +static const Fts5ExtensionApi sFts5Api = { + 2, /* iVersion */ + fts5ApiUserData, + fts5ApiColumnCount, + fts5ApiRowCount, + fts5ApiColumnTotalSize, + fts5ApiTokenize, + fts5ApiPhraseCount, + fts5ApiPhraseSize, + fts5ApiInstCount, + fts5ApiInst, + fts5ApiRowid, + fts5ApiColumnText, + fts5ApiColumnSize, + fts5ApiQueryPhrase, + fts5ApiSetAuxdata, + fts5ApiGetAuxdata, + fts5ApiPhraseFirst, + fts5ApiPhraseNext, + fts5ApiPhraseFirstColumn, + fts5ApiPhraseNextColumn, +}; + +/* +** Implementation of API function xQueryPhrase(). +*/ +static int fts5ApiQueryPhrase( + Fts5Context *pCtx, + int iPhrase, + void *pUserData, + int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int rc; + Fts5Cursor *pNew = 0; + + rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); + if( rc==SQLITE_OK ){ + pNew->ePlan = FTS5_PLAN_MATCH; + pNew->iFirstRowid = SMALLEST_INT64; + pNew->iLastRowid = LARGEST_INT64; + pNew->base.pVtab = (sqlite3_vtab*)pTab; + rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr); + } + + if( rc==SQLITE_OK ){ + for(rc = fts5CursorFirst(pTab, pNew, 0); + rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; + rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) + ){ + rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + break; + } + } + } + + fts5CloseMethod((sqlite3_vtab_cursor*)pNew); + return rc; +} + +static void fts5ApiInvoke( + Fts5Auxiliary *pAux, + Fts5Cursor *pCsr, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( pCsr->pAux==0 ); + pCsr->pAux = pAux; + pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); + pCsr->pAux = 0; +} + +static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ + Fts5Cursor *pCsr; + for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->iCsrId==iCsrId ) break; + } + return pCsr; +} + +static void fts5ApiCallback( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + + Fts5Auxiliary *pAux; + Fts5Cursor *pCsr; + i64 iCsrId; + + assert( argc>=1 ); + pAux = (Fts5Auxiliary*)sqlite3_user_data(context); + iCsrId = sqlite3_value_int64(argv[0]); + + pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); + if( pCsr==0 || pCsr->ePlan==0 ){ + char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else{ + fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + } +} + + +/* +** Given cursor id iId, return a pointer to the corresponding Fts5Table +** object. Or NULL If the cursor id does not exist. +*/ +static Fts5Table *sqlite3Fts5TableFromCsrid( + Fts5Global *pGlobal, /* FTS5 global context for db handle */ + i64 iCsrId /* Id of cursor to find */ +){ + Fts5Cursor *pCsr; + pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); + if( pCsr ){ + return (Fts5Table*)pCsr->base.pVtab; + } + return 0; +} + +/* +** Return a "position-list blob" corresponding to the current position of +** cursor pCsr via sqlite3_result_blob(). A position-list blob contains +** the current position-list for each phrase in the query associated with +** cursor pCsr. +** +** A position-list blob begins with (nPhrase-1) varints, where nPhrase is +** the number of phrases in the query. Following the varints are the +** concatenated position lists for each phrase, in order. +** +** The first varint (if it exists) contains the size of the position list +** for phrase 0. The second (same disclaimer) contains the size of position +** list 1. And so on. There is no size field for the final position list, +** as it can be derived from the total size of the blob. +*/ +static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ + int i; + int rc = SQLITE_OK; + int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + Fts5Buffer val; + + memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; ipExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && ipExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; + } + + sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); + return rc; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +*/ +static int fts5ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + + if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ + if( iCol==pConfig->nCol ){ + sqlite3_result_int64(pCtx, pCsr->iSpecial); + } + }else + + if( iCol==pConfig->nCol ){ + /* User is requesting the value of the special column with the same name + ** as the table. Return the cursor integer id number. This value is only + ** useful in that it may be passed as the first argument to an FTS5 + ** auxiliary function. */ + sqlite3_result_int64(pCtx, pCsr->iCsrId); + }else if( iCol==pConfig->nCol+1 ){ + + /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ + fts5PoslistBlob(pCtx, pCsr); + }else if( + pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + ){ + if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ + fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); + } + } + }else if( !fts5IsContentless(pTab) ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + pConfig->pzErrmsg = 0; + } + return rc; +} + + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts5FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nUnused, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* OUT: User data for *pxFunc */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Auxiliary *pAux; + + UNUSED_PARAM(nUnused); + pAux = fts5FindAuxiliary(pTab, zName); + if( pAux ){ + *pxFunc = fts5ApiCallback; + *ppArg = (void*)pAux; + return 1; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + +/* +** Implementation of FTS5 xRename method. Rename an fts5 table. +*/ +static int fts5RenameMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + const char *zName /* New name of table */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + return sqlite3Fts5StorageRename(pTab->pStorage, zName); +} + +static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ + fts5TripCursors((Fts5FullTable*)pTab); + return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage); +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint); + return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint); + return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); + fts5TripCursors(pTab); + return sqlite3Fts5StorageRollback(pTab->pStorage); +} + +/* +** Register a new auxiliary function with global context pGlobal. +*/ +static int fts5CreateAux( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_extension_function xFunc, /* Aux. function implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = sqlite3_overload_function(pGlobal->db, zName, -1); + if( rc==SQLITE_OK ){ + Fts5Auxiliary *pAux; + sqlite3_int64 nName; /* Size of zName in bytes, including \0 */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5Auxiliary) + nName; + pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte); + if( pAux ){ + memset(pAux, 0, (size_t)nByte); + pAux->zFunc = (char*)&pAux[1]; + memcpy(pAux->zFunc, zName, nName); + pAux->pGlobal = pGlobal; + pAux->pUserData = pUserData; + pAux->xFunc = xFunc; + pAux->xDestroy = xDestroy; + pAux->pNext = pGlobal->pAux; + pGlobal->pAux = pAux; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + int rc = SQLITE_OK; + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); + if( pNew ){ + memset(pNew, 0, (size_t)nByte); + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->x = *pTokenizer; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + }else{ + rc = SQLITE_NOMEM; + } + + return rc; +} + +static Fts5TokenizerModule *fts5LocateTokenizer( + Fts5Global *pGlobal, + const char *zName +){ + Fts5TokenizerModule *pMod = 0; + + if( zName==0 ){ + pMod = pGlobal->pDfltTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; + } + } + + return pMod; +} + +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer() method. +*/ +static int fts5FindTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void **ppUserData, + fts5_tokenizer *pTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + *pTokenizer = pMod->x; + *ppUserData = pMod->pUserData; + }else{ + memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + rc = SQLITE_ERROR; + } + + return rc; +} + +static int sqlite3Fts5GetTokenizer( + Fts5Global *pGlobal, + const char **azArg, + int nArg, + Fts5Config *pConfig, + char **pzErr +){ + Fts5TokenizerModule *pMod; + int rc = SQLITE_OK; + + pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + if( pMod==0 ){ + assert( nArg>0 ); + rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + }else{ + rc = pMod->x.xCreate( + pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok + ); + pConfig->pTokApi = &pMod->x; + if( rc!=SQLITE_OK ){ + if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + }else{ + pConfig->ePattern = sqlite3Fts5TokenizerPattern( + pMod->x.xCreate, pConfig->pTok + ); + } + } + + if( rc!=SQLITE_OK ){ + pConfig->pTokApi = 0; + pConfig->pTok = 0; + } + + return rc; +} + +static void fts5ModuleDestroy(void *pCtx){ + Fts5TokenizerModule *pTok, *pNextTok; + Fts5Auxiliary *pAux, *pNextAux; + Fts5Global *pGlobal = (Fts5Global*)pCtx; + + for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){ + pNextAux = pAux->pNext; + if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData); + sqlite3_free(pAux); + } + + for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){ + pNextTok = pTok->pNext; + if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData); + sqlite3_free(pTok); + } + + sqlite3_free(pGlobal); +} + +static void fts5Fts5Func( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + fts5_api **ppApi; + UNUSED_PARAM(nArg); + assert( nArg==1 ); + ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr"); + if( ppApi ) *ppApi = &pGlobal->api; +} + +/* +** Implementation of fts5_source_id() function. +*/ +static void fts5SourceIdFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apUnused /* Function arguments */ +){ + assert( nArg==0 ); + UNUSED_PARAM2(nArg, apUnused); + sqlite3_result_text(pCtx, "fts5: 2021-04-19 18:32:05 1b256d97b553a9611efca188a3d995a2fff712759044ba480f9a0c9e98fae886", -1, SQLITE_TRANSIENT); +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts5ShadowName(const char *zName){ + static const char *azName[] = { + "config", "content", "data", "docsize", "idx" + }; + unsigned int i; + for(i=0; idb = db; + pGlobal->api.iVersion = 2; + pGlobal->api.xCreateFunction = fts5CreateAux; + pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; + pGlobal->api.xFindTokenizer = fts5FindTokenizer; + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); + if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 + ); + } + } + + /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file + ** fts5_test_mi.c is compiled and linked into the executable. And call + ** its entry point to enable the matchinfo() demo. */ +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + if( rc==SQLITE_OK ){ + extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); + rc = sqlite3Fts5TestRegisterMatchinfo(db); + } +#endif + + return rc; +} + +/* +** The following functions are used to register the module with SQLite. If +** this module is being built as part of the SQLite core (SQLITE_CORE is +** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. +** +** Or, if this module is being built as a loadable extension, +** sqlite3Fts5Init() is omitted and the two standard entry points +** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. +*/ +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts5_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} +#else +int sqlite3Fts5Init(sqlite3 *db){ + return fts5Init(db); +} +#endif + +#line 1 "fts5_storage.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +struct Fts5Storage { + Fts5Config *pConfig; + Fts5Index *pIndex; + int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ + i64 nTotalRow; /* Total number of rows in FTS table */ + i64 *aTotalSize; /* Total sizes of each column */ + sqlite3_stmt *aStmt[11]; +}; + + +#if FTS5_STMT_SCAN_ASC!=0 +# error "FTS5_STMT_SCAN_ASC mismatch" +#endif +#if FTS5_STMT_SCAN_DESC!=1 +# error "FTS5_STMT_SCAN_DESC mismatch" +#endif +#if FTS5_STMT_LOOKUP!=2 +# error "FTS5_STMT_LOOKUP mismatch" +#endif + +#define FTS5_STMT_INSERT_CONTENT 3 +#define FTS5_STMT_REPLACE_CONTENT 4 +#define FTS5_STMT_DELETE_CONTENT 5 +#define FTS5_STMT_REPLACE_DOCSIZE 6 +#define FTS5_STMT_DELETE_DOCSIZE 7 +#define FTS5_STMT_LOOKUP_DOCSIZE 8 +#define FTS5_STMT_REPLACE_CONFIG 9 +#define FTS5_STMT_SCAN 10 + +/* +** Prepare the two insert statements - Fts5Storage.pInsertContent and +** Fts5Storage.pInsertDocsize - if they have not already been prepared. +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageGetStmt( + Fts5Storage *p, /* Storage handle */ + int eStmt, /* FTS5_STMT_XXX constant */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ + char **pzErrMsg /* OUT: Error message (if any) */ +){ + int rc = SQLITE_OK; + + /* If there is no %_docsize table, there should be no requests for + ** statements to operate on it. */ + assert( p->pConfig->bColumnsize || ( + eStmt!=FTS5_STMT_REPLACE_DOCSIZE + && eStmt!=FTS5_STMT_DELETE_DOCSIZE + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE + )); + + assert( eStmt>=0 && eStmtaStmt) ); + if( p->aStmt[eStmt]==0 ){ + const char *azStmt[] = { + "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", + "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + + "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ + "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ + "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ + + "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + + "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ + "SELECT %s FROM %s AS T", /* SCAN */ + }; + Fts5Config *pC = p->pConfig; + char *zSql = 0; + + switch( eStmt ){ + case FTS5_STMT_SCAN: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent + ); + break; + + case FTS5_STMT_SCAN_ASC: + case FTS5_STMT_SCAN_DESC: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, + pC->zContent, pC->zContentRowid, pC->zContentRowid, + pC->zContentRowid + ); + break; + + case FTS5_STMT_LOOKUP: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent, pC->zContentRowid + ); + break; + + case FTS5_STMT_INSERT_CONTENT: + case FTS5_STMT_REPLACE_CONTENT: { + int nCol = pC->nCol + 1; + char *zBind; + int i; + + zBind = sqlite3_malloc64(1 + nCol*2); + if( zBind ){ + for(i=0; izDb, pC->zName, zBind); + sqlite3_free(zBind); + } + break; + } + + default: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); + break; + } + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int f = SQLITE_PREPARE_PERSISTENT; + if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + p->pConfig->bLock++; + rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); + p->pConfig->bLock--; + sqlite3_free(zSql); + if( rc!=SQLITE_OK && pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); + } + } + } + + *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); + return rc; +} + + +static int fts5ExecPrintf( + sqlite3 *db, + char **pzErr, + const char *zFormat, + ... +){ + int rc; + va_list ap; /* ... printf arguments */ + char *zSql; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, 0, 0, pzErr); + sqlite3_free(zSql); + } + + va_end(ap); + return rc; +} + +/* +** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error +** code otherwise. +*/ +static int sqlite3Fts5DropAll(Fts5Config *pConfig){ + int rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_data';" + "DROP TABLE IF EXISTS %Q.'%q_idx';" + "DROP TABLE IF EXISTS %Q.'%q_config';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName + ); + } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_content';", + pConfig->zDb, pConfig->zName + ); + } + return rc; +} + +static void fts5StorageRenameOne( + Fts5Config *pConfig, /* Current FTS5 configuration */ + int *pRc, /* IN/OUT: Error code */ + const char *zTail, /* Tail of table name e.g. "data", "config" */ + const char *zName /* New name of FTS5 table */ +){ + if( *pRc==SQLITE_OK ){ + *pRc = fts5ExecPrintf(pConfig->db, 0, + "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';", + pConfig->zDb, pConfig->zName, zTail, zName, zTail + ); + } +} + +static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ + Fts5Config *pConfig = pStorage->pConfig; + int rc = sqlite3Fts5StorageSync(pStorage); + + fts5StorageRenameOne(pConfig, &rc, "data", zName); + fts5StorageRenameOne(pConfig, &rc, "idx", zName); + fts5StorageRenameOne(pConfig, &rc, "config", zName); + if( pConfig->bColumnsize ){ + fts5StorageRenameOne(pConfig, &rc, "docsize", zName); + } + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5StorageRenameOne(pConfig, &rc, "content", zName); + } + return rc; +} + +/* +** Create the shadow table named zPost, with definition zDefn. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int sqlite3Fts5CreateTable( + Fts5Config *pConfig, /* FTS5 configuration */ + const char *zPost, /* Shadow table to create (e.g. "content") */ + const char *zDefn, /* Columns etc. for shadow table */ + int bWithout, /* True for without rowid */ + char **pzErr /* OUT: Error message */ +){ + int rc; + char *zErr = 0; + + rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", + pConfig->zDb, pConfig->zName, zPost, zDefn, +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID + bWithout?" WITHOUT ROWID": +#endif + "" + ); + if( zErr ){ + *pzErr = sqlite3_mprintf( + "fts5: error creating shadow table %q_%s: %s", + pConfig->zName, zPost, zErr + ); + sqlite3_free(zErr); + } + + return rc; +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying tables +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +static int sqlite3Fts5StorageOpen( + Fts5Config *pConfig, + Fts5Index *pIndex, + int bCreate, + Fts5Storage **pp, + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + Fts5Storage *p; /* New object */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ + *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + + memset(p, 0, (size_t)nByte); + p->aTotalSize = (i64*)&p[1]; + p->pConfig = pConfig; + p->pIndex = pIndex; + + if( bCreate ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + int nDefn = 32 + pConfig->nCol*10; + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10); + if( zDefn==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + int iOff; + sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); + iOff = (int)strlen(zDefn); + for(i=0; inCol; i++){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); + } + sqlite3_free(zDefn); + } + + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = sqlite3Fts5CreateTable( + pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable( + pConfig, "config", "k PRIMARY KEY, v", 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + } + + if( rc ){ + sqlite3Fts5StorageClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen(). +*/ +static int sqlite3Fts5StorageClose(Fts5Storage *p){ + int rc = SQLITE_OK; + if( p ){ + int i; + + /* Finalize all SQL statements */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + + sqlite3_free(p); + } + return rc; +} + +typedef struct Fts5InsertCtx Fts5InsertCtx; +struct Fts5InsertCtx { + Fts5Storage *pStorage; + int iCol; + int szCol; /* Size of column value in tokens */ +}; + +/* +** Tokenization callback used when inserting tokens into the FTS index. +*/ +static int fts5StorageInsertCallback( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; + Fts5Index *pIdx = pCtx->pStorage->pIndex; + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); +} + +/* +** If a row with rowid iDel is present in the %_content table, add the +** delete-markers to the FTS index necessary to delete it. Do not actually +** remove the %_content row at this time though. +*/ +static int fts5StorageDeleteFromIndex( + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal +){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ + int rc; /* Return code */ + int rc2; /* sqlite3_reset() return code */ + int iCol; + Fts5InsertCtx ctx; + + if( apVal==0 ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } + } + + ctx.pStorage = p; + ctx.iCol = -1; + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ + if( pConfig->abUnindexed[iCol-1]==0 ){ + const char *zText; + int nText; + if( pSeek ){ + zText = (const char*)sqlite3_column_text(pSeek, iCol); + nText = sqlite3_column_bytes(pSeek, iCol); + }else{ + zText = (const char*)sqlite3_value_text(apVal[iCol-1]); + nText = sqlite3_value_bytes(apVal[iCol-1]); + } + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + zText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + } + } + if( rc==SQLITE_OK && p->nTotalRow<1 ){ + rc = FTS5_CORRUPT; + }else{ + p->nTotalRow--; + } + + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + + +/* +** Insert a record into the %_docsize table. Specifically, do: +** +** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); +** +** If there is no %_docsize table (as happens if the columnsize=0 option +** is specified when the FTS5 table is created), this function is a no-op. +*/ +static int fts5StorageInsertDocsize( + Fts5Storage *p, /* Storage module to write to */ + i64 iRowid, /* id value */ + Fts5Buffer *pBuf /* sz value */ +){ + int rc = SQLITE_OK; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pReplace, 1, iRowid); + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } + } + return rc; +} + +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. If successful, and if +** argument bCache is true, set the p->bTotalsValid flag to indicate +** that the contents of aTotalSize[] and nTotalRow are valid until +** further notice. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ + int rc = SQLITE_OK; + if( p->bTotalsValid==0 ){ + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); + p->bTotalsValid = bCache; + } + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; iaTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + +/* +** Remove a row from the FTS table. +*/ +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ + Fts5Config *pConfig = p->pConfig; + int rc; + sqlite3_stmt *pDel = 0; + + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); + rc = fts5StorageLoadTotals(p, 1); + + /* Delete the index records */ + if( rc==SQLITE_OK ){ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + } + + /* Delete the %_docsize record */ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + /* Delete the %_content record */ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + return rc; +} + +/* +** Delete all entries in the FTS5 index. +*/ +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ + Fts5Config *pConfig = p->pConfig; + int rc; + + p->bTotalsValid = 0; + + /* Delete the contents of the %_data and %_docsize tables. */ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_idx';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName + ); + } + + /* Reinitialize the %_data table. This call creates the initial structure + ** and averages records. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p->pIndex); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + return rc; +} + +static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ + Fts5Buffer buf = {0,0,0}; + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pScan = 0; + Fts5InsertCtx ctx; + int rc, rc2; + + memset(&ctx, 0, sizeof(Fts5InsertCtx)); + ctx.pStorage = p; + rc = sqlite3Fts5StorageDeleteAll(p); + if( rc==SQLITE_OK ){ + rc = fts5StorageLoadTotals(p, 1); + } + + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ + i64 iRowid = sqlite3_column_int64(pScan, 0); + + sqlite3Fts5BufferZero(&buf); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1); + int nText = sqlite3_column_bytes(pScan, ctx.iCol+1); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + zText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + } + sqlite3_free(buf.p); + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + return rc; +} + +static int sqlite3Fts5StorageOptimize(Fts5Storage *p){ + return sqlite3Fts5IndexOptimize(p->pIndex); +} + +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ + return sqlite3Fts5IndexMerge(p->pIndex, nMerge); +} + +static int sqlite3Fts5StorageReset(Fts5Storage *p){ + return sqlite3Fts5IndexReset(p->pIndex); +} + +/* +** Allocate a new rowid. This is used for "external content" tables when +** a NULL value is inserted into the rowid column. The new rowid is allocated +** by inserting a dummy row into the %_docsize table. The dummy will be +** overwritten later. +** +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In +** this case the user is required to provide a rowid explicitly. +*/ +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ + int rc = SQLITE_MISMATCH; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pReplace, 1); + sqlite3_bind_null(pReplace, 2); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + if( rc==SQLITE_OK ){ + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); + } + } + return rc; +} + +/* +** Insert a new row into the FTS content table. +*/ +static int sqlite3Fts5StorageContentInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 *piRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; + + /* Insert the new row into the %_content table. */ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + *piRowid = sqlite3_value_int64(apVal[1]); + }else{ + rc = fts5StorageNewRowid(p, piRowid); + } + }else{ + sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ + int i; /* Counter variable */ + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); + for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + rc = sqlite3_bind_value(pInsert, i, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + *piRowid = sqlite3_last_insert_rowid(pConfig->db); + } + + return rc; +} + +/* +** Insert new entries into the FTS index and %_docsize table. +*/ +static int sqlite3Fts5StorageIndexInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 iRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; /* Return code */ + Fts5InsertCtx ctx; /* Tokenization callback context object */ + Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ + + memset(&buf, 0, sizeof(Fts5Buffer)); + ctx.pStorage = p; + rc = fts5StorageLoadTotals(p, 1); + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + } + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]); + int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + zText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + /* Write the %_docsize record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + sqlite3_free(buf.p); + + return rc; +} + +static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ + Fts5Config *pConfig = p->pConfig; + char *zSql; + int rc; + + zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", + pConfig->zDb, pConfig->zName, zSuffix + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pCnt = 0; + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pCnt) ){ + *pnRow = sqlite3_column_int64(pCnt, 0); + } + rc = sqlite3_finalize(pCnt); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** Context object used by sqlite3Fts5StorageIntegrity(). +*/ +typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; +struct Fts5IntegrityCtx { + i64 iRowid; + int iCol; + int szCol; + u64 cksum; + Fts5Termset *pTermset; + Fts5Config *pConfig; +}; + + +/* +** Tokenization callback used by integrity check. +*/ +static int fts5StorageIntegrityCallback( + void *pContext, /* Pointer to Fts5IntegrityCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; + Fts5Termset *pTermset = pCtx->pTermset; + int bPresent; + int ii; + int rc = SQLITE_OK; + int iPos; + int iCol; + + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + + switch( pCtx->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + iPos = pCtx->szCol-1; + iCol = pCtx->iCol; + break; + + case FTS5_DETAIL_COLUMNS: + iPos = pCtx->iCol; + iCol = 0; + break; + + default: + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); + iPos = 0; + iCol = 0; + break; + } + + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, 0, pToken, nToken + ); + } + + for(ii=0; rc==SQLITE_OK && iipConfig->nPrefix; ii++){ + const int nChar = pCtx->pConfig->aPrefix[ii]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); + if( bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte + ); + } + } + } + + return rc; +} + +/* +** Check that the contents of the FTS index match that of the %_content +** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return +** some other SQLite error code if an error occurs while attempting to +** determine this. +*/ +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; /* Return code */ + int *aColSize; /* Array of size pConfig->nCol */ + i64 *aTotalSize; /* Array of size pConfig->nCol */ + Fts5IntegrityCtx ctx; + sqlite3_stmt *pScan; + int bUseCksum; + + memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); + ctx.pConfig = p->pConfig; + aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64))); + if( !aTotalSize ) return SQLITE_NOMEM; + aColSize = (int*)&aTotalSize[pConfig->nCol]; + memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); + + bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL + || (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg) + ); + if( bUseCksum ){ + /* Generate the expected index checksum based on the contents of the + ** %_content table. This block stores the checksum in ctx.cksum. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + if( rc==SQLITE_OK ){ + int rc2; + while( SQLITE_ROW==sqlite3_step(pScan) ){ + int i; + ctx.iRowid = sqlite3_column_int64(pScan, 0); + ctx.szCol = 0; + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pConfig->abUnindexed[i] ) continue; + ctx.iCol = i; + ctx.szCol = 0; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + if( rc==SQLITE_OK ){ + const char *zText = (const char*)sqlite3_column_text(pScan, i+1); + int nText = sqlite3_column_bytes(pScan, i+1); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + zText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + } + if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } + } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + + if( rc!=SQLITE_OK ) break; + } + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* Test that the "totals" (sometimes called "averages") record looks Ok */ + if( rc==SQLITE_OK ){ + int i; + rc = fts5StorageLoadTotals(p, 0); + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; + } + } + + /* Check that the %_docsize and %_content tables contain the expected + ** number of rows. */ + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "content", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "docsize", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + } + + /* Pass the expected checksum down to the FTS index module. It will + ** verify, amongst other things, that it matches the checksum generated by + ** inspecting the index itself. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum); + } + + sqlite3_free(aTotalSize); + return rc; +} + +/* +** Obtain an SQLite statement handle that may be used to read data from the +** %_content table. +*/ +static int sqlite3Fts5StorageStmt( + Fts5Storage *p, + int eStmt, + sqlite3_stmt **pp, + char **pzErrMsg +){ + int rc; + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg); + if( rc==SQLITE_OK ){ + assert( p->aStmt[eStmt]==*pp ); + p->aStmt[eStmt] = 0; + } + return rc; +} + +/* +** Release an SQLite statement handle obtained via an earlier call to +** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function +** must match that passed to the sqlite3Fts5StorageStmt() call. +*/ +static void sqlite3Fts5StorageStmtRelease( + Fts5Storage *p, + int eStmt, + sqlite3_stmt *pStmt +){ + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + if( p->aStmt[eStmt]==0 ){ + sqlite3_reset(pStmt); + p->aStmt[eStmt] = pStmt; + }else{ + sqlite3_finalize(pStmt); + } +} + +static int fts5StorageDecodeSizeArray( + int *aCol, int nCol, /* Array to populate */ + const u8 *aBlob, int nBlob /* Record to read varints from */ +){ + int i; + int iOff = 0; + for(i=0; i=nBlob ) return 1; + iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); + } + return (iOff!=nBlob); +} + +/* +** Argument aCol points to an array of integers containing one entry for +** each table column. This function reads the %_docsize record for the +** specified rowid and populates aCol[] with the results. +** +** An SQLite error code is returned if an error occurs, or SQLITE_OK +** otherwise. +*/ +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + int bCorrupt = 1; + sqlite3_bind_int64(pLookup, 1, iRowid); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + const u8 *aBlob = sqlite3_column_blob(pLookup, 0); + int nBlob = sqlite3_column_bytes(pLookup, 0); + if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){ + bCorrupt = 0; + } + } + rc = sqlite3_reset(pLookup); + if( bCorrupt && rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + } + } + + return rc; +} + +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + *pnToken = 0; + if( iCol<0 ){ + int i; + for(i=0; ipConfig->nCol; i++){ + *pnToken += p->aTotalSize[i]; + } + }else if( iColpConfig->nCol ){ + *pnToken = p->aTotalSize[iCol]; + }else{ + rc = SQLITE_RANGE; + } + } + return rc; +} + +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + /* nTotalRow being zero does not necessarily indicate a corrupt + ** database - it might be that the FTS5 table really does contain zero + ** rows. However this function is only called from the xRowCount() API, + ** and there is no way for that API to be invoked if the table contains + ** no rows. Hence the FTS5_CORRUPT return. */ + *pnRow = p->nTotalRow; + if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT; + } + return rc; +} + +/* +** Flush any data currently held in-memory to disk. +*/ +static int sqlite3Fts5StorageSync(Fts5Storage *p){ + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); + if( p->bTotalsValid ){ + rc = fts5StorageSaveTotals(p); + p->bTotalsValid = 0; + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; +} + +static int sqlite3Fts5StorageRollback(Fts5Storage *p){ + p->bTotalsValid = 0; + return sqlite3Fts5IndexRollback(p->pIndex); +} + +static int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, + const char *z, + sqlite3_value *pVal, + int iVal +){ + sqlite3_stmt *pReplace = 0; + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); + if( pVal ){ + sqlite3_bind_value(pReplace, 2, pVal); + }else{ + sqlite3_bind_int(pReplace, 2, iVal); + } + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 1); + } + if( rc==SQLITE_OK && pVal ){ + int iNew = p->pConfig->iCookie + 1; + rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); + if( rc==SQLITE_OK ){ + p->pConfig->iCookie = iNew; + } + } + return rc; +} + +#line 1 "fts5_tokenize.c" +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + +/************************************************************************** +** Start of ascii tokenizer implementation. +*/ + +/* +** For tokenizers with no "unicode" modifier, the set of token characters +** is the same as the set of ASCII range alphanumeric characters. +*/ +static unsigned char aAsciiTokenChar[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */ +}; + +typedef struct AsciiTokenizer AsciiTokenizer; +struct AsciiTokenizer { + unsigned char aTokenChar[128]; +}; + +static void fts5AsciiAddExceptions( + AsciiTokenizer *p, + const char *zArg, + int bTokenChars +){ + int i; + for(i=0; zArg[i]; i++){ + if( (zArg[i] & 0x80)==0 ){ + p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars; + } + } +} + +/* +** Delete a "ascii" tokenizer. +*/ +static void fts5AsciiDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Create an "ascii" tokenizer. +*/ +static int fts5AsciiCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + AsciiTokenizer *p = 0; + UNUSED_PARAM(pUnused); + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = sqlite3_malloc(sizeof(AsciiTokenizer)); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + memset(p, 0, sizeof(AsciiTokenizer)); + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); + for(i=0; rc==SQLITE_OK && i='A' && c<='Z' ) c += 32; + aOut[i] = c; + } +} + +/* +** Tokenize some text using the ascii tokenizer. +*/ +static int fts5AsciiTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; + int rc = SQLITE_OK; + int ie; + int is = 0; + + char aFold[64]; + int nFold = sizeof(aFold); + char *pFold = aFold; + unsigned char *a = p->aTokenChar; + + UNUSED_PARAM(iUnused); + + while( isnFold ){ + if( pFold!=aFold ) sqlite3_free(pFold); + pFold = sqlite3_malloc64((sqlite3_int64)nByte*2); + if( pFold==0 ){ + rc = SQLITE_NOMEM; + break; + } + nFold = nByte*2; + } + asciiFold(pFold, &pText[is], nByte); + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, pFold, nByte, is, ie); + is = ie+1; + } + + if( pFold!=aFold ) sqlite3_free(pFold); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of unicode61 tokenizer implementation. +*/ + + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (unsigned char)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct Unicode61Tokenizer Unicode61Tokenizer; +struct Unicode61Tokenizer { + unsigned char aTokenChar[128]; /* ASCII range token characters */ + char *aFold; /* Buffer to fold text into */ + int nFold; /* Size of aFold[] in bytes */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int nException; + int *aiException; + + unsigned char aCategory[32]; /* True for token char categories */ +}; + +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 + +static int fts5UnicodeAddExceptions( + Unicode61Tokenizer *p, /* Tokenizer object */ + const char *z, /* Characters to treat as exceptions */ + int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ +){ + int rc = SQLITE_OK; + int n = (int)strlen(z); + int *aNew; + + if( n>0 ){ + aNew = (int*)sqlite3_realloc64(p->aiException, + (n+p->nException)*sizeof(int)); + if( aNew ){ + int nNew = p->nException; + const unsigned char *zCsr = (const unsigned char*)z; + const unsigned char *zTerm = (const unsigned char*)&z[n]; + while( zCsraTokenChar[iCode] = (unsigned char)bTokenChars; + }else{ + bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]; + assert( (bToken==0 || bToken==1) ); + assert( (bTokenChars==0 || bTokenChars==1) ); + if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ + int i; + for(i=0; iiCode ) break; + } + memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int)); + aNew[i] = iCode; + nNew++; + } + } + } + p->aiException = aNew; + p->nException = nNew; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Delete a "unicode61" tokenizer. +*/ +static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok; + sqlite3_free(p->aiException); + sqlite3_free(p->aFold); + sqlite3_free(p); + } + return; +} + +static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){ + const char *z = zCat; + + while( *z ){ + while( *z==' ' || *z=='\t' ) z++; + if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){ + return SQLITE_ERROR; + } + while( *z!=' ' && *z!='\t' && *z!='\0' ) z++; + } + + sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar); + return SQLITE_OK; +} + +/* +** Create a "unicode61" tokenizer. +*/ +static int fts5UnicodeCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; /* Return code */ + Unicode61Tokenizer *p = 0; /* New tokenizer object */ + + UNUSED_PARAM(pUnused); + + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + if( p ){ + const char *zCat = "L* N* Co"; + int i; + memset(p, 0, sizeof(Unicode61Tokenizer)); + + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; + p->nFold = 64; + p->aFold = sqlite3_malloc64(p->nFold * sizeof(char)); + if( p->aFold==0 ){ + rc = SQLITE_NOMEM; + } + + /* Search for a "categories" argument */ + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); + } + }else + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 1); + }else + if( 0==sqlite3_stricmp(azArg[i], "separators") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 0); + }else + if( 0==sqlite3_stricmp(azArg[i], "categories") ){ + /* no-op */ + }else{ + rc = SQLITE_ERROR; + } + } + + }else{ + rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + fts5UnicodeDelete((Fts5Tokenizer*)p); + p = 0; + } + *ppOut = (Fts5Tokenizer*)p; + } + return rc; +} + +/* +** Return true if, for the purposes of tokenizing with the tokenizer +** passed as the first argument, codepoint iCode is considered a token +** character (not a separator). +*/ +static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ + return ( + p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)] + ^ fts5UnicodeIsException(p, iCode) + ); +} + +static int fts5UnicodeTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; + int rc = SQLITE_OK; + unsigned char *a = p->aTokenChar; + + unsigned char *zTerm = (unsigned char*)&pText[nText]; + unsigned char *zCsr = (unsigned char *)pText; + + /* Output buffer */ + char *aFold = p->aFold; + int nFold = p->nFold; + const char *pEnd = &aFold[nFold-6]; + + UNUSED_PARAM(iUnused); + + /* Each iteration of this loop gobbles up a contiguous run of separators, + ** then the next token. */ + while( rc==SQLITE_OK ){ + u32 iCode; /* non-ASCII codepoint read from input */ + char *zOut = aFold; + int is; + int ie; + + /* Skip any separator characters. */ + while( 1 ){ + if( zCsr>=zTerm ) goto tokenize_done; + if( *zCsr & 0x80 ) { + /* A character outside of the ascii range. Skip past it if it is + ** a separator character. Or break out of the loop if it is not. */ + is = zCsr - (unsigned char*)pText; + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p, iCode) ){ + goto non_ascii_tokenchar; + } + }else{ + if( a[*zCsr] ){ + is = zCsr - (unsigned char*)pText; + goto ascii_tokenchar; + } + zCsr++; + } + } + + /* Run through the tokenchars. Fold them into the output buffer along + ** the way. */ + while( zCsrpEnd ){ + aFold = sqlite3_malloc64((sqlite3_int64)nFold*2); + if( aFold==0 ){ + rc = SQLITE_NOMEM; + goto tokenize_done; + } + zOut = &aFold[zOut - p->aFold]; + memcpy(aFold, p->aFold, nFold); + sqlite3_free(p->aFold); + p->aFold = aFold; + p->nFold = nFold = nFold*2; + pEnd = &aFold[nFold-6]; + } + + if( *zCsr & 0x80 ){ + /* An non-ascii-range character. Fold it into the output buffer if + ** it is a token character, or break out of the loop if it is not. */ + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ + non_ascii_tokenchar: + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); + if( iCode ) WRITE_UTF8(zOut, iCode); + }else{ + break; + } + }else if( a[*zCsr]==0 ){ + /* An ascii-range separator character. End of token. */ + break; + }else{ + ascii_tokenchar: + if( *zCsr>='A' && *zCsr<='Z' ){ + *zOut++ = *zCsr + 32; + }else{ + *zOut++ = *zCsr; + } + zCsr++; + } + ie = zCsr - (unsigned char*)pText; + } + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); + } + + tokenize_done: + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of porter stemmer implementation. +*/ + +/* Any tokens larger than this (in bytes) are passed through without +** stemming. */ +#define FTS5_PORTER_MAX_TOKEN 64 + +typedef struct PorterTokenizer PorterTokenizer; +struct PorterTokenizer { + fts5_tokenizer tokenizer; /* Parent tokenizer module */ + Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ + char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; +}; + +/* +** Delete a "porter" tokenizer. +*/ +static void fts5PorterDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + PorterTokenizer *p = (PorterTokenizer*)pTok; + if( p->pTokenizer ){ + p->tokenizer.xDelete(p->pTokenizer); + } + sqlite3_free(p); + } +} + +/* +** Create a "porter" tokenizer. +*/ +static int fts5PorterCreate( + void *pCtx, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + fts5_api *pApi = (fts5_api*)pCtx; + int rc = SQLITE_OK; + PorterTokenizer *pRet; + void *pUserdata = 0; + const char *zBase = "unicode61"; + + if( nArg>0 ){ + zBase = azArg[0]; + } + + pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + if( pRet ){ + memset(pRet, 0, sizeof(PorterTokenizer)); + rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + }else{ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + int nArg2 = (nArg>0 ? nArg-1 : 0); + const char **azArg2 = (nArg2 ? &azArg[1] : 0); + rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + } + + if( rc!=SQLITE_OK ){ + fts5PorterDelete((Fts5Tokenizer*)pRet); + pRet = 0; + } + *ppOut = (Fts5Tokenizer*)pRet; + return rc; +} + +typedef struct PorterContext PorterContext; +struct PorterContext { + void *pCtx; + int (*xToken)(void*, int, const char*, int, int, int); + char *aBuf; +}; + +typedef struct PorterRule PorterRule; +struct PorterRule { + const char *zSuffix; + int nSuffix; + int (*xCond)(char *zStem, int nStem); + const char *zOutput; + int nOutput; +}; + +#if 0 +static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){ + int ret = -1; + int nBuf = *pnBuf; + PorterRule *p; + + for(p=aRule; p->zSuffix; p++){ + assert( strlen(p->zSuffix)==p->nSuffix ); + assert( strlen(p->zOutput)==p->nOutput ); + if( nBufnSuffix ) continue; + if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break; + } + + if( p->zSuffix ){ + int nStem = nBuf - p->nSuffix; + if( p->xCond==0 || p->xCond(aBuf, nStem) ){ + memcpy(&aBuf[nStem], p->zOutput, p->nOutput); + *pnBuf = nStem + p->nOutput; + ret = p - aRule; + } + } + + return ret; +} +#endif + +static int fts5PorterIsVowel(char c, int bYIsVowel){ + return ( + c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y') + ); +} + +static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){ + int i; + int bCons = bPrevCons; + + /* Scan for a vowel */ + for(i=0; i 0) */ +static int fts5Porter_MGt0(char *zStem, int nStem){ + return !!fts5PorterGobbleVC(zStem, nStem, 0); +} + +/* porter rule condition: (m > 1) */ +static int fts5Porter_MGt1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (m = 1) */ +static int fts5Porter_MEq1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (*o) */ +static int fts5Porter_Ostar(char *zStem, int nStem){ + if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){ + return 0; + }else{ + int i; + int mask = 0; + int bCons = 0; + for(i=0; i 1 and (*S or *T)) */ +static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){ + assert( nStem>0 ); + return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') + && fts5Porter_MGt1(zStem, nStem); +} + +/* porter rule condition: (*v*) */ +static int fts5Porter_Vowel(char *zStem, int nStem){ + int i; + for(i=0; i0) ){ + return 1; + } + } + return 0; +} + + +/************************************************************************** +*************************************************************************** +** GENERATED CODE STARTS HERE (mkportersteps.tcl) +*/ + +static int fts5PorterStep4(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'e': + if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'l': + if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt1(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'o': + if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 's': + if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 't': + if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'z': + if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ate", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'b': + if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ble", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ize", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + } + return ret; +} + + +static int fts5PorterStep2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ate", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "tion", 4); + *pnBuf = nBuf - 6 + 4; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ence", 4); + *pnBuf = nBuf - 4 + 4; + } + }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ance", 4); + *pnBuf = nBuf - 4 + 4; + } + } + break; + + case 'e': + if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ize", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'g': + if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "log", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'l': + if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ble", 3); + *pnBuf = nBuf - 3 + 3; + } + }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "al", 2); + *pnBuf = nBuf - 4 + 2; + } + }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ent", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "e", 1); + *pnBuf = nBuf - 3 + 1; + } + }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ous", 3); + *pnBuf = nBuf - 5 + 3; + } + } + break; + + case 'o': + if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ize", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ate", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ate", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 's': + if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ive", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ful", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ous", 3); + *pnBuf = nBuf - 7 + 3; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ive", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "ble", 3); + *pnBuf = nBuf - 6 + 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep3(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ic", 2); + *pnBuf = nBuf - 4 + 2; + } + } + break; + + case 's': + if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + } + break; + + case 'z': + if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'e': + if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ee", 2); + *pnBuf = nBuf - 3 + 2; + } + }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_Vowel(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + ret = 1; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_Vowel(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + ret = 1; + } + } + break; + + } + return ret; +} + +/* +** GENERATED CODE ENDS HERE (mkportersteps.tcl) +*************************************************************************** +**************************************************************************/ + +static void fts5PorterStep1A(char *aBuf, int *pnBuf){ + int nBuf = *pnBuf; + if( aBuf[nBuf-1]=='s' ){ + if( aBuf[nBuf-2]=='e' ){ + if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') + || (nBuf>3 && aBuf[nBuf-3]=='i' ) + ){ + *pnBuf = nBuf-2; + }else{ + *pnBuf = nBuf-1; + } + } + else if( aBuf[nBuf-2]!='s' ){ + *pnBuf = nBuf-1; + } + } +} + +static int fts5PorterCb( + void *pCtx, + int tflags, + const char *pToken, + int nToken, + int iStart, + int iEnd +){ + PorterContext *p = (PorterContext*)pCtx; + + char *aBuf; + int nBuf; + + if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through; + aBuf = p->aBuf; + nBuf = nToken; + memcpy(aBuf, pToken, nBuf); + + /* Step 1. */ + fts5PorterStep1A(aBuf, &nBuf); + if( fts5PorterStep1B(aBuf, &nBuf) ){ + if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){ + char c = aBuf[nBuf-1]; + if( fts5PorterIsVowel(c, 0)==0 + && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] + ){ + nBuf--; + }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){ + aBuf[nBuf++] = 'e'; + } + } + } + + /* Step 1C. */ + if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){ + aBuf[nBuf-1] = 'i'; + } + + /* Steps 2 through 4. */ + fts5PorterStep2(aBuf, &nBuf); + fts5PorterStep3(aBuf, &nBuf); + fts5PorterStep4(aBuf, &nBuf); + + /* Step 5a. */ + assert( nBuf>0 ); + if( aBuf[nBuf-1]=='e' ){ + if( fts5Porter_MGt1(aBuf, nBuf-1) + || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1)) + ){ + nBuf--; + } + } + + /* Step 5b. */ + if( nBuf>1 && aBuf[nBuf-1]=='l' + && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) + ){ + nBuf--; + } + + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); + + pass_through: + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); +} + +/* +** Tokenize using the porter tokenizer. +*/ +static int fts5PorterTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + PorterTokenizer *p = (PorterTokenizer*)pTokenizer; + PorterContext sCtx; + sCtx.xToken = xToken; + sCtx.pCtx = pCtx; + sCtx.aBuf = p->aBuf; + return p->tokenizer.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + ); +} + +/************************************************************************** +** Start of trigram implementation. +*/ +typedef struct TrigramTokenizer TrigramTokenizer; +struct TrigramTokenizer { + int bFold; /* True to fold to lower-case */ +}; + +/* +** Free a trigram tokenizer. +*/ +static void fts5TriDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Allocate a trigram tokenizer. +*/ +static int fts5TriCreate( + void *pUnused, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + UNUSED_PARAM(pUnused); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + pNew->bFold = 1; + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else{ + rc = SQLITE_ERROR; + } + } + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } + } + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Trigram tokenizer tokenize routine. +*/ +static int fts5TriTokenize( + Fts5Tokenizer *pTok, + void *pCtx, + int unusedFlags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + int rc = SQLITE_OK; + char aBuf[32]; + const unsigned char *zIn = (const unsigned char*)pText; + const unsigned char *zEof = &zIn[nText]; + u32 iCode; + + UNUSED_PARAM(unusedFlags); + while( 1 ){ + char *zOut = aBuf; + int iStart = zIn - (const unsigned char*)pText; + const unsigned char *zNext; + + READ_UTF8(zIn, zEof, iCode); + if( iCode==0 ) break; + zNext = zIn; + if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); + WRITE_UTF8(zOut, iCode); + READ_UTF8(zIn, zEof, iCode); + if( iCode==0 ) break; + }else{ + break; + } + if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); + WRITE_UTF8(zOut, iCode); + READ_UTF8(zIn, zEof, iCode); + if( iCode==0 ) break; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); + WRITE_UTF8(zOut, iCode); + }else{ + break; + } + rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf); + if( rc!=SQLITE_OK ) break; + zIn = zNext; + } + + return rc; +} + +/* +** Argument xCreate is a pointer to a constructor function for a tokenizer. +** pTok is a tokenizer previously created using the same method. This function +** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB +** indicating the style of pattern matching that the tokenizer can support. +** In practice, this is: +** +** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB +** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE +** all other tokenizers - FTS5_PATTERN_NONE +*/ +static int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +){ + if( xCreate==fts5TriCreate ){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + } + return FTS5_PATTERN_NONE; +} + +/* +** Register all built-in tokenizers with FTS5. +*/ +static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ + struct BuiltinTokenizer { + const char *zName; + fts5_tokenizer x; + } aBuiltin[] = { + { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, + { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, + { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, + { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, + }; + + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && ixCreateTokenizer(pApi, + aBuiltin[i].zName, + (void*)pApi, + &aBuiltin[i].x, + 0 + ); + } + + return rc; +} + +#line 1 "fts5_unicode2.c" +/* +** 2012-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#include "libc/assert.h" + + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int fts5_remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +static int sqlite3Fts5UnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +static int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} + + +static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ + aArray[0] = 1; + switch( zCat[0] ){ + case 'C': + switch( zCat[1] ){ + case 'c': aArray[1] = 1; break; + case 'f': aArray[2] = 1; break; + case 'n': aArray[3] = 1; break; + case 's': aArray[4] = 1; break; + case 'o': aArray[31] = 1; break; + case '*': + aArray[1] = 1; + aArray[2] = 1; + aArray[3] = 1; + aArray[4] = 1; + aArray[31] = 1; + break; + default: return 1; } + break; + + case 'L': + switch( zCat[1] ){ + case 'l': aArray[5] = 1; break; + case 'm': aArray[6] = 1; break; + case 'o': aArray[7] = 1; break; + case 't': aArray[8] = 1; break; + case 'u': aArray[9] = 1; break; + case 'C': aArray[30] = 1; break; + case '*': + aArray[5] = 1; + aArray[6] = 1; + aArray[7] = 1; + aArray[8] = 1; + aArray[9] = 1; + aArray[30] = 1; + break; + default: return 1; } + break; + + case 'M': + switch( zCat[1] ){ + case 'c': aArray[10] = 1; break; + case 'e': aArray[11] = 1; break; + case 'n': aArray[12] = 1; break; + case '*': + aArray[10] = 1; + aArray[11] = 1; + aArray[12] = 1; + break; + default: return 1; } + break; + + case 'N': + switch( zCat[1] ){ + case 'd': aArray[13] = 1; break; + case 'l': aArray[14] = 1; break; + case 'o': aArray[15] = 1; break; + case '*': + aArray[13] = 1; + aArray[14] = 1; + aArray[15] = 1; + break; + default: return 1; } + break; + + case 'P': + switch( zCat[1] ){ + case 'c': aArray[16] = 1; break; + case 'd': aArray[17] = 1; break; + case 'e': aArray[18] = 1; break; + case 'f': aArray[19] = 1; break; + case 'i': aArray[20] = 1; break; + case 'o': aArray[21] = 1; break; + case 's': aArray[22] = 1; break; + case '*': + aArray[16] = 1; + aArray[17] = 1; + aArray[18] = 1; + aArray[19] = 1; + aArray[20] = 1; + aArray[21] = 1; + aArray[22] = 1; + break; + default: return 1; } + break; + + case 'S': + switch( zCat[1] ){ + case 'c': aArray[23] = 1; break; + case 'k': aArray[24] = 1; break; + case 'm': aArray[25] = 1; break; + case 'o': aArray[26] = 1; break; + case '*': + aArray[23] = 1; + aArray[24] = 1; + aArray[25] = 1; + aArray[26] = 1; + break; + default: return 1; } + break; + + case 'Z': + switch( zCat[1] ){ + case 'l': aArray[27] = 1; break; + case 'p': aArray[28] = 1; break; + case 's': aArray[29] = 1; break; + case '*': + aArray[27] = 1; + aArray[28] = 1; + aArray[29] = 1; + break; + default: return 1; } + break; + + } + return 0; +} + +static u16 aFts5UnicodeBlock[] = { + 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760, + 1760, 1760, 1760, 1760, 1760, 1763, 1765, + }; +static u16 aFts5UnicodeMap[] = { + 0, 32, 33, 36, 37, 40, 41, 42, 43, 44, + 45, 46, 48, 58, 60, 63, 65, 91, 92, 93, + 94, 95, 96, 97, 123, 124, 125, 126, 127, 160, + 161, 162, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 180, 181, 182, 184, 185, + 186, 187, 188, 191, 192, 215, 216, 223, 247, 248, + 256, 312, 313, 329, 330, 377, 383, 385, 387, 388, + 391, 394, 396, 398, 402, 403, 405, 406, 409, 412, + 414, 415, 417, 418, 423, 427, 428, 431, 434, 436, + 437, 440, 442, 443, 444, 446, 448, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 477, 478, 496, + 497, 498, 499, 500, 503, 505, 506, 564, 570, 572, + 573, 575, 577, 580, 583, 584, 592, 660, 661, 688, + 706, 710, 722, 736, 741, 748, 749, 750, 751, 768, + 880, 884, 885, 886, 890, 891, 894, 900, 902, 903, + 904, 908, 910, 912, 913, 931, 940, 975, 977, 978, + 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072, + 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369, + 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473, + 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545, + 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611, + 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758, + 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791, + 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984, + 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075, + 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210, + 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369, + 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416, + 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482, + 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519, + 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561, + 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622, + 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677, + 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749, + 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790, + 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869, + 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902, + 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947, + 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006, + 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059, + 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134, + 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199, + 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263, + 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302, + 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402, + 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458, + 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544, + 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655, + 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737, + 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773, + 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860, + 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896, + 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967, + 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046, + 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153, + 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190, + 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229, + 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295, + 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704, + 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888, + 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743, + 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906, + 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068, + 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107, + 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160, + 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435, + 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480, + 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679, + 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754, + 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824, + 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978, + 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043, + 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098, + 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168, + 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288, + 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406, + 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616, + 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976, + 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033, + 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118, + 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141, + 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184, + 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219, + 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249, + 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275, + 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317, + 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413, + 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459, + 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484, + 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500, + 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523, + 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597, + 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623, + 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972, + 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180, + 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665, + 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, + 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, + 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, + 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, + 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, + 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, + 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, + 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, + 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, + 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, + 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, + 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, + 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, + 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, + 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, + 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, + 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, + 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, + 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, + 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, + 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, + 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, + 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, + 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, + 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, + 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, + 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, + 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, + 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, + 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, + 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, + 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, + 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, + 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, + 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, + 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, + 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, + 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, + 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, + 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, + 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, + 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, + 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, + 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, + 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, + 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, + 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, + 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, + 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, + 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, + 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263, + 311, 320, 373, 377, 394, 400, 464, 509, 640, 672, + 768, 800, 816, 833, 834, 842, 896, 927, 928, 968, + 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103, + 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432, + 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623, + 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912, + 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178, + 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285, + 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416, + 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760, + 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216, + 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, + 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, + 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, + 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, + 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, + 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, + 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, + 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, + 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, + 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, + 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, + 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, + 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, + 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, + 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, + 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, + 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488, + 1, 32, 256, 0, 65533, + }; +static u16 aFts5UnicodeData[] = { + 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53, + 49, 85, 333, 85, 121, 85, 841, 54, 53, 50, + 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61, + 53, 151, 58, 53, 56, 58, 39, 52, 57, 34, + 58, 56, 58, 57, 79, 56, 37, 85, 56, 47, + 39, 51, 111, 53, 745, 57, 233, 773, 57, 261, + 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126, + 126, 73, 69, 137, 37, 73, 37, 105, 101, 73, + 37, 73, 37, 190, 158, 37, 126, 126, 73, 37, + 126, 94, 37, 39, 94, 69, 135, 41, 40, 37, + 41, 40, 37, 41, 40, 37, 542, 37, 606, 37, + 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37, + 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582, + 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596, + 158, 38, 56, 94, 38, 101, 53, 88, 41, 53, + 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105, + 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541, + 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38, + 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76, + 53, 76, 53, 44, 871, 103, 85, 162, 121, 85, + 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684, + 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58, + 204, 70, 76, 58, 140, 71, 333, 103, 90, 39, + 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333, + 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300, + 38, 108, 38, 172, 501, 807, 108, 53, 39, 359, + 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268, + 138, 44, 74, 39, 236, 327, 76, 85, 333, 53, + 38, 199, 231, 44, 74, 263, 71, 711, 231, 39, + 135, 44, 39, 106, 140, 74, 74, 44, 39, 42, + 71, 103, 76, 333, 71, 87, 207, 58, 55, 76, + 42, 199, 71, 711, 231, 71, 71, 71, 44, 106, + 76, 76, 108, 44, 135, 39, 333, 76, 103, 44, + 76, 42, 295, 103, 711, 231, 71, 167, 44, 39, + 106, 172, 76, 42, 74, 44, 39, 71, 76, 333, + 53, 55, 44, 74, 263, 71, 711, 231, 71, 167, + 44, 39, 42, 44, 42, 140, 74, 74, 44, 44, + 42, 71, 103, 76, 333, 58, 39, 207, 44, 39, + 199, 103, 135, 71, 39, 71, 71, 103, 391, 74, + 44, 74, 106, 106, 44, 39, 42, 333, 111, 218, + 55, 58, 106, 263, 103, 743, 327, 167, 39, 108, + 138, 108, 140, 76, 71, 71, 76, 333, 239, 58, + 74, 263, 103, 743, 327, 167, 44, 39, 42, 44, + 170, 44, 74, 74, 76, 74, 39, 71, 76, 333, + 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106, + 44, 39, 42, 71, 76, 333, 207, 58, 199, 74, + 583, 775, 295, 39, 231, 44, 106, 108, 44, 266, + 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268, + 53, 333, 85, 71, 39, 71, 39, 39, 135, 231, + 103, 39, 39, 71, 135, 44, 71, 204, 76, 39, + 167, 38, 204, 333, 135, 39, 122, 501, 58, 53, + 122, 76, 218, 333, 335, 58, 44, 58, 44, 58, + 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42, + 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90, + 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76, + 74, 76, 39, 333, 213, 199, 74, 76, 135, 108, + 39, 106, 71, 234, 103, 140, 423, 44, 74, 76, + 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41, + 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319, + 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151, + 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551, + 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108, + 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76, + 42, 236, 266, 44, 74, 364, 117, 38, 117, 55, + 39, 44, 333, 335, 213, 49, 149, 108, 61, 333, + 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138, + 76, 106, 74, 44, 202, 108, 58, 85, 333, 967, + 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76, + 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44, + 74, 268, 202, 332, 44, 333, 333, 245, 38, 213, + 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44, + 74, 231, 333, 245, 346, 300, 314, 76, 42, 967, + 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415, + 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159, + 266, 268, 74, 76, 181, 333, 103, 333, 967, 198, + 85, 277, 108, 53, 428, 42, 236, 135, 44, 135, + 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260, + 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265, + 261, 265, 197, 201, 261, 41, 41, 41, 94, 229, + 265, 453, 261, 264, 261, 264, 261, 264, 165, 69, + 137, 40, 56, 37, 120, 101, 69, 137, 40, 120, + 133, 69, 137, 120, 261, 169, 120, 101, 69, 137, + 40, 88, 381, 162, 209, 85, 52, 51, 54, 84, + 51, 54, 52, 277, 59, 60, 162, 61, 309, 52, + 51, 149, 80, 117, 57, 54, 50, 373, 57, 53, + 48, 341, 61, 162, 194, 47, 38, 207, 121, 54, + 50, 38, 335, 121, 54, 50, 422, 855, 428, 139, + 44, 107, 396, 90, 41, 154, 41, 90, 37, 105, + 69, 105, 37, 58, 41, 90, 57, 169, 218, 41, + 58, 41, 58, 41, 58, 137, 58, 37, 137, 37, + 135, 37, 90, 69, 73, 185, 94, 101, 58, 57, + 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186, + 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018, + 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666, + 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217, + 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57, + 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281, + 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69, + 254, 105, 37, 94, 37, 94, 165, 70, 105, 37, + 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221, + 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231, + 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52, + 51, 117, 52, 51, 53, 52, 51, 309, 49, 85, + 49, 53, 52, 51, 85, 52, 51, 54, 50, 54, + 50, 54, 50, 54, 50, 181, 38, 341, 81, 858, + 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 90, + 54, 50, 54, 50, 54, 50, 54, 50, 49, 54, + 82, 58, 302, 140, 74, 49, 166, 90, 110, 38, + 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887, + 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178, + 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274, + 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38, + 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333, + 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798, + 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69, + 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382, + 70, 37, 231, 44, 103, 44, 135, 44, 743, 74, + 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74, + 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333, + 903, 268, 85, 743, 364, 74, 53, 935, 108, 42, + 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333, + 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263, + 44, 42, 333, 149, 519, 38, 199, 122, 39, 42, + 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44, + 39, 71, 38, 85, 359, 42, 76, 74, 85, 39, + 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74, + 44, 74, 44, 74, 53, 42, 44, 333, 39, 39, + 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399, + 229, 165, 39, 44, 327, 57, 423, 167, 39, 71, + 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55, + 58, 524, 245, 54, 50, 53, 236, 53, 81, 80, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 85, 54, 50, 149, + 112, 117, 149, 49, 54, 50, 54, 50, 54, 50, + 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34, + 117, 55, 117, 54, 50, 53, 57, 53, 49, 85, + 333, 85, 121, 85, 841, 54, 53, 50, 56, 48, + 56, 837, 54, 57, 50, 57, 54, 50, 53, 54, + 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199, + 103, 87, 57, 56, 58, 87, 58, 153, 90, 98, + 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455, + 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575, + 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263, + 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71, + 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799, + 71, 39, 108, 76, 140, 135, 103, 871, 108, 44, + 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615, + 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655, + 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34, + 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149, + 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383, + 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182, + 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898, + 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236, + 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837, + 841, 229, 581, 841, 837, 41, 73, 41, 73, 137, + 265, 133, 37, 229, 357, 841, 837, 73, 137, 265, + 233, 837, 73, 137, 169, 41, 233, 837, 841, 837, + 841, 837, 841, 837, 841, 837, 841, 837, 841, 901, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71, + 39, 39, 327, 135, 39, 39, 39, 39, 39, 39, + 103, 71, 39, 39, 39, 39, 39, 39, 71, 39, + 135, 231, 135, 135, 39, 327, 551, 103, 167, 551, + 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946, + 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210, + 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266, + 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351, + 34, 3074, 7692, 63, 63, + }; + +static int sqlite3Fts5UnicodeCategory(u32 iCode) { + int iRes = -1; + int iHi; + int iLo; + int ret; + u16 iKey; + + if( iCode>=(1<<20) ){ + return 0; + } + iLo = aFts5UnicodeBlock[(iCode>>16)]; + iHi = aFts5UnicodeBlock[1+(iCode>>16)]; + iKey = (iCode & 0xFFFF); + while( iHi>iLo ){ + int iTest = (iHi + iLo) / 2; + assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest; + } + } + + if( iRes<0 ) return 0; + if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0; + ret = aFts5UnicodeData[iRes] & 0x1F; + if( ret!=30 ) return ret; + return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9; +} + +static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ + int i = 0; + int iTbl = 0; + while( i<128 ){ + int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; + int n = (aFts5UnicodeData[iTbl] >> 5) + i; + for(; i<128 && i3 && n<=9 ); + return n; + } +} + + +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + *v = a; + return 1; + } + + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + a &= 0x7f; + a = a<<7; + a |= b; + *v = a; + return 2; + } + + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_2_0; + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_2_0; + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= SLOT_2_0; + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= SLOT_2_0; + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_4_2_0; + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= SLOT_2_0; + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +#ifdef SQLITE_NOINLINE +# define FTS5_NOINLINE SQLITE_NOINLINE +#else +# define FTS5_NOINLINE +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; + return 1; + } + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; + return 2; + } + return fts5PutVarint64(p,v); +} + + +static int sqlite3Fts5GetVarintLen(u32 iVal){ +#if 0 + if( iVal<(1 << 7 ) ) return 1; +#endif + assert( iVal>=(1 << 7) ); + if( iVal<(1 << 14) ) return 2; + if( iVal<(1 << 21) ) return 3; + if( iVal<(1 << 28) ) return 4; + return 5; +} + +#line 1 "fts5_vocab.c" +/* +** 2015 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite virtual table module implementing direct access to an +** existing FTS5 index. The module may create several different types of +** tables: +** +** col: +** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col)); +** +** One row for each term/column combination. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term within column $col. Field $cnt is set to the total number of +** instances of term $term in column $col (in any row of the fts5 table). +** +** row: +** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term)); +** +** One row for each term in the database. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term. Field $cnt is set to the total number of instances of term +** $term in the database. +** +** instance: +** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY()); +** +** One row for each term instance in the database. +*/ + +/* #include "third_party/sqlite3/fts5Int.h" */ + + +typedef struct Fts5VocabTable Fts5VocabTable; +typedef struct Fts5VocabCursor Fts5VocabCursor; + +struct Fts5VocabTable { + sqlite3_vtab base; + char *zFts5Tbl; /* Name of fts5 table */ + char *zFts5Db; /* Db containing fts5 table */ + sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* FTS5 global object for this database */ + int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */ + unsigned bBusy; /* True if busy */ +}; + +struct Fts5VocabCursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ + Fts5Table *pFts5; /* Associated FTS5 table */ + + int bEof; /* True if this cursor is at EOF */ + Fts5IndexIter *pIter; /* Term/rowid iterator object */ + + int nLeTerm; /* Size of zLeTerm in bytes */ + char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + + /* These are used by 'col' tables only */ + int iCol; + i64 *aCnt; + i64 *aDoc; + + /* Output values used by all tables. */ + i64 rowid; /* This table's current rowid value */ + Fts5Buffer term; /* Current value of 'term' column */ + + /* Output values Used by 'instance' tables only */ + i64 iInstPos; + int iInstOff; +}; + +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_INSTANCE 2 + +#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" +#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" + +/* +** Bits for the mask used as the idxNum value by xBestIndex/xFilter. +*/ +#define FTS5_VOCAB_TERM_EQ 0x01 +#define FTS5_VOCAB_TERM_GE 0x02 +#define FTS5_VOCAB_TERM_LE 0x04 + + +/* +** Translate a string containing an fts5vocab table type to an +** FTS5_VOCAB_XXX constant. If successful, set *peType to the output +** value and return SQLITE_OK. Otherwise, set *pzErr to an error message +** and return SQLITE_ERROR. +*/ +static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){ + int rc = SQLITE_OK; + char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1); + if( rc==SQLITE_OK ){ + sqlite3Fts5Dequote(zCopy); + if( sqlite3_stricmp(zCopy, "col")==0 ){ + *peType = FTS5_VOCAB_COL; + }else + + if( sqlite3_stricmp(zCopy, "row")==0 ){ + *peType = FTS5_VOCAB_ROW; + }else + if( sqlite3_stricmp(zCopy, "instance")==0 ){ + *peType = FTS5_VOCAB_INSTANCE; + }else + { + *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); + rc = SQLITE_ERROR; + } + sqlite3_free(zCopy); + } + + return rc; +} + + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5vocab") +** argv[1] -> database name +** argv[2] -> table name +** +** then: +** +** argv[3] -> name of fts5 table +** argv[4] -> type of fts5vocab table +** +** or, for tables in the TEMP schema only. +** +** argv[3] -> name of fts5 tables database +** argv[4] -> name of fts5 table +** argv[5] -> type of fts5vocab table +*/ +static int fts5VocabInitVtab( + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Pointer to Fts5Global object */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + const char *azSchema[] = { + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" + }; + + Fts5VocabTable *pRet = 0; + int rc = SQLITE_OK; /* Return code */ + int bDb; + + bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0); + + if( argc!=5 && bDb==0 ){ + *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); + rc = SQLITE_ERROR; + }else{ + int nByte; /* Bytes of space to allocate */ + const char *zDb = bDb ? argv[3] : argv[1]; + const char *zTab = bDb ? argv[4] : argv[3]; + const char *zType = bDb ? argv[5] : argv[4]; + int nDb = (int)strlen(zDb)+1; + int nTab = (int)strlen(zTab)+1; + int eType = 0; + + rc = fts5VocabTableType(zType, pzErr, &eType); + if( rc==SQLITE_OK ){ + assert( eType>=0 && eTypepGlobal = (Fts5Global*)pAux; + pRet->eType = eType; + pRet->db = db; + pRet->zFts5Tbl = (char*)&pRet[1]; + pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; + memcpy(pRet->zFts5Tbl, zTab, nTab); + memcpy(pRet->zFts5Db, zDb, nDb); + sqlite3Fts5Dequote(pRet->zFts5Tbl); + sqlite3Fts5Dequote(pRet->zFts5Db); + } + } + + *ppVTab = (sqlite3_vtab*)pRet; + return rc; +} + + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5VocabInitVtab(). +*/ +static int fts5VocabConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5VocabCreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method. +** +** Only constraints of the form: +** +** term <= ? +** term == ? +** term >= ? +** +** are interpreted. Less-than and less-than-or-equal are treated +** identically, as are greater-than and greater-than-or-equal. +*/ +static int fts5VocabBestIndexMethod( + sqlite3_vtab *pUnused, + sqlite3_index_info *pInfo +){ + int i; + int iTermEq = -1; + int iTermGe = -1; + int iTermLe = -1; + int idxNum = 0; + int nArg = 0; + + UNUSED_PARAM(pUnused); + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->iColumn==0 ){ /* term column */ + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i; + } + } + + if( iTermEq>=0 ){ + idxNum |= FTS5_VOCAB_TERM_EQ; + pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg; + pInfo->estimatedCost = 100; + }else{ + pInfo->estimatedCost = 1000000; + if( iTermGe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_GE; + pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + if( iTermLe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_LE; + pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + } + + /* This virtual table always delivers results in ascending order of + ** the "term" column (column 0). So if the user has requested this + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the + ** sqlite3_index_info.orderByConsumed flag to tell the core the results + ** are already in sorted order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + pInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts5VocabOpenMethod( + sqlite3_vtab *pVTab, + sqlite3_vtab_cursor **ppCsr +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; + Fts5Table *pFts5 = 0; + Fts5VocabCursor *pCsr = 0; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + char *zSql = 0; + + if( pTab->bBusy ){ + pVTab->zErrMsg = sqlite3_mprintf( + "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + return SQLITE_ERROR; + } + zSql = sqlite3Fts5Mprintf(&rc, + "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", + pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl + ); + if( zSql ){ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc==SQLITE_ERROR ) rc = SQLITE_OK; + + pTab->bBusy = 1; + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + i64 iId = sqlite3_column_int64(pStmt, 0); + pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId); + } + pTab->bBusy = 0; + + if( rc==SQLITE_OK ){ + if( pFts5==0 ){ + rc = sqlite3_finalize(pStmt); + pStmt = 0; + if( rc==SQLITE_OK ){ + pVTab->zErrMsg = sqlite3_mprintf( + "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + rc = SQLITE_ERROR; + } + }else{ + rc = sqlite3Fts5FlushToDisk(pFts5); + } + } + + if( rc==SQLITE_OK ){ + int nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor); + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); + } + + if( pCsr ){ + pCsr->pFts5 = pFts5; + pCsr->pStmt = pStmt; + pCsr->aCnt = (i64*)&pCsr[1]; + pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol]; + }else{ + sqlite3_finalize(pStmt); + } + + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + pCsr->rowid = 0; + sqlite3Fts5IterClose(pCsr->pIter); + pCsr->pIter = 0; + sqlite3_free(pCsr->zLeTerm); + pCsr->nLeTerm = -1; + pCsr->zLeTerm = 0; + pCsr->bEof = 0; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + fts5VocabResetCursor(pCsr); + sqlite3Fts5BufferFree(&pCsr->term); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + } + return rc; +} + +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){ + int eDetail = pCsr->pFts5->pConfig->eDetail; + int rc = SQLITE_OK; + Fts5IndexIter *pIter = pCsr->pIter; + i64 *pp = &pCsr->iInstPos; + int *po = &pCsr->iInstOff; + + assert( sqlite3Fts5IterEof(pIter)==0 ); + assert( pCsr->bEof==0 ); + while( eDetail==FTS5_DETAIL_NONE + || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) + ){ + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabInstanceNewTerm(pCsr); + if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break; + } + if( rc ){ + pCsr->bEof = 1; + break; + } + } + + return rc; +} + +/* +** Advance the cursor to the next row in the table. +*/ +static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; + int rc = SQLITE_OK; + int nCol = pCsr->pFts5->pConfig->nCol; + + pCsr->rowid++; + + if( pTab->eType==FTS5_VOCAB_INSTANCE ){ + return fts5VocabInstanceNext(pCsr); + } + + if( pTab->eType==FTS5_VOCAB_COL ){ + for(pCsr->iCol++; pCsr->iColiCol++){ + if( pCsr->aDoc[pCsr->iCol] ) break; + } + } + + if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){ + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + assert( nTerm>=0 ); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + return SQLITE_OK; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + memset(pCsr->aCnt, 0, nCol * sizeof(i64)); + memset(pCsr->aDoc, 0, nCol * sizeof(i64)); + pCsr->iCol = 0; + + assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); + while( rc==SQLITE_OK ){ + int eDetail = pCsr->pFts5->pConfig->eDetail; + const u8 *pPos; int nPos; /* Position list */ + i64 iPos = 0; /* 64-bit position read from poslist */ + int iOff = 0; /* Current offset within position list */ + + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; + + switch( pTab->eType ){ + case FTS5_VOCAB_ROW: + if( eDetail==FTS5_DETAIL_FULL ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + } + pCsr->aDoc[0]++; + break; + + case FTS5_VOCAB_COL: + if( eDetail==FTS5_DETAIL_FULL ){ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + if( iCol!=ii ){ + if( ii>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[ii]++; + iCol = ii; + } + pCsr->aCnt[ii]++; + } + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPos=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[iPos]++; + } + }else{ + assert( eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + } + break; + + default: + assert( pTab->eType==FTS5_VOCAB_INSTANCE ); + break; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + } + if( pTab->eType==FTS5_VOCAB_INSTANCE ) break; + + if( rc==SQLITE_OK ){ + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( nTerm!=pCsr->term.n + || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm)) + ){ + break; + } + if( sqlite3Fts5IterEof(pCsr->pIter) ) break; + } + } + } + } + + if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ + for(/* noop */; pCsr->iColaDoc[pCsr->iCol]==0; pCsr->iCol++); + if( pCsr->iCol==nCol ){ + rc = FTS5_CORRUPT; + } + } + return rc; +} + +/* +** This is the xFilter implementation for the virtual table. +*/ +static int fts5VocabFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *zUnused, /* Unused */ + int nUnused, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eType = pTab->eType; + int rc = SQLITE_OK; + + int iVal = 0; + int f = FTS5INDEX_QUERY_SCAN; + const char *zTerm = 0; + int nTerm = 0; + + sqlite3_value *pEq = 0; + sqlite3_value *pGe = 0; + sqlite3_value *pLe = 0; + + UNUSED_PARAM2(zUnused, nUnused); + + fts5VocabResetCursor(pCsr); + if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + + if( pEq ){ + zTerm = (const char *)sqlite3_value_text(pEq); + nTerm = sqlite3_value_bytes(pEq); + f = 0; + }else{ + if( pGe ){ + zTerm = (const char *)sqlite3_value_text(pGe); + nTerm = sqlite3_value_bytes(pGe); + } + if( pLe ){ + const char *zCopy = (const char *)sqlite3_value_text(pLe); + if( zCopy==0 ) zCopy = ""; + pCsr->nLeTerm = sqlite3_value_bytes(pLe); + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + if( pCsr->zLeTerm==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); + } + } + } + + if( rc==SQLITE_OK ){ + Fts5Index *pIndex = pCsr->pFts5->pIndex; + rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + } + if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){ + rc = fts5VocabInstanceNewTerm(pCsr); + } + if( rc==SQLITE_OK && !pCsr->bEof + && (eType!=FTS5_VOCAB_INSTANCE + || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE) + ){ + rc = fts5VocabNextMethod(pCursor); + } + + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + return pCsr->bEof; +} + +static int fts5VocabColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pFts5->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; + + if( iCol==0 ){ + sqlite3_result_text( + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT + ); + }else if( eType==FTS5_VOCAB_COL ){ + assert( iCol==1 || iCol==2 || iCol==3 ); + if( iCol==1 ){ + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + }else if( iCol==2 ){ + iVal = pCsr->aDoc[pCsr->iCol]; + }else{ + iVal = pCsr->aCnt[pCsr->iCol]; + } + }else if( eType==FTS5_VOCAB_ROW ){ + assert( iCol==1 || iCol==2 ); + if( iCol==1 ){ + iVal = pCsr->aDoc[0]; + }else{ + iVal = pCsr->aCnt[0]; + } + }else{ + assert( eType==FTS5_VOCAB_INSTANCE ); + switch( iCol ){ + case 1: + sqlite3_result_int64(pCtx, pCsr->pIter->iRowid); + break; + case 2: { + int ii = -1; + if( eDetail==FTS5_DETAIL_FULL ){ + ii = FTS5_POS2COLUMN(pCsr->iInstPos); + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + ii = (int)pCsr->iInstPos; + } + if( ii>=0 && iipFts5->pConfig->nCol ){ + const char *z = pCsr->pFts5->pConfig->azCol[ii]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + break; + } + default: { + assert( iCol==3 ); + if( eDetail==FTS5_DETAIL_FULL ){ + int ii = FTS5_POS2OFFSET(pCsr->iInstPos); + sqlite3_result_int(pCtx, ii); + } + break; + } + } + } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); + return SQLITE_OK; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. The +** rowid should be written to *pRowid. +*/ +static int fts5VocabRowidMethod( + sqlite3_vtab_cursor *pCursor, + sqlite_int64 *pRowid +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + *pRowid = pCsr->rowid; + return SQLITE_OK; +} + +static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ + static const sqlite3_module fts5Vocab = { + /* iVersion */ 2, + /* xCreate */ fts5VocabCreateMethod, + /* xConnect */ fts5VocabConnectMethod, + /* xBestIndex */ fts5VocabBestIndexMethod, + /* xDisconnect */ fts5VocabDisconnectMethod, + /* xDestroy */ fts5VocabDestroyMethod, + /* xOpen */ fts5VocabOpenMethod, + /* xClose */ fts5VocabCloseMethod, + /* xFilter */ fts5VocabFilterMethod, + /* xNext */ fts5VocabNextMethod, + /* xEof */ fts5VocabEofMethod, + /* xColumn */ fts5VocabColumnMethod, + /* xRowid */ fts5VocabRowidMethod, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0 + }; + void *p = (void*)pGlobal; + + return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); +} + + + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ diff --git a/third_party/sqlite3/fts5.inc b/third_party/sqlite3/fts5.inc new file mode 100644 index 000000000..5aa56821f --- /dev/null +++ b/third_party/sqlite3/fts5.inc @@ -0,0 +1,576 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ +/* clang-format off */ + + +#ifndef _FTS5_H +#define _FTS5_H + +#include "third_party/sqlite3/sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
+** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ diff --git a/third_party/sqlite3/func.c b/third_party/sqlite3/func.c new file mode 100644 index 000000000..7eb0b492b --- /dev/null +++ b/third_party/sqlite3/func.c @@ -0,0 +1,2288 @@ +/* +** 2002 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C-language implementations for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) +*/ +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "third_party/sqlite3/sqliteInt.inc" +#ifndef SQLITE_OMIT_FLOATING_POINT +#include "libc/math.h" +#endif +#include "third_party/sqlite3/vdbeInt.inc" +/* clang-format off */ + +/* +** Return the collating function associated with a function. +*/ +static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ + VdbeOp *pOp; + assert( context->pVdbe!=0 ); + pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; +} + +/* +** Indicate that the accumulator load should be skipped on this +** iteration of the aggregate loop. +*/ +static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + assert( context->isError<=0 ); + context->isError = -1; + context->skipFlag = 1; +} + +/* +** Implementation of the non-aggregate min() and max() functions +*/ +static void minmaxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int mask; /* 0 for min() or 0xffffffff for max() */ + int iBest; + CollSeq *pColl; + + assert( argc>1 ); + mask = sqlite3_user_data(context)==0 ? 0 : -1; + pColl = sqlite3GetFuncCollSeq(context); + assert( pColl ); + assert( mask==-1 || mask==0 ); + iBest = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + for(i=1; i=0 ){ + testcase( mask==0 ); + iBest = i; + } + } + sqlite3_result_value(context, argv[iBest]); +} + +/* +** Return the type of the argument. +*/ +static void typeofFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + static const char *azType[] = { "integer", "real", "text", "blob", "null" }; + int i = sqlite3_value_type(argv[0]) - 1; + UNUSED_PARAMETER(NotUsed); + assert( i>=0 && i=0xc0 ){ + while( (*z & 0xc0)==0x80 ){ z++; z0++; } + } + } + sqlite3_result_int(context, (int)(z-z0)); + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + +/* +** Implementation of the abs() function. +** +** IMP: R-23979-26855 The abs(X) function returns the absolute value of +** the numeric argument X. +*/ +static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal<0 ){ + if( iVal==SMALLEST_INT64 ){ + /* IMP: R-31676-45509 If X is the integer -9223372036854775808 + ** then abs(X) throws an integer overflow error since there is no + ** equivalent positive 64-bit two complement value. */ + sqlite3_result_error(context, "integer overflow", -1); + return; + } + iVal = -iVal; + } + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ + sqlite3_result_null(context); + break; + } + default: { + /* Because sqlite3_value_double() returns 0.0 if the argument is not + ** something that can be converted into a number, we have: + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. + */ + double rVal = sqlite3_value_double(argv[0]); + if( rVal<0 ) rVal = -rVal; + sqlite3_result_double(context, rVal); + break; + } + } +} + +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + unsigned char firstChar; + sqlite3_value *pC1 = 0; + sqlite3_value *pC2 = 0; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( nNeedle>0 ){ + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else if( typeHaystack!=SQLITE_BLOB && typeNeedle!=SQLITE_BLOB ){ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + }else{ + pC1 = sqlite3_value_dup(argv[0]); + zHaystack = sqlite3_value_text(pC1); + if( zHaystack==0 ) goto endInstrOOM; + nHaystack = sqlite3_value_bytes(pC1); + pC2 = sqlite3_value_dup(argv[1]); + zNeedle = sqlite3_value_text(pC2); + if( zNeedle==0 ) goto endInstrOOM; + nNeedle = sqlite3_value_bytes(pC2); + isText = 1; + } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) goto endInstrOOM; + firstChar = zNeedle[0]; + while( nNeedle<=nHaystack + && (zHaystack[0]!=firstChar || memcmp(zHaystack, zNeedle, nNeedle)!=0) + ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + } + sqlite3_result_int(context, N); +endInstr: + sqlite3_value_free(pC1); + sqlite3_value_free(pC2); + return; +endInstrOOM: + sqlite3_result_error_nomem(context); + goto endInstr; +} + +/* +** Implementation of the printf() function. +*/ +static void printfFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + PrintfArguments x; + StrAccum str; + const char *zFormat; + int n; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + x.nArg = argc-1; + x.nUsed = 0; + x.apArg = argv+1; + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + str.printfFlags = SQLITE_PRINTF_SQLFUNC; + sqlite3_str_appendf(&str, zFormat, &x); + n = str.nChar; + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, + SQLITE_DYNAMIC); + } +} + +/* +** Implementation of the substr() function. +** +** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. +** p1 is 1-indexed. So substr(x,1,1) returns the first character +** of x. If x is text, then we actually count UTF-8 characters. +** If x is a blob, then we count bytes. +** +** If p1 is negative, then we begin abs(p1) from the end of x[]. +** +** If p2 is negative, return the p2 characters preceding p1. +*/ +static void substrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z; + const unsigned char *z2; + int len; + int p0type; + i64 p1, p2; + int negP2 = 0; + + assert( argc==3 || argc==2 ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL + || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) + ){ + return; + } + p0type = sqlite3_value_type(argv[0]); + p1 = sqlite3_value_int(argv[1]); + if( p0type==SQLITE_BLOB ){ + len = sqlite3_value_bytes(argv[0]); + z = sqlite3_value_blob(argv[0]); + if( z==0 ) return; + assert( len==sqlite3_value_bytes(argv[0]) ); + }else{ + z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + if( p1<0 ){ + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } + } + } +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + if( p1==0 ) p1 = 1; /* */ +#endif + if( argc==3 ){ + p2 = sqlite3_value_int(argv[2]); + if( p2<0 ){ + p2 = -p2; + negP2 = 1; + } + }else{ + p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + } + if( p1<0 ){ + p1 += len; + if( p1<0 ){ + p2 += p1; + if( p2<0 ) p2 = 0; + p1 = 0; + } + }else if( p1>0 ){ + p1--; + }else if( p2>0 ){ + p2--; + } + if( negP2 ){ + p1 -= p2; + if( p1<0 ){ + p2 += p1; + p1 = 0; + } + } + assert( p1>=0 && p2>=0 ); + if( p0type!=SQLITE_BLOB ){ + while( *z && p1 ){ + SQLITE_SKIP_UTF8(z); + p1--; + } + for(z2=z; *z2 && p2; p2--){ + SQLITE_SKIP_UTF8(z2); + } + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); + }else{ + if( p1+p2>len ){ + p2 = len-p1; + if( p2<0 ) p2 = 0; + } + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); + } +} + +/* +** Implementation of the round() function +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + int n = 0; + double r; + char *zBuf; + assert( argc==1 || argc==2 ); + if( argc==2 ){ + if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; + n = sqlite3_value_int(argv[1]); + if( n>30 ) n = 30; + if( n<0 ) n = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + r = sqlite3_value_double(argv[0]); + /* If Y==0 and X will fit in a 64-bit int, + ** handle the rounding directly, + ** otherwise use printf. + */ + if( r<-4503599627370496.0 || r>+4503599627370496.0 ){ + /* The value has no fractional part so there is nothing to round */ + }else if( n==0 ){ + r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); + }else{ + zBuf = sqlite3_mprintf("%.*f",n,r); + if( zBuf==0 ){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); + sqlite3_free(zBuf); + } + sqlite3_result_double(context, r); +} +#endif + +/* +** Allocate nByte bytes of space using sqlite3Malloc(). If the +** allocation fails, call sqlite3_result_error_nomem() to notify +** the database handle that malloc() has failed and return NULL. +** If nByte is larger than the maximum string or blob length, then +** raise an SQLITE_TOOBIG exception and return NULL. +*/ +static void *contextMalloc(sqlite3_context *context, i64 nByte){ + char *z; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( nByte>0 ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + z = 0; + }else{ + z = sqlite3Malloc(nByte); + if( !z ){ + sqlite3_result_error_nomem(context); + } + } + return z; +} + +/* +** Implementation of the upper() and lower() SQL functions. +*/ +static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + char *z1; + const char *z2; + int i, n; + UNUSED_PARAMETER(argc); + z2 = (char*)sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + /* Verify that the call to _bytes() does not invalidate the _text() pointer */ + assert( z2==(char*)sqlite3_value_text(argv[0]) ); + if( z2 ){ + z1 = contextMalloc(context, ((i64)n)+1); + if( z1 ){ + for(i=0; imatchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ + + while( (c = Utf8Read(zPattern))!=0 ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ + while( (c=Utf8Read(zPattern)) == matchAll + || (c == matchOne && matchOne!=0) ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ + return SQLITE_NOWILDCARDMATCH; + } + } + if( c==0 ){ + return SQLITE_MATCH; /* "*" at the end of the pattern matches */ + }else if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return SQLITE_NOWILDCARDMATCH; + }else{ + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString ){ + int bMatch = patternCompare(&zPattern[-1],zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + SQLITE_SKIP_UTF8(zString); + } + return SQLITE_NOWILDCARDMATCH; + } + } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively continue the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + char zStop[3]; + int bMatch; + if( noCase ){ + zStop[0] = sqlite3Toupper(c); + zStop[1] = sqlite3Tolower(c); + zStop[2] = 0; + }else{ + zStop[0] = c; + zStop[1] = 0; + } + while(1){ + zString += strcspn((const char*)zString, zStop); + if( zString[0]==0 ) break; + zString++; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + } + }else{ + int bMatch; + while( (c2 = Utf8Read(zString))!=0 ){ + if( c2!=c ) continue; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + } + } + return SQLITE_NOWILDCARDMATCH; + } + if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return SQLITE_NOMATCH; + zEscaped = zPattern; + }else{ + u32 prior_c = 0; + int seen = 0; + int invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return SQLITE_NOMATCH; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return SQLITE_NOMATCH; + } + continue; + } + } + c2 = Utf8Read(zString); + if( c==c2 ) continue; + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return SQLITE_NOMATCH; + } + return *zString==0 ? SQLITE_MATCH : SQLITE_NOMATCH; +} + +/* +** The sqlite3_strglob() interface. Return 0 on a match (like strcmp()) and +** non-zero if there is no match. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); +} + +/* +** The sqlite3_strlike() interface. Return 0 on a match and non-zero for +** a miss - like strcmp(). +*/ +int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); +} + +/* +** Count the number of times that the LIKE operator (or GLOB which is +** just a variation of LIKE) gets called. This is used for testing +** only. +*/ +#ifdef SQLITE_TEST +int sqlite3_like_count = 0; +#endif + + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B,A). +** +** This same function (with a different compareInfo structure) computes +** the GLOB operator. +*/ +static void likeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA, *zB; + u32 escape; + int nPat; + sqlite3 *db = sqlite3_context_db_handle(context); + struct compareInfo *pInfo = sqlite3_user_data(context); + struct compareInfo backupInfo; + +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + || sqlite3_value_type(argv[1])==SQLITE_BLOB + ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, 0); + return; + } +#endif + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + nPat = sqlite3_value_bytes(argv[0]); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]+1 ); + if( nPat > db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + const unsigned char *zEsc = sqlite3_value_text(argv[2]); + if( zEsc==0 ) return; + if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + escape = sqlite3Utf8Read(&zEsc); + if( escape==pInfo->matchAll || escape==pInfo->matchOne ){ + memcpy(&backupInfo, pInfo, sizeof(backupInfo)); + pInfo = &backupInfo; + if( escape==pInfo->matchAll ) pInfo->matchAll = 0; + if( escape==pInfo->matchOne ) pInfo->matchOne = 0; + } + }else{ + escape = pInfo->matchSet; + } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); + if( zA && zB ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, + patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); + } +} + +/* +** Implementation of the NULLIF(x,y) function. The result is the first +** argument if the arguments are different. The result is NULL if the +** arguments are equal to each other. +*/ +static void nullifFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + UNUSED_PARAMETER(NotUsed); + if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ + sqlite3_result_value(context, argv[0]); + } +} + +/* +** Implementation of the sqlite_version() function. The result is the version +** of the SQLite library that is running. +*/ +static void versionFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-48699-48617 This function is an SQL wrapper around the + ** sqlite3_libversion() C-interface. */ + sqlite3_result_text(context, sqlite3_libversion(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_source_id() function. The result is a string +** that identifies the particular version of the source code used to build +** SQLite. +*/ +static void sourceidFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-24470-31136 This function is an SQL wrapper around the + ** sqlite3_sourceid() C interface. */ + sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + +/* +** Implementation of the sqlite_compileoption_used() function. +** The result is an integer that identifies if the compiler option +** was used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptionusedFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zOptName; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ + if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); + } +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options +** used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptiongetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ + n = sqlite3_value_int(argv[0]); + sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' +}; + +/* +** Implementation of the QUOTE() function. This function takes a single +** argument. If the argument is numeric, the return value is the same as +** the argument. If the argument is NULL, the return value is the string +** "NULL". Otherwise, the argument is enclosed in single quotes with +** single-quote escapes. +*/ +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_FLOAT: { + double r1, r2; + char zBuf[50]; + r1 = sqlite3_value_double(argv[0]); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + break; + } + case SQLITE_INTEGER: { + sqlite3_result_value(context, argv[0]); + break; + } + case SQLITE_BLOB: { + char *zText = 0; + char const *zBlob = sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); + if( zText ){ + int i; + for(i=0; i>4)&0x0F]; + zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; + } + zText[(nBlob*2)+2] = '\''; + zText[(nBlob*2)+3] = '\0'; + zText[0] = 'X'; + zText[1] = '\''; + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + break; + } + case SQLITE_TEXT: { + int i,j; + u64 n; + const unsigned char *zArg = sqlite3_value_text(argv[0]); + char *z; + + if( zArg==0 ) return; + for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } + z = contextMalloc(context, ((i64)i)+((i64)n)+3); + if( z ){ + z[0] = '\''; + for(i=0, j=1; zArg[i]; i++){ + z[j++] = zArg[i]; + if( zArg[i]=='\'' ){ + z[j++] = '\''; + } + } + z[j++] = '\''; + z[j] = 0; + sqlite3_result_text(context, z, j, sqlite3_free); + } + break; + } + default: { + assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); + sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + break; + } + } +} + +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc64( argc*4+1 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); +} + +/* +** The hex() function. Interpret the argument as a blob. Return +** a hexadecimal rendering as text. +*/ +static void hexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i, n; + const unsigned char *pBlob; + char *zHex, *z; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + pBlob = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + z = zHex = contextMalloc(context, ((i64)n)*2 + 1); + if( zHex ){ + for(i=0; i>4)&0xf]; + *(z++) = hexdigits[c&0xf]; + } + *z = 0; + sqlite3_result_text(context, zHex, n*2, sqlite3_free); + } +} + +/* +** The zeroblob(N) function returns a zero-filled blob of size N bytes. +*/ +static void zeroblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + i64 n; + int rc; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + n = sqlite3_value_int64(argv[0]); + if( n<0 ) n = 0; + rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ + if( rc ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** The replace() function. Three arguments are all strings: call +** them A, B, and C. The result is also a string which is derived +** from A by replacing every occurrence of B with C. The match +** must be exact. Collating sequences are not used. +*/ +static void replaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zStr; /* The input string A */ + const unsigned char *zPattern; /* The pattern string B */ + const unsigned char *zRep; /* The replacement string C */ + unsigned char *zOut; /* The output */ + int nStr; /* Size of zStr */ + int nPattern; /* Size of zPattern */ + int nRep; /* Size of zRep */ + i64 nOut; /* Maximum size of zOut */ + int loopLimit; /* Last zStr[] that might match zPattern[] */ + int i, j; /* Loop counters */ + unsigned cntExpand; /* Number zOut expansions */ + sqlite3 *db = sqlite3_context_db_handle(context); + + assert( argc==3 ); + UNUSED_PARAMETER(argc); + zStr = sqlite3_value_text(argv[0]); + if( zStr==0 ) return; + nStr = sqlite3_value_bytes(argv[0]); + assert( zStr==sqlite3_value_text(argv[0]) ); /* No encoding change */ + zPattern = sqlite3_value_text(argv[1]); + if( zPattern==0 ){ + assert( sqlite3_value_type(argv[1])==SQLITE_NULL + || sqlite3_context_db_handle(context)->mallocFailed ); + return; + } + if( zPattern[0]==0 ){ + assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); + sqlite3_result_value(context, argv[0]); + return; + } + nPattern = sqlite3_value_bytes(argv[1]); + assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ + zRep = sqlite3_value_text(argv[2]); + if( zRep==0 ) return; + nRep = sqlite3_value_bytes(argv[2]); + assert( zRep==sqlite3_value_text(argv[2]) ); + nOut = nStr + 1; + assert( nOutnPattern ){ + nOut += nRep - nPattern; + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + cntExpand++; + if( (cntExpand&(cntExpand-1))==0 ){ + /* Grow the size of the output buffer only on substitutions + ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ + u8 *zOld; + zOld = zOut; + zOut = sqlite3Realloc(zOut, (int)nOut + (nOut - nStr - 1)); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + } + } + memcpy(&zOut[j], zRep, nRep); + j += nRep; + i += nPattern-1; + } + } + assert( j+nStr-i+1<=nOut ); + memcpy(&zOut[j], &zStr[i], nStr-i); + j += nStr - i; + assert( j<=nOut ); + zOut[j] = 0; + sqlite3_result_text(context, (char*)zOut, j, sqlite3_free); +} + +/* +** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. +** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. +*/ +static void trimFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; /* Input string */ + const unsigned char *zCharSet; /* Set of characters to trim */ + int nIn; /* Number of bytes in input */ + int flags; /* 1: trimleft 2: trimright 3: trim */ + int i; /* Loop counter */ + unsigned char *aLen = 0; /* Length of each character in zCharSet */ + unsigned char **azChar = 0; /* Individual characters in zCharSet */ + int nChar; /* Number of characters in zCharSet */ + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + return; + } + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + nIn = sqlite3_value_bytes(argv[0]); + assert( zIn==sqlite3_value_text(argv[0]) ); + if( argc==1 ){ + static const unsigned char lenOne[] = { 1 }; + static unsigned char * const azOne[] = { (u8*)" " }; + nChar = 1; + aLen = (u8*)lenOne; + azChar = (unsigned char **)azOne; + zCharSet = 0; + }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ + return; + }else{ + const unsigned char *z; + for(z=zCharSet, nChar=0; *z; nChar++){ + SQLITE_SKIP_UTF8(z); + } + if( nChar>0 ){ + azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); + if( azChar==0 ){ + return; + } + aLen = (unsigned char*)&azChar[nChar]; + for(z=zCharSet, nChar=0; *z; nChar++){ + azChar[nChar] = (unsigned char *)z; + SQLITE_SKIP_UTF8(z); + aLen[nChar] = (u8)(z - azChar[nChar]); + } + } + } + if( nChar>0 ){ + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); + if( flags & 1 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + zIn += len; + nIn -= len; + } + } + if( flags & 2 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + nIn -= len; + } + } + if( zCharSet ){ + sqlite3_free(azChar); + } + } + sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); +} + + +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + + +/* IMP: R-25361-16150 This function is omitted from SQLite by default. It +** is only available if the SQLITE_SOUNDEX compile-time option is used +** when SQLite is built. +*/ +#ifdef SQLITE_SOUNDEX +/* +** Compute the soundex encoding of a word. +** +** IMP: R-59782-00072 The soundex(X) function returns a string that is the +** soundex encoding of the string X. +*/ +static void soundexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char zResult[8]; + const u8 *zIn; + int i, j; + static const unsigned char iCode[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + }; + assert( argc==1 ); + zIn = (u8*)sqlite3_value_text(argv[0]); + if( zIn==0 ) zIn = (u8*)""; + for(i=0; zIn[i] && !sqlite3Isalpha(zIn[i]); i++){} + if( zIn[i] ){ + u8 prevcode = iCode[zIn[i]&0x7f]; + zResult[0] = sqlite3Toupper(zIn[i]); + for(j=1; j<4 && zIn[i]; i++){ + int code = iCode[zIn[i]&0x7f]; + if( code>0 ){ + if( code!=prevcode ){ + prevcode = code; + zResult[j++] = code + '0'; + } + }else{ + prevcode = 0; + } + } + while( j<4 ){ + zResult[j++] = '0'; + } + zResult[j] = 0; + sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); + }else{ + /* IMP: R-64894-50321 The string "?000" is returned if the argument + ** is NULL or contains no ASCII alphabetic characters. */ + sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); + } +} +#endif /* SQLITE_SOUNDEX */ + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** A function that loads a shared-library extension then returns NULL. +*/ +static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *zFile = (const char *)sqlite3_value_text(argv[0]); + const char *zProc; + sqlite3 *db = sqlite3_context_db_handle(context); + char *zErrMsg = 0; + + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + + if( argc==2 ){ + zProc = (const char *)sqlite3_value_text(argv[1]); + }else{ + zProc = 0; + } + if( zFile && sqlite3_load_extension(db, zFile, zProc, &zErrMsg) ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + } +} +#endif + + +/* +** An instance of the following structure holds the context of a +** sum() or avg() aggregate computation. +*/ +typedef struct SumCtx SumCtx; +struct SumCtx { + double rSum; /* Floating point sum */ + i64 iSum; /* Integer sum */ + i64 cnt; /* Number of elements summed */ + u8 overflow; /* True if integer overflow seen */ + u8 approx; /* True if non-integer value was input to the sum */ +}; + +/* +** Routines used to compute the sum, average, and total. +** +** The SUM() function follows the (broken) SQL standard which means +** that it returns NULL if it sums over no inputs. TOTAL returns +** 0.0 in that case. In addition, TOTAL always returns a float where +** SUM might return an integer if it never encounters a floating point +** value. TOTAL never fails, but SUM might through an exception if +** it overflows an integer. +*/ +static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + if( p && type!=SQLITE_NULL ){ + p->cnt++; + if( type==SQLITE_INTEGER ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum += v; + if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ + p->approx = p->overflow = 1; + } + }else{ + p->rSum += sqlite3_value_double(argv[0]); + p->approx = 1; + } + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + /* p is always non-NULL because sumStep() will have been called first + ** to initialize it */ + if( ALWAYS(p) && type!=SQLITE_NULL ){ + assert( p->cnt>0 ); + p->cnt--; + assert( type==SQLITE_INTEGER || p->approx ); + if( type==SQLITE_INTEGER && p->approx==0 ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum -= v; + p->iSum -= v; + }else{ + p->rSum -= sqlite3_value_double(argv[0]); + } + } +} +#else +# define sumInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void sumFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + if( p->overflow ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( p->approx ){ + sqlite3_result_double(context, p->rSum); + }else{ + sqlite3_result_int64(context, p->iSum); + } + } +} +static void avgFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + sqlite3_result_double(context, p->rSum/(double)p->cnt); + } +} +static void totalFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + sqlite3_result_double(context, p ? p->rSum : (double)0); +} + +/* +** The following structure keeps track of state information for the +** count() aggregate function. +*/ +typedef struct CountCtx CountCtx; +struct CountCtx { + i64 n; +#ifdef SQLITE_DEBUG + int bInverse; /* True if xInverse() ever called */ +#endif +}; + +/* +** Routines to implement the count() aggregate function. +*/ +static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){ + p->n++; + } + +#ifndef SQLITE_OMIT_DEPRECATED + /* The sqlite3_aggregate_count() function is deprecated. But just to make + ** sure it still operates correctly, verify that its count agrees with our + ** internal count when using count(*) and when the total count can be + ** expressed as a 32-bit integer. */ + assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse + || p->n==sqlite3_aggregate_count(context) ); +#endif +} +static void countFinalize(sqlite3_context *context){ + CountCtx *p; + p = sqlite3_aggregate_context(context, 0); + sqlite3_result_int64(context, p ? p->n : 0); +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(ctx, sizeof(*p)); + /* p is always non-NULL since countStep() will have been called first */ + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && ALWAYS(p) ){ + p->n--; +#ifdef SQLITE_DEBUG + p->bInverse = 1; +#endif + } +} +#else +# define countInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Routines to implement min() and max() aggregate functions. +*/ +static void minmaxStep( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + Mem *pArg = (Mem *)argv[0]; + Mem *pBest; + UNUSED_PARAMETER(NotUsed); + + pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); + if( !pBest ) return; + + if( sqlite3_value_type(pArg)==SQLITE_NULL ){ + if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); + }else if( pBest->flags ){ + int max; + int cmp; + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + /* This step function is used for both the min() and max() aggregates, + ** the only difference between the two being that the sense of the + ** comparison is inverted. For the max() aggregate, the + ** sqlite3_user_data() function returns (void *)-1. For min() it + ** returns (void *)db, where db is the sqlite3* database pointer. + ** Therefore the next statement sets variable 'max' to 1 for the max() + ** aggregate, or 0 for min(). + */ + max = sqlite3_user_data(context)!=0; + cmp = sqlite3MemCompare(pBest, pArg, pColl); + if( (max && cmp<0) || (!max && cmp>0) ){ + sqlite3VdbeMemCopy(pBest, pArg); + }else{ + sqlite3SkipAccumulatorLoad(context); + } + }else{ + pBest->db = sqlite3_context_db_handle(context); + sqlite3VdbeMemCopy(pBest, pArg); + } +} +static void minMaxValueFinalize(sqlite3_context *context, int bValue){ + sqlite3_value *pRes; + pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); + if( pRes ){ + if( pRes->flags ){ + sqlite3_result_value(context, pRes); + } + if( bValue==0 ) sqlite3VdbeMemRelease(pRes); + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void minMaxValue(sqlite3_context *context){ + minMaxValueFinalize(context, 1); +} +#else +# define minMaxValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void minMaxFinalize(sqlite3_context *context){ + minMaxValueFinalize(context, 0); +} + +/* +** group_concat(EXPR, ?SEPARATOR?) +*/ +static void groupConcatStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zVal; + StrAccum *pAccum; + const char *zSep; + int nVal, nSep; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + + if( pAccum ){ + sqlite3 *db = sqlite3_context_db_handle(context); + int firstTerm = pAccum->mxAlloc==0; + pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( !firstTerm ){ + if( argc==2 ){ + zSep = (char*)sqlite3_value_text(argv[1]); + nSep = sqlite3_value_bytes(argv[1]); + }else{ + zSep = ","; + nSep = 1; + } + if( zSep ) sqlite3_str_append(pAccum, zSep, nSep); + } + zVal = (char*)sqlite3_value_text(argv[0]); + nVal = sqlite3_value_bytes(argv[0]); + if( zVal ) sqlite3_str_append(pAccum, zVal, nVal); + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + StrAccum *pAccum; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + /* pAccum is always non-NULL since groupConcatStep() will have always + ** run frist to initialize it */ + if( ALWAYS(pAccum) ){ + n = sqlite3_value_bytes(argv[0]); + if( argc==2 ){ + n += sqlite3_value_bytes(argv[1]); + }else{ + n++; + } + if( n>=(int)pAccum->nChar ){ + pAccum->nChar = 0; + }else{ + pAccum->nChar -= n; + memmove(pAccum->zText, &pAccum->zText[n], pAccum->nChar); + } + if( pAccum->nChar==0 ) pAccum->mxAlloc = 0; + } +} +#else +# define groupConcatInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void groupConcatFinalize(sqlite3_context *context){ + StrAccum *pAccum; + pAccum = sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, + sqlite3_free); + } + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatValue(sqlite3_context *context){ + sqlite3_str *pAccum; + pAccum = (sqlite3_str*)sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + const char *zText = sqlite3_str_value(pAccum); + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + } + } +} +#else +# define groupConcatValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** This routine does per-connection function registration. Most +** of the built-in functions above are part of the global function set. +** This routine only deals with those that are not global. +*/ +void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ + int rc = sqlite3_overload_function(db, "MATCH", 2); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } +} + +/* +** Re-register the built-in LIKE functions. The caseSensitive +** parameter determines whether or not the LIKE operator is case +** sensitive. +*/ +void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + struct compareInfo *pInfo; + int flags; + if( caseSensitive ){ + pInfo = (struct compareInfo*)&likeInfoAlt; + flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; + }else{ + pInfo = (struct compareInfo*)&likeInfoNorm; + flags = SQLITE_FUNC_LIKE; + } + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; + sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; +} + +/* +** pExpr points to an expression which implements a function. If +** it is appropriate to apply the LIKE optimization to that function +** then set aWc[0] through aWc[2] to the wildcard characters and the +** escape character and then return TRUE. If the function is not a +** LIKE-style function then return FALSE. +** +** The expression "a LIKE b ESCAPE c" is only considered a valid LIKE +** operator if c is a string literal that is exactly one byte in length. +** That one byte is stored in aWc[3]. aWc[3] is set to zero if there is +** no ESCAPE clause. +** +** *pIsNocase is set to true if uppercase and lowercase are equivalent for +** the function (default for LIKE). If the function makes the distinction +** between uppercase and lowercase (as does GLOB) then *pIsNocase is set to +** false. +*/ +int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ + FuncDef *pDef; + int nExpr; + assert( pExpr!=0 ); + assert( pExpr->op==TK_FUNCTION ); + if( !pExpr->x.pList ){ + return 0; + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + nExpr = pExpr->x.pList->nExpr; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, nExpr, SQLITE_UTF8, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#endif + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ + return 0; + } + + /* The memcpy() statement assumes that the wildcard characters are + ** the first three statements in the compareInfo structure. The + ** asserts() that follow verify that assumption + */ + memcpy(aWc, pDef->pUserData, 3); + assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); + assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); + assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + + if( nExpr<3 ){ + aWc[3] = 0; + }else{ + Expr *pEscape = pExpr->x.pList->a[2].pExpr; + char *zEscape; + if( pEscape->op!=TK_STRING ) return 0; + zEscape = pEscape->u.zToken; + if( zEscape[0]==0 || zEscape[1]!=0 ) return 0; + if( zEscape[0]==aWc[0] ) return 0; + if( zEscape[0]==aWc[1] ) return 0; + aWc[3] = zEscape[0]; + } + + *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; + return 1; +} + +/* Mathematical Constants */ +#ifndef M_PI +# define M_PI 3.141592653589793238462643383279502884 +#endif +#ifndef M_LN10 +# define M_LN10 2.302585092994045684017991454684364208 +#endif +#ifndef M_LN2 +# define M_LN2 0.693147180559945309417232121458176568 +#endif + + +/* Extra math functions that require linking with -lm +*/ +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS +/* +** Implementation SQL functions: +** +** ceil(X) +** ceiling(X) +** floor(X) +** +** The sqlite3_user_data() pointer is a pointer to the libm implementation +** of the underlying C function. +*/ +static void ceilingFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: { + sqlite3_result_int64(context, sqlite3_value_int64(argv[0])); + break; + } + case SQLITE_FLOAT: { + double (*x)(double) = (double(*)(double))sqlite3_user_data(context); + sqlite3_result_double(context, x(sqlite3_value_double(argv[0]))); + break; + } + default: { + break; + } + } +} + +/* +** On some systems, ceil() and floor() are intrinsic function. You are +** unable to take a pointer to these functions. Hence, we here wrap them +** in our own actual functions. +*/ +static double xCeil(double x){ return ceil(x); } +static double xFloor(double x){ return floor(x); } + +/* +** Implementation of SQL functions: +** +** ln(X) - natural logarithm +** log(X) - log X base 10 +** log10(X) - log X base 10 +** log(B,X) - log X base B +*/ +static void logFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x, b, ans; + assert( argc==1 || argc==2 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + x = sqlite3_value_double(argv[0]); + if( x<=0.0 ) return; + break; + default: + return; + } + if( argc==2 ){ + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + b = log(x); + if( b<=0.0 ) return; + x = sqlite3_value_double(argv[1]); + if( x<=0.0 ) return; + break; + default: + return; + } + ans = log(x)/b; + }else{ + ans = log(x); + switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ + case 1: + /* Convert from natural logarithm to log base 10 */ + ans *= 1.0/M_LN10; + break; + case 2: + /* Convert from natural logarithm to log base 2 */ + ans *= 1.0/M_LN2; + break; + default: + break; + } + } + sqlite3_result_double(context, ans); +} + +/* +** Functions to converts degrees to radians and radians to degrees. +*/ +static double degToRad(double x){ return x*(M_PI/180.0); } +static double radToDeg(double x){ return x*(180.0/M_PI); } + +/* +** Implementation of 1-argument SQL math functions: +** +** exp(X) - Compute e to the X-th power +*/ +static void math1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double v0, ans; + double (*x)(double); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + x = (double(*)(double))sqlite3_user_data(context); + ans = x(v0); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 2-argument SQL math functions: +** +** power(X,Y) - Compute X to the Y-th power +*/ +static void math2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0, type1; + double v0, v1, ans; + double (*x)(double,double); + assert( argc==2 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + type1 = sqlite3_value_numeric_type(argv[1]); + if( type1!=SQLITE_INTEGER && type1!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + v1 = sqlite3_value_double(argv[1]); + x = (double(*)(double,double))sqlite3_user_data(context); + ans = x(v0, v1); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 2-argument SQL math functions: +** +** power(X,Y) - Compute X to the Y-th power +*/ +static void piFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==0 ); + sqlite3_result_double(context, M_PI); +} + +#endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ + +/* +** Implementation of sign(X) function. +*/ +static void signFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double x; + UNUSED_PARAMETER(argc); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + x = sqlite3_value_double(argv[0]); + sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); +} + +/* +** All of the FuncDef structures in the aBuiltinFunc[] array above +** to the global function hash table. This occurs at start-time (as +** a consequence of calling sqlite3_initialize()). +** +** After this routine runs +*/ +void sqlite3RegisterBuiltinFunctions(void){ + /* + ** The following array holds FuncDef structures for all of the functions + ** defined in this file. + ** + ** The array cannot be constant since changes are made to the + ** FuncDef.pHash elements at start-time. The elements of this array + ** are read-only after initialization is complete. + ** + ** For peak efficiency, put the most frequently used function last. + */ + static FuncDef aBuiltinFunc[] = { +/***** Functions only available with SQLITE_TESTCTRL_INTERNAL_FUNCTIONS *****/ + TEST_FUNC(implies_nonnull_row, 2, INLINEFUNC_implies_nonnull_row, 0), + TEST_FUNC(expr_compare, 2, INLINEFUNC_expr_compare, 0), + TEST_FUNC(expr_implies_expr, 2, INLINEFUNC_expr_implies_expr, 0), +#ifdef SQLITE_DEBUG + TEST_FUNC(affinity, 1, INLINEFUNC_affinity, 0), +#endif +/***** Regular functions *****/ +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + SFUNCTION(load_extension, 1, 0, 0, loadExt ), + SFUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + INLINE_FUNC(unlikely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likelihood, 2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET| + SQLITE_FUNC_TYPEOF), +#endif + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, + SQLITE_FUNC_MINMAX ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, + SQLITE_FUNC_MINMAX ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), + FUNCTION(printf, -1, 0, 0, printfFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), +#ifndef SQLITE_OMIT_FLOATING_POINT + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), +#endif + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(hex, 1, 0, 0, hexFunc ), + INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), + VFUNCTION(random, 0, 0, 0, randomFunc ), + VFUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), + FUNCTION(quote, 1, 0, 0, quoteFunc ), + VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + VFUNCTION(changes, 0, 0, 0, changes ), + VFUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(substring, 2, 0, 0, substrFunc ), + FUNCTION(substring, 3, 0, 0, substrFunc ), + WAGGREGATE(sum, 1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0), + WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0), + WAGGREGATE(avg, 1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0), + WAGGREGATE(count, 0,0,0, countStep, + countFinalize, countFinalize, countInverse, SQLITE_FUNC_COUNT ), + WAGGREGATE(count, 1,0,0, countStep, + countFinalize, countFinalize, countInverse, 0 ), + WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + FUNCTION(unknown, -1, 0, 0, unknownFunc ), +#endif + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS + MFUNCTION(ceil, 1, xCeil, ceilingFunc ), + MFUNCTION(ceiling, 1, xCeil, ceilingFunc ), + MFUNCTION(floor, 1, xFloor, ceilingFunc ), +#if SQLITE_HAVE_C99_MATH_FUNCS + MFUNCTION(trunc, 1, trunc, ceilingFunc ), +#endif + FUNCTION(ln, 1, 0, 0, logFunc ), + FUNCTION(log, 1, 1, 0, logFunc ), + FUNCTION(log10, 1, 1, 0, logFunc ), + FUNCTION(log2, 1, 2, 0, logFunc ), + FUNCTION(log, 2, 0, 0, logFunc ), + MFUNCTION(exp, 1, exp, math1Func ), + MFUNCTION(pow, 2, pow, math2Func ), + MFUNCTION(power, 2, pow, math2Func ), + MFUNCTION(mod, 2, fmod, math2Func ), + MFUNCTION(acos, 1, acos, math1Func ), + MFUNCTION(asin, 1, asin, math1Func ), + MFUNCTION(atan, 1, atan, math1Func ), + MFUNCTION(atan2, 2, atan2, math2Func ), + MFUNCTION(cos, 1, cos, math1Func ), + MFUNCTION(sin, 1, sin, math1Func ), + MFUNCTION(tan, 1, tan, math1Func ), + MFUNCTION(cosh, 1, cosh, math1Func ), + MFUNCTION(sinh, 1, sinh, math1Func ), + MFUNCTION(tanh, 1, tanh, math1Func ), +#if SQLITE_HAVE_C99_MATH_FUNCS + MFUNCTION(acosh, 1, acosh, math1Func ), + MFUNCTION(asinh, 1, asinh, math1Func ), + MFUNCTION(atanh, 1, atanh, math1Func ), +#endif + MFUNCTION(sqrt, 1, sqrt, math1Func ), + MFUNCTION(radians, 1, degToRad, math1Func ), + MFUNCTION(degrees, 1, radToDeg, math1Func ), + FUNCTION(pi, 0, 0, 0, piFunc ), +#endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ + FUNCTION(sign, 1, 0, 0, signFunc ), + INLINE_FUNC(coalesce, -1, INLINEFUNC_coalesce, 0 ), + INLINE_FUNC(iif, 3, INLINEFUNC_iif, 0 ), + }; +#ifndef SQLITE_OMIT_ALTERTABLE + sqlite3AlterFunctions(); +#endif + sqlite3WindowFunctions(); + sqlite3RegisterDateTimeFunctions(); + sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); + +#if 0 /* Enable to print out how the built-in functions are hashed */ + { + int i; + FuncDef *p; + for(i=0; iu.pHash){ + int n = sqlite3Strlen30(p->zName); + int h = p->zName[0] + n; + printf(" %s(%d)", p->zName, h); + } + printf("\n"); + } + } +#endif +} diff --git a/third_party/sqlite3/geopoly.inc b/third_party/sqlite3/geopoly.inc new file mode 100644 index 000000000..7904f6b01 --- /dev/null +++ b/third_party/sqlite3/geopoly.inc @@ -0,0 +1,1811 @@ +/* +** 2018-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an alternative R-Tree virtual table that +** uses polygons to express the boundaries of 2-dimensional objects. +** +** This file is #include-ed onto the end of "rtree.c" so that it has +** access to all of the R-Tree internals. +*/ +#include "third_party/gdtoa/gdtoa.h" +/* clang-format off */ + +/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */ +#ifdef GEOPOLY_ENABLE_DEBUG + static int geo_debug = 0; +# define GEODEBUG(X) if(geo_debug)printf X +#else +# define GEODEBUG(X) +#endif + +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else +/* Use the standard library for separate compilation */ +#include "libc/str/str.h" /* amalgamator: keep */ +#define safe_isdigit(x) isdigit((unsigned char)(x)) +#define safe_isalnum(x) isalnum((unsigned char)(x)) +#define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function. +*/ +static const char geopolyIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#endif /* JSON NULL - back to original code */ + +/* Compiler and version */ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* Datatype for coordinates +*/ +typedef float GeoCoord; + +/* +** Internal representation of a polygon. +** +** The polygon consists of a sequence of vertexes. There is a line +** segment between each pair of vertexes, and one final segment from +** the last vertex back to the first. (This differs from the GeoJSON +** standard in which the final vertex is a repeat of the first.) +** +** The polygon follows the right-hand rule. The area to the right of +** each segment is "outside" and the area to the left is "inside". +** +** The on-disk representation consists of a 4-byte header followed by +** the values. The 4-byte header is: +** +** encoding (1 byte) 0=big-endian, 1=little-endian +** nvertex (3 bytes) Number of vertexes as a big-endian integer +** +** Enough space is allocated for 4 coordinates, to work around over-zealous +** warnings coming from some compiler (notably, clang). In reality, the size +** of each GeoPoly memory allocate is adjusted as necessary so that the +** GeoPoly.a[] array at the end is the appropriate size. +*/ +typedef struct GeoPoly GeoPoly; +struct GeoPoly { + int nVertex; /* Number of vertexes */ + unsigned char hdr[4]; /* Header for on-disk representation */ + GeoCoord a[8]; /* 2*nVertex values. X (longitude) first, then Y */ +}; + +/* The size of a memory allocation needed for a GeoPoly object sufficient +** to hold N coordinate pairs. +*/ +#define GEOPOLY_SZ(N) (sizeof(GeoPoly) + sizeof(GeoCoord)*2*((N)-4)) + +/* Macros to access coordinates of a GeoPoly. +** We have to use these macros, rather than just say p->a[i] in order +** to silence (incorrect) UBSAN warnings if the array index is too large. +*/ +#define GeoX(P,I) (((GeoCoord*)(P)->a)[(I)*2]) +#define GeoY(P,I) (((GeoCoord*)(P)->a)[(I)*2+1]) + + +/* +** State of a parse of a GeoJSON input. +*/ +typedef struct GeoParse GeoParse; +struct GeoParse { + const unsigned char *z; /* Unparsed input */ + int nVertex; /* Number of vertexes in a[] */ + int nAlloc; /* Space allocated to a[] */ + int nErr; /* Number of errors encountered */ + GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */ +}; + +/* Do a 4-byte byte swap */ +static void geopolySwab32(unsigned char *a){ + unsigned char t = a[0]; + a[0] = a[3]; + a[3] = t; + t = a[1]; + a[1] = a[2]; + a[2] = t; +} + +/* Skip whitespace. Return the next non-whitespace character. */ +static char geopolySkipSpace(GeoParse *p){ + while( safe_isspace(p->z[0]) ) p->z++; + return p->z[0]; +} + +/* Parse out a number. Write the value into *pVal if pVal!=0. +** return non-zero on success and zero if the next token is not a number. +*/ +static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ + char c = geopolySkipSpace(p); + const unsigned char *z = p->z; + int j = 0; + int seenDP = 0; + int seenE = 0; + if( c=='-' ){ + j = 1; + c = z[j]; + } + if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0; + for(;; j++){ + c = z[j]; + if( safe_isdigit(c) ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return 0; + if( seenDP ) return 0; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return 0; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return 0; + continue; + } + break; + } + if( z[j-1]<'0' ) return 0; + if( pVal ){ +#ifdef SQLITE_AMALGAMATION + /* The sqlite3AtoF() routine is much much faster than atof(), if it + ** is available */ + double r; + (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + *pVal = r; +#else + *pVal = (GeoCoord)atof((const char*)p->z); +#endif + } + p->z += j; + return 1; +} + +/* +** If the input is a well-formed JSON array of coordinates with at least +** four coordinates and where each coordinate is itself a two-value array, +** then convert the JSON into a GeoPoly object and return a pointer to +** that object. +** +** If any error occurs, return NULL. +*/ +static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ + GeoParse s; + int rc = SQLITE_OK; + memset(&s, 0, sizeof(s)); + s.z = z; + if( geopolySkipSpace(&s)=='[' ){ + s.z++; + while( geopolySkipSpace(&s)=='[' ){ + int ii = 0; + char c; + s.z++; + if( s.nVertex>=s.nAlloc ){ + GeoCoord *aNew; + s.nAlloc = s.nAlloc*2 + 16; + aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + s.nErr++; + break; + } + s.a = aNew; + } + while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){ + ii++; + if( ii==2 ) s.nVertex++; + c = geopolySkipSpace(&s); + s.z++; + if( c==',' ) continue; + if( c==']' && ii>=2 ) break; + s.nErr++; + rc = SQLITE_ERROR; + goto parse_json_err; + } + if( geopolySkipSpace(&s)==',' ){ + s.z++; + continue; + } + break; + } + if( geopolySkipSpace(&s)==']' + && s.nVertex>=4 + && s.a[0]==s.a[s.nVertex*2-2] + && s.a[1]==s.a[s.nVertex*2-1] + && (s.z++, geopolySkipSpace(&s)==0) + ){ + GeoPoly *pOut; + int x = 1; + s.nVertex--; /* Remove the redundant vertex at the end */ + pOut = sqlite3_malloc64( GEOPOLY_SZ((sqlite3_int64)s.nVertex) ); + x = 1; + if( pOut==0 ) goto parse_json_err; + pOut->nVertex = s.nVertex; + memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); + pOut->hdr[0] = *(unsigned char*)&x; + pOut->hdr[1] = (s.nVertex>>16)&0xff; + pOut->hdr[2] = (s.nVertex>>8)&0xff; + pOut->hdr[3] = s.nVertex&0xff; + sqlite3_free(s.a); + if( pRc ) *pRc = SQLITE_OK; + return pOut; + }else{ + s.nErr++; + rc = SQLITE_ERROR; + } + } +parse_json_err: + if( pRc ) *pRc = rc; + sqlite3_free(s.a); + return 0; +} + +/* +** Given a function parameter, try to interpret it as a polygon, either +** in the binary format or JSON text. Compute a GeoPoly object and +** return a pointer to that object. Or if the input is not a well-formed +** polygon, put an error message in sqlite3_context and return NULL. +*/ +static GeoPoly *geopolyFuncParam( + sqlite3_context *pCtx, /* Context for error messages */ + sqlite3_value *pVal, /* The value to decode */ + int *pRc /* Write error here */ +){ + GeoPoly *p = 0; + int nByte; + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + ){ + const unsigned char *a = sqlite3_value_blob(pVal); + int nVertex; + nVertex = (a[1]<<16) + (a[2]<<8) + a[3]; + if( (a[0]==0 || a[0]==1) + && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte + ){ + p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + if( pCtx ) sqlite3_result_error_nomem(pCtx); + }else{ + int x = 1; + p->nVertex = nVertex; + memcpy(p->hdr, a, nByte); + if( a[0] != *(unsigned char*)&x ){ + int ii; + for(ii=0; iihdr[0] ^= 1; + } + } + } + if( pRc ) *pRc = SQLITE_OK; + return p; + }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + const unsigned char *zJson = sqlite3_value_text(pVal); + if( zJson==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + return geopolyParseJson(zJson, pRc); + }else{ + if( pRc ) *pRc = SQLITE_ERROR; + return 0; + } +} + +/* +** Implementation of the geopoly_blob(X) function. +** +** If the input is a well-formed Geopoly BLOB or JSON string +** then return the BLOB representation of the polygon. Otherwise +** return NULL. +*/ +static void geopolyBlobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_json(X) +** +** Interpret X as a polygon and render it as a JSON array +** of coordinates. Or, if X is not a valid polygon, return NULL. +*/ +static void geopolyJsonFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + sqlite3_str_append(x, "[", 1); + for(i=0; inVertex; i++){ + sqlite3_str_appendf(x, "[%!g,%!g],", GeoX(p,i), GeoY(p,i)); + } + sqlite3_str_appendf(x, "[%!g,%!g]]", GeoX(p,0), GeoY(p,0)); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_svg(X, ....) +** +** Interpret X as a polygon and render it as a SVG . +** Additional arguments are added as attributes to the . +*/ +static void geopolySvgFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p; + if( argc<1 ) return; + p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + char cSep = '\''; + sqlite3_str_appendf(x, ""); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL Function: geopoly_xform(poly, A, B, C, D, E, F) +** +** Transform and/or translate a polygon as follows: +** +** x1 = A*x0 + B*y0 + E +** y1 = C*x0 + D*y0 + F +** +** For a translation: +** +** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset) +** +** Rotate by R around the point (0,0): +** +** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0) +*/ +static void geopolyXformFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + double A = sqlite3_value_double(argv[1]); + double B = sqlite3_value_double(argv[2]); + double C = sqlite3_value_double(argv[3]); + double D = sqlite3_value_double(argv[4]); + double E = sqlite3_value_double(argv[5]); + double F = sqlite3_value_double(argv[6]); + GeoCoord x1, y1, x0, y0; + int ii; + if( p ){ + for(ii=0; iinVertex; ii++){ + x0 = GeoX(p,ii); + y0 = GeoY(p,ii); + x1 = (GeoCoord)(A*x0 + B*y0 + E); + y1 = (GeoCoord)(C*x0 + D*y0 + F); + GeoX(p,ii) = x1; + GeoY(p,ii) = y1; + } + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** Compute the area enclosed by the polygon. +** +** This routine can also be used to detect polygons that rotate in +** the wrong direction. Polygons are suppose to be counter-clockwise (CCW). +** This routine returns a negative value for clockwise (CW) polygons. +*/ +static double geopolyArea(GeoPoly *p){ + double rArea = 0.0; + int ii; + for(ii=0; iinVertex-1; ii++){ + rArea += (GeoX(p,ii) - GeoX(p,ii+1)) /* (x0 - x1) */ + * (GeoY(p,ii) + GeoY(p,ii+1)) /* (y0 + y1) */ + * 0.5; + } + rArea += (GeoX(p,ii) - GeoX(p,0)) /* (xN - x0) */ + * (GeoY(p,ii) + GeoY(p,0)) /* (yN + y0) */ + * 0.5; + return rArea; +} + +/* +** Implementation of the geopoly_area(X) function. +** +** If the input is a well-formed Geopoly BLOB then return the area +** enclosed by the polygon. If the polygon circulates clockwise instead +** of counterclockwise (as it should) then return the negative of the +** enclosed area. Otherwise return NULL. +*/ +static void geopolyAreaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_double(context, geopolyArea(p)); + sqlite3_free(p); + } +} + +/* +** Implementation of the geopoly_ccw(X) function. +** +** If the rotation of polygon X is clockwise (incorrect) instead of +** counter-clockwise (the correct winding order according to RFC7946) +** then reverse the order of the vertexes in polygon X. +** +** In other words, this routine returns a CCW polygon regardless of the +** winding order of its input. +** +** Use this routine to sanitize historical inputs that that sometimes +** contain polygons that wind in the wrong direction. +*/ +static void geopolyCcwFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + if( geopolyArea(p)<0.0 ){ + int ii, jj; + for(ii=1, jj=p->nVertex-1; iihdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +#define GEOPOLY_PI 3.1415926535897932385 + +/* Fast approximation for sine(X) for X between -0.5*pi and 2*pi +*/ +static double geopolySine(double r){ + assert( r>=-0.5*GEOPOLY_PI && r<=2.0*GEOPOLY_PI ); + if( r>=1.5*GEOPOLY_PI ){ + r -= 2.0*GEOPOLY_PI; + } + if( r>=0.5*GEOPOLY_PI ){ + return -geopolySine(r-GEOPOLY_PI); + }else{ + double r2 = r*r; + double r3 = r2*r; + double r5 = r3*r2; + return 0.9996949*r - 0.1656700*r3 + 0.0075134*r5; + } +} + +/* +** Function: geopoly_regular(X,Y,R,N) +** +** Construct a simple, convex, regular polygon centered at X, Y +** with circumradius R and with N sides. +*/ +static void geopolyRegularFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x = sqlite3_value_double(argv[0]); + double y = sqlite3_value_double(argv[1]); + double r = sqlite3_value_double(argv[2]); + int n = sqlite3_value_int(argv[3]); + int i; + GeoPoly *p; + + if( n<3 || r<=0.0 ) return; + if( n>1000 ) n = 1000; + p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + i = 1; + p->hdr[0] = *(unsigned char*)&i; + p->hdr[1] = 0; + p->hdr[2] = (n>>8)&0xff; + p->hdr[3] = n&0xff; + for(i=0; ihdr, 4+8*n, SQLITE_TRANSIENT); + sqlite3_free(p); +} + +/* +** If pPoly is a polygon, compute its bounding box. Then: +** +** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL +** (2) otherwise, compute a GeoPoly for the bounding box and return the +** new GeoPoly +** +** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from +** the bounding box in aCoord and return a pointer to that GeoPoly. +*/ +static GeoPoly *geopolyBBox( + sqlite3_context *context, /* For recording the error */ + sqlite3_value *pPoly, /* The polygon */ + RtreeCoord *aCoord, /* Results here */ + int *pRc /* Error code here */ +){ + GeoPoly *pOut = 0; + GeoPoly *p; + float mnX, mxX, mnY, mxY; + if( pPoly==0 && aCoord!=0 ){ + p = 0; + mnX = aCoord[0].f; + mxX = aCoord[1].f; + mnY = aCoord[2].f; + mxY = aCoord[3].f; + goto geopolyBboxFill; + }else{ + p = geopolyFuncParam(context, pPoly, pRc); + } + if( p ){ + int ii; + mnX = mxX = GeoX(p,0); + mnY = mxY = GeoY(p,0); + for(ii=1; iinVertex; ii++){ + double r = GeoX(p,ii); + if( rmxX ) mxX = (float)r; + r = GeoY(p,ii); + if( rmxY ) mxY = (float)r; + } + if( pRc ) *pRc = SQLITE_OK; + if( aCoord==0 ){ + geopolyBboxFill: + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); + if( pOut==0 ){ + sqlite3_free(p); + if( context ) sqlite3_result_error_nomem(context); + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + pOut->nVertex = 4; + ii = 1; + pOut->hdr[0] = *(unsigned char*)ⅈ + pOut->hdr[1] = 0; + pOut->hdr[2] = 0; + pOut->hdr[3] = 4; + GeoX(pOut,0) = mnX; + GeoY(pOut,0) = mnY; + GeoX(pOut,1) = mxX; + GeoY(pOut,1) = mnY; + GeoX(pOut,2) = mxX; + GeoY(pOut,2) = mxY; + GeoX(pOut,3) = mnX; + GeoY(pOut,3) = mxY; + }else{ + sqlite3_free(p); + aCoord[0].f = mnX; + aCoord[1].f = mxX; + aCoord[2].f = mnY; + aCoord[3].f = mxY; + } + }else{ + memset(aCoord, 0, sizeof(RtreeCoord)*4); + } + return pOut; +} + +/* +** Implementation of the geopoly_bbox(X) SQL function. +*/ +static void geopolyBBoxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** State vector for the geopoly_group_bbox() aggregate function. +*/ +typedef struct GeoBBox GeoBBox; +struct GeoBBox { + int isInit; + RtreeCoord a[4]; +}; + + +/* +** Implementation of the geopoly_group_bbox(X) aggregate SQL function. +*/ +static void geopolyBBoxStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + RtreeCoord a[4]; + int rc = SQLITE_OK; + (void)geopolyBBox(context, argv[0], a, &rc); + if( rc==SQLITE_OK ){ + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); + if( pBBox==0 ) return; + if( pBBox->isInit==0 ){ + pBBox->isInit = 1; + memcpy(pBBox->a, a, sizeof(RtreeCoord)*4); + }else{ + if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0]; + if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1]; + if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2]; + if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3]; + } + } +} +static void geopolyBBoxFinal( + sqlite3_context *context +){ + GeoPoly *p; + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0); + if( pBBox==0 ) return; + p = geopolyBBox(context, 0, pBBox->a, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + + +/* +** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). +** Returns: +** +** +2 x0,y0 is on the line segement +** +** +1 x0,y0 is beneath line segment +** +** 0 x0,y0 is not on or beneath the line segment or the line segment +** is vertical and x0,y0 is not on the line segment +** +** The left-most coordinate min(x1,x2) is not considered to be part of +** the line segment for the purposes of this analysis. +*/ +static int pointBeneathLine( + double x0, double y0, + double x1, double y1, + double x2, double y2 +){ + double y; + if( x0==x1 && y0==y1 ) return 2; + if( x1x2 ) return 0; + }else if( x1>x2 ){ + if( x0<=x2 || x0>x1 ) return 0; + }else{ + /* Vertical line segment */ + if( x0!=x1 ) return 0; + if( y0y1 && y0>y2 ) return 0; + return 2; + } + y = y1 + (y2-y1)*(x0-x1)/(x2-x1); + if( y0==y ) return 2; + if( y0nVertex-1; ii++){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,ii+1),GeoY(p1,ii+1)); + if( v==2 ) break; + cnt += v; + } + if( v!=2 ){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,0), GeoY(p1,0)); + } + if( v==2 ){ + sqlite3_result_int(context, 1); + }else if( ((v+cnt)&1)==0 ){ + sqlite3_result_int(context, 0); + }else{ + sqlite3_result_int(context, 2); + } + sqlite3_free(p1); +} + +/* Forward declaration */ +static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2); + +/* +** SQL function: geopoly_within(P1,P2) +** +** Return +2 if P1 and P2 are the same polygon +** Return +1 if P2 is contained within P1 +** Return 0 if any part of P2 is on the outside of P1 +** +*/ +static void geopolyWithinFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* Objects used by the overlap algorihm. */ +typedef struct GeoEvent GeoEvent; +typedef struct GeoSegment GeoSegment; +typedef struct GeoOverlap GeoOverlap; +struct GeoEvent { + double x; /* X coordinate at which event occurs */ + int eType; /* 0 for ADD, 1 for REMOVE */ + GeoSegment *pSeg; /* The segment to be added or removed */ + GeoEvent *pNext; /* Next event in the sorted list */ +}; +struct GeoSegment { + double C, B; /* y = C*x + B */ + double y; /* Current y value */ + float y0; /* Initial y value */ + unsigned char side; /* 1 for p1, 2 for p2 */ + unsigned int idx; /* Which segment within the side */ + GeoSegment *pNext; /* Next segment in a list sorted by y */ +}; +struct GeoOverlap { + GeoEvent *aEvent; /* Array of all events */ + GeoSegment *aSegment; /* Array of all segments */ + int nEvent; /* Number of events */ + int nSegment; /* Number of segments */ +}; + +/* +** Add a single segment and its associated events. +*/ +static void geopolyAddOneSegment( + GeoOverlap *p, + GeoCoord x0, + GeoCoord y0, + GeoCoord x1, + GeoCoord y1, + unsigned char side, + unsigned int idx +){ + GeoSegment *pSeg; + GeoEvent *pEvent; + if( x0==x1 ) return; /* Ignore vertical segments */ + if( x0>x1 ){ + GeoCoord t = x0; + x0 = x1; + x1 = t; + t = y0; + y0 = y1; + y1 = t; + } + pSeg = p->aSegment + p->nSegment; + p->nSegment++; + pSeg->C = (y1-y0)/(x1-x0); + pSeg->B = y1 - x1*pSeg->C; + pSeg->y0 = y0; + pSeg->side = side; + pSeg->idx = idx; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x0; + pEvent->eType = 0; + pEvent->pSeg = pSeg; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x1; + pEvent->eType = 1; + pEvent->pSeg = pSeg; +} + + + +/* +** Insert all segments and events for polygon pPoly. +*/ +static void geopolyAddSegments( + GeoOverlap *p, /* Add segments to this Overlap object */ + GeoPoly *pPoly, /* Take all segments from this polygon */ + unsigned char side /* The side of pPoly */ +){ + unsigned int i; + GeoCoord *x; + for(i=0; i<(unsigned)pPoly->nVertex-1; i++){ + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i); + } + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i); +} + +/* +** Merge two lists of sorted events by X coordinate +*/ +static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){ + GeoEvent head, *pLast; + head.pNext = 0; + pLast = &head; + while( pRight && pLeft ){ + if( pRight->x <= pLeft->x ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort an array of nEvent event objects into a list. +*/ +static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){ + int mx = 0; + int i, j; + GeoEvent *p; + GeoEvent *a[50]; + for(i=0; ipNext = 0; + for(j=0; j=mx ) mx = j+1; + } + p = 0; + for(i=0; iy - pLeft->y; + if( r==0.0 ) r = pRight->C - pLeft->C; + if( r<0.0 ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort a list of GeoSegments in order of increasing Y and in the event of +** a tie, increasing C (slope). +*/ +static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ + int mx = 0; + int i; + GeoSegment *p; + GeoSegment *a[50]; + while( pList ){ + p = pList; + pList = pList->pNext; + p->pNext = 0; + for(i=0; i=mx ) mx = i+1; + } + p = 0; + for(i=0; inVertex + p2->nVertex + 2; + GeoOverlap *p; + sqlite3_int64 nByte; + GeoEvent *pThisEvent; + double rX; + int rc = 0; + int needSort = 0; + GeoSegment *pActive = 0; + GeoSegment *pSeg; + unsigned char aOverlap[4]; + + nByte = sizeof(GeoEvent)*nVertex*2 + + sizeof(GeoSegment)*nVertex + + sizeof(GeoOverlap); + p = sqlite3_malloc64( nByte ); + if( p==0 ) return -1; + p->aEvent = (GeoEvent*)&p[1]; + p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; + p->nEvent = p->nSegment = 0; + geopolyAddSegments(p, p1, 1); + geopolyAddSegments(p, p2, 2); + pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent); + rX = pThisEvent && pThisEvent->x==0.0 ? -1.0 : 0.0; + memset(aOverlap, 0, sizeof(aOverlap)); + while( pThisEvent ){ + if( pThisEvent->x!=rX ){ + GeoSegment *pPrev = 0; + int iMask = 0; + GEODEBUG(("Distinct X: %g\n", pThisEvent->x)); + rX = pThisEvent->x; + if( needSort ){ + GEODEBUG(("SORT\n")); + pActive = geopolySortSegmentsByYAndC(pActive); + needSort = 0; + } + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pPrev ){ + if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + pPrev = 0; + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + double y = pSeg->C*rX + pSeg->B; + GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y)); + pSeg->y = y; + if( pPrev ){ + if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){ + rc = 1; + GEODEBUG(("Crossing: %d.%d and %d.%d\n", + pPrev->side, pPrev->idx, + pSeg->side, pSeg->idx)); + goto geopolyOverlapDone; + }else if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + } + GEODEBUG(("%s %d.%d C=%g B=%g\n", + pThisEvent->eType ? "RM " : "ADD", + pThisEvent->pSeg->side, pThisEvent->pSeg->idx, + pThisEvent->pSeg->C, + pThisEvent->pSeg->B)); + if( pThisEvent->eType==0 ){ + /* Add a segment */ + pSeg = pThisEvent->pSeg; + pSeg->y = pSeg->y0; + pSeg->pNext = pActive; + pActive = pSeg; + needSort = 1; + }else{ + /* Remove a segment */ + if( pActive==pThisEvent->pSeg ){ + pActive = pActive->pNext; + }else{ + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pSeg->pNext==pThisEvent->pSeg ){ + pSeg->pNext = pSeg->pNext->pNext; + break; + } + } + } + } + pThisEvent = pThisEvent->pNext; + } + if( aOverlap[3]==0 ){ + rc = 0; + }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){ + rc = 3; + }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){ + rc = 2; + }else if( aOverlap[1]==0 && aOverlap[2]==0 ){ + rc = 4; + }else{ + rc = 1; + } + +geopolyOverlapDone: + sqlite3_free(p); + return rc; +} + +/* +** SQL function: geopoly_overlap(P1,P2) +** +** Determine whether or not P1 and P2 overlap. Return value: +** +** 0 The two polygons are disjoint +** 1 They overlap +** 2 P1 is completely contained within P2 +** 3 P2 is completely contained within P1 +** 4 P1 and P2 are the same polygon +** NULL Either P1 or P2 or both are not valid polygons +*/ +static void geopolyOverlapFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* +** Enable or disable debugging output +*/ +static void geopolyDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ +#ifdef GEOPOLY_ENABLE_DEBUG + geo_debug = sqlite3_value_int(argv[0]); +#endif +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the geopoly virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int geopolyInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ + sqlite3_str *pSql; + char *zSql; + int ii; + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = RTREE_COORD_REAL32; + pRtree->nDim = 2; + pRtree->nDim2 = 4; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape"); + pRtree->nAux = 1; /* Add one for _shape */ + pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */ + for(ii=3; iinAux++; + sqlite3_str_appendf(pSql, ",%s", argv[ii]); + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); + if( rc ) goto geopolyInit_fail; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto geopolyInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto geopolyInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +geopolyInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** GEOPOLY virtual table module xCreate method. +*/ +static int geopolyCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** GEOPOLY virtual table module xConnect method. +*/ +static int geopolyConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + + +/* +** GEOPOLY virtual table module xFilter method. +** +** Query plans: +** +** 1 rowid lookup +** 2 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 3 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 4 full table scan +*/ +static int geopolyFilter( + sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */ + int idxNum, /* Query plan */ + const char *idxStr, /* Not Used */ + int argc, sqlite3_value **argv /* Parameters to the query plan */ +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && idxNum<=3 ){ + RtreeCoord bbox[4]; + RtreeConstraint *p; + assert( argc==1 ); + geopolyBBox(0, argv[0], bbox, &rc); + if( rc ){ + goto geopoly_filter_end; + } + pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4); + pCsr->nConstraint = 4; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + if( idxNum==2 ){ + /* Overlap query */ + p->op = 'B'; + p->iCoord = 0; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 1; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 2; + p->u.rValue = bbox[3].f; + p++; + p->op = 'D'; + p->iCoord = 3; + p->u.rValue = bbox[2].f; + }else{ + /* Within query */ + p->op = 'D'; + p->iCoord = 0; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 1; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 2; + p->u.rValue = bbox[2].f; + p++; + p->op = 'B'; + p->iCoord = 3; + p->u.rValue = bbox[3].f; + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto geopoly_filter_end; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + +geopoly_filter_end: + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 "rowid" Direct lookup by rowid. +** 2 "rtree" R-tree overlap query using geopoly_overlap() +** 3 "rtree" R-tree within query using geopoly_within() +** 4 "fullscan" full-table scan. +** ------------------------------------------------ +*/ +static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + int iRowidTerm = -1; + int iFuncTerm = -1; + int idxNum = 0; + + for(ii=0; iinConstraint; ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + if( !p->usable ) continue; + if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + iRowidTerm = ii; + break; + } + if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap() + ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within(). + ** See geopolyFindFunction() */ + iFuncTerm = ii; + idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2; + } + } + + if( iRowidTerm>=0 ){ + pIdxInfo->idxNum = 1; + pIdxInfo->idxStr = "rowid"; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + if( iFuncTerm>=0 ){ + pIdxInfo->idxNum = idxNum; + pIdxInfo->idxStr = "rtree"; + pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0; + pIdxInfo->estimatedCost = 300.0; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->idxNum = 4; + pIdxInfo->idxStr = "fullscan"; + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + return SQLITE_OK; +} + + +/* +** GEOPOLY virtual table module xColumn method. +*/ +static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; + if( i<=pRtree->nAux ){ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2)); + } + return SQLITE_OK; +} + + +/* +** The xUpdate method for GEOPOLY module virtual tables. +** +** For DELETE: +** +** argv[0] = the rowid to be deleted +** +** For INSERT: +** +** argv[0] = SQL NULL +** argv[1] = rowid to insert, or an SQL NULL to select automatically +** argv[2] = _shape column +** argv[3] = first application-defined column.... +** +** For UPDATE: +** +** argv[0] = rowid to modify. Never NULL +** argv[1] = rowid after the change. Never NULL +** argv[2] = new value for _shape +** argv[3] = new value for first application-defined column.... +*/ +static int geopolyUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + i64 oldRowid; /* The old rowid */ + int oldRowidValid; /* True if oldRowid is valid */ + i64 newRowid; /* The new rowid */ + int newRowidValid; /* True if newRowid is valid */ + int coordChange = 0; /* Change in coordinates */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;; + oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0; + newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL; + newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; + cell.iRowid = newRowid; + + if( nData>1 /* not a DELETE */ + && (!oldRowidValid /* INSERT */ + || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || oldRowid!=newRowid) /* Rowid change */ + ){ + geopolyBBox(0, aData[2], cell.aCoord, &rc); + if( rc ){ + if( rc==SQLITE_ERROR ){ + pVtab->zErrMsg = + sqlite3_mprintf("_shape does not contain a valid polygon"); + } + goto geopoly_update_end; + } + coordChange = 1; + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + } + } + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){ + rc = rtreeDeleteRowid(pRtree, oldRowid); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 && coordChange ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + if( !newRowidValid ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + /* Change the data */ + if( rc==SQLITE_OK && nData>1 ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + int nChange = 0; + sqlite3_bind_int64(pUp, 1, cell.iRowid); + assert( pRtree->nAux>=1 ); + if( sqlite3_value_nochange(aData[2]) ){ + sqlite3_bind_null(pUp, 2); + }else{ + GeoPoly *p = 0; + if( sqlite3_value_type(aData[2])==SQLITE_TEXT + && (p = geopolyFuncParam(0, aData[2], &rc))!=0 + && rc==SQLITE_OK + ){ + sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_value(pUp, 2, aData[2]); + } + sqlite3_free(p); + nChange = 1; + } + for(jj=1; jjnAux; jj++){ + nChange++; + sqlite3_bind_value(pUp, jj+2, aData[jj+2]); + } + if( nChange ){ + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +geopoly_update_end: + rtreeRelease(pRtree); + return rc; +} + +/* +** Report that geopoly_overlap() is an overloaded function suitable +** for use in xBestIndex. +*/ +static int geopolyFindFunction( + sqlite3_vtab *pVtab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ + *pxFunc = geopolyOverlapFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION; + } + if( sqlite3_stricmp(zName, "geopoly_within")==0 ){ + *pxFunc = geopolyWithinFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION+1; + } + return 0; +} + + +static sqlite3_module geopolyModule = { + 3, /* iVersion */ + geopolyCreate, /* xCreate - create a table */ + geopolyConnect, /* xConnect - connect to an existing table */ + geopolyBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + geopolyFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + geopolyColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + geopolyUpdate, /* xUpdate - write data */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ + geopolyFindFunction, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + rtreeSavepoint, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + rtreeShadowName /* xShadowName */ +}; + +static int sqlite3_geopoly_init(sqlite3 *db){ + int rc = SQLITE_OK; + static const struct { + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + signed char nArg; + unsigned char bPure; + const char *zName; + } aFunc[] = { + { geopolyAreaFunc, 1, 1, "geopoly_area" }, + { geopolyBlobFunc, 1, 1, "geopoly_blob" }, + { geopolyJsonFunc, 1, 1, "geopoly_json" }, + { geopolySvgFunc, -1, 1, "geopoly_svg" }, + { geopolyWithinFunc, 2, 1, "geopoly_within" }, + { geopolyContainsPointFunc, 3, 1, "geopoly_contains_point" }, + { geopolyOverlapFunc, 2, 1, "geopoly_overlap" }, + { geopolyDebugFunc, 1, 0, "geopoly_debug" }, + { geopolyBBoxFunc, 1, 1, "geopoly_bbox" }, + { geopolyXformFunc, 7, 1, "geopoly_xform" }, + { geopolyRegularFunc, 4, 1, "geopoly_regular" }, + { geopolyCcwFunc, 1, 1, "geopoly_ccw" }, + }; + static const struct { + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + const char *zName; + } aAgg[] = { + { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, + }; + int i; + for(i=0; i? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; + +/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards +** compatibility for legacy applications, the URI filename capability is +** disabled by default. +** +** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled +** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. +*/ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif + +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) +# define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif +#endif + +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. +*/ +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 250 +#endif + +/* Statement journals spill to disk when their size exceeds the following +** threshold (in bytes). 0 means that statement journals are created and +** written to disk immediately (the default behavior for SQLite versions +** before 3.12.0). -1 means always keep the entire statement journal in +** memory. (The statement journal is also always held entirely in memory +** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this +** setting.) +*/ +#ifndef SQLITE_STMTJRNL_SPILL +# define SQLITE_STMTJRNL_SPILL (64*1024) +#endif + +/* +** The default lookaside-configuration, the format "SZ,N". SZ is the +** number of bytes in each lookaside slot (should be a multiple of 8) +** and N is the number of slots. The lookaside-configuration can be +** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) +** or at run-time for an individual database connection using +** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); +** +** With the two-size-lookaside enhancement, less lookaside is required. +** The default configuration of 1200,40 actually provides 30 1200-byte slots +** and 93 128-byte slots, which is more lookaside than is available +** using the older 1200,100 configuration without two-size-lookaside. +*/ +#ifndef SQLITE_DEFAULT_LOOKASIDE +# ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define SQLITE_DEFAULT_LOOKASIDE 1200,100 /* 120KB of memory */ +# else +# define SQLITE_DEFAULT_LOOKASIDE 1200,40 /* 48KB of memory */ +# endif +#endif + + +/* The default maximum size of an in-memory database created using +** sqlite3_deserialize() +*/ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + +/* +** The following singleton contains the global configuration for +** the SQLite library. +*/ +SQLITE_WSD struct Sqlite3Config sqlite3Config = { + SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ + 1, /* bCoreMutex */ + SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0, /* bSmallMalloc */ + 1, /* bExtraSchemaChecks */ + 0x7ffffffe, /* mxStrlen */ + 0, /* neverCorrupt */ + SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ + SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ + {0,0,0,0,0,0,0,0}, /* m */ + {0,0,0,0,0,0,0,0,0}, /* mutex */ + {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ + (void*)0, /* pHeap */ + 0, /* nHeap */ + 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ + (void*)0, /* pPage */ + 0, /* szPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ + 0, /* mxParserStack */ + 0, /* sharedCacheEnabled */ + SQLITE_SORTER_PMASZ, /* szPma */ + /* All the rest should always be initialized to zero */ + 0, /* isInit */ + 0, /* inProgress */ + 0, /* isMutexInit */ + 0, /* isMallocInit */ + 0, /* isPCacheInit */ + 0, /* nRefInitMutex */ + 0, /* pInitMutex */ + 0, /* xLog */ + 0, /* pLogArg */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0, /* pSqllogArg */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ +#endif +#ifdef SQLITE_ENABLE_DESERIALIZE + SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ +#endif +#ifndef SQLITE_UNTESTABLE + 0, /* xTestCallback */ +#endif + 0, /* bLocaltimeFault */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ +}; + +/* +** Hash table for global functions - functions common to all +** database connections. After initialization, this table is +** read-only. +*/ +FuncDefHash sqlite3BuiltinFunctions; + +#ifdef VDBE_PROFILE +/* +** The following performance counter can be used in place of +** sqlite3Hwtime() for profiling. This is a no-op on standard builds. +*/ +sqlite3_uint64 sqlite3NProfileCnt = 0; +#endif + +/* +** The value of the "pending" byte must be 0x40000000 (1 byte past the +** 1-gibabyte boundary) in a compatible database. SQLite never uses +** the database page that contains the pending byte. It never attempts +** to read or write that page. The pending byte page is set aside +** for use by the VFS layers as space for managing file locks. +** +** During testing, it is often desirable to move the pending byte to +** a different position in the file. This allows code that has to +** deal with the pending byte to run on files that are much smaller +** than 1 GiB. The sqlite3_test_control() interface can be used to +** move the pending byte. +** +** IMPORTANT: Changing the pending byte to any value other than +** 0x40000000 results in an incompatible database file format! +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. +*/ +#ifndef SQLITE_OMIT_WSD +int sqlite3PendingByte = 0x40000000; +#endif + +/* +** Tracing flags set by SQLITE_TESTCTRL_TRACEFLAGS. +*/ +u32 sqlite3SelectTrace = 0; +u32 sqlite3WhereTrace = 0; + +#include "third_party/sqlite3/opcodes.inc" +/* +** Properties of opcodes. The OPFLG_INITIALIZER macro is +** created by mkopcodeh.awk during compilation. Data is obtained +** from the comments following the "case OP_xxxx:" statements in +** the vdbe.c file. +*/ +const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; + +/* +** Name of the default collating sequence +*/ +const char sqlite3StrBINARY[] = "BINARY"; diff --git a/third_party/sqlite3/hash.c b/third_party/sqlite3/hash.c new file mode 100644 index 000000000..3f1244043 --- /dev/null +++ b/third_party/sqlite3/hash.c @@ -0,0 +1,270 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables +** used in SQLite. +*/ +#include "libc/assert.h" +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +*/ +void sqlite3HashInit(Hash *pNew){ + assert( pNew!=0 ); + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +void sqlite3HashClear(Hash *pH){ + HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + sqlite3_free(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + HashElem *next_elem = elem->next; + sqlite3_free(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** The hashing function. +*/ +static unsigned int strHash(const char *z){ + unsigned int h = 0; + unsigned char c; + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). + ** 0x9e3779b1 is 2654435761 which is the closest prime number to + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ + h += sqlite3UpperToLower[c]; + h *= 0x9e3779b1; + } + return h; +} + + +/* Link pNew element into the hash table pH. If pEntry!=0 then also +** insert pNew into the pEntry hash bucket. +*/ +static void insertElement( + Hash *pH, /* The complete hash table */ + struct _ht *pEntry, /* The entry into which pNew is inserted */ + HashElem *pNew /* The element to be inserted */ +){ + HashElem *pHead; /* First element already in pEntry */ + if( pEntry ){ + pHead = pEntry->count ? pEntry->chain : 0; + pEntry->count++; + pEntry->chain = pNew; + }else{ + pHead = 0; + } + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** +** The hash table might fail to resize if sqlite3_malloc() fails or +** if the new size is the same as the prior size. +** Return TRUE if the resize occurs and false if not. +*/ +static int rehash(Hash *pH, unsigned int new_size){ + struct _ht *new_ht; /* The new hash table */ + HashElem *elem, *next_elem; /* For looping over existing elements */ + +#if SQLITE_MALLOC_SOFT_LIMIT>0 + if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){ + new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht); + } + if( new_size==pH->htsize ) return 0; +#endif + + /* The inability to allocates space for a larger hash table is + ** a performance hit but it is not a fatal error. So mark the + ** allocation as a benign. Use sqlite3Malloc()/memset(0) instead of + ** sqlite3MallocZero() to make the allocation, as sqlite3MallocZero() + ** only zeroes the requested number of bytes whereas this module will + ** use the actual amount of space allocated for the hash table (which + ** may be larger than the requested amount). + */ + sqlite3BeginBenignMalloc(); + new_ht = (struct _ht *)sqlite3Malloc( new_size*sizeof(struct _ht) ); + sqlite3EndBenignMalloc(); + + if( new_ht==0 ) return 0; + sqlite3_free(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); + memset(new_ht, 0, new_size*sizeof(struct _ht)); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + unsigned int h = strHash(elem->pKey) % new_size; + next_elem = elem->next; + insertElement(pH, &new_ht[h], elem); + } + return 1; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. If no element is found, +** a pointer to a static null element with HashElem.data==0 is returned. +** If pH is not NULL, then the hash for this key is written to *pH. +*/ +static HashElem *findElementWithHash( + const Hash *pH, /* The pH to be searched */ + const char *pKey, /* The key we are searching for */ + unsigned int *pHash /* Write the hash value here */ +){ + HashElem *elem; /* Used to loop thru the element list */ + unsigned int count; /* Number of elements left to test */ + unsigned int h; /* The computed hash */ + static HashElem nullElement = { 0, 0, 0, 0 }; + + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ + struct _ht *pEntry; + h = strHash(pKey) % pH->htsize; + pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + }else{ + h = 0; + elem = pH->first; + count = pH->count; + } + if( pHash ) *pHash = h; + while( count-- ){ + assert( elem!=0 ); + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + return elem; + } + elem = elem->next; + } + return &nullElement; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void removeElementGivenHash( + Hash *pH, /* The pH containing "elem" */ + HashElem* elem, /* The element to be removed from the pH */ + unsigned int h /* Hash value for the element */ +){ + struct _ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + if( pH->ht ){ + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + assert( pEntry->count>0 ); + pEntry->count--; + } + sqlite3_free( elem ); + pH->count--; + if( pH->count==0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + sqlite3HashClear(pH); + } +} + +/* Attempt to locate an element of the hash table pH with a key +** that matches pKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +void *sqlite3HashFind(const Hash *pH, const char *pKey){ + assert( pH!=0 ); + assert( pKey!=0 ); + return findElementWithHash(pH, pKey, 0)->data; +} + +/* Insert an element into the hash table pH. The key is pKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created and NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ + unsigned int h; /* the hash of the key modulo hash table size */ + HashElem *elem; /* Used to loop thru the element list */ + HashElem *new_elem; /* New element added to the pH */ + + assert( pH!=0 ); + assert( pKey!=0 ); + elem = findElementWithHash(pH,pKey,&h); + if( elem->data ){ + void *old_data = elem->data; + if( data==0 ){ + removeElementGivenHash(pH,elem,h); + }else{ + elem->data = data; + elem->pKey = pKey; + } + return old_data; + } + if( data==0 ) return 0; + new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); + if( new_elem==0 ) return data; + new_elem->pKey = pKey; + new_elem->data = data; + pH->count++; + if( pH->count>=10 && pH->count > 2*pH->htsize ){ + if( rehash(pH, pH->count*2) ){ + assert( pH->htsize>0 ); + h = strHash(pKey) % pH->htsize; + } + } + insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); + return 0; +} diff --git a/third_party/sqlite3/hash.inc b/third_party/sqlite3/hash.inc new file mode 100644 index 000000000..e3ebd15ad --- /dev/null +++ b/third_party/sqlite3/hash.inc @@ -0,0 +1,97 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. +*/ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H +/* clang-format off */ + +/* Forward declarations of structures. */ +typedef struct Hash Hash; +typedef struct HashElem HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, some of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +** +** All elements of the hash table are on a single doubly-linked list. +** Hash.first points to the head of this list. +** +** There are Hash.htsize buckets. Each bucket points to a spot in +** the global doubly-linked list. The contents of the bucket are the +** element pointed to plus the next _ht.count-1 elements in the list. +** +** Hash.htsize and Hash.ht may be zero. In that case lookup is done +** by a linear search of the global list. For small tables, the +** Hash.ht table is never allocated because if there are few elements +** in the table, it is faster to do a linear search than to manage +** the hash table. +*/ +struct Hash { + unsigned int htsize; /* Number of buckets in the hash table */ + unsigned int count; /* Number of entries in this table */ + HashElem *first; /* The first element of the array */ + struct _ht { /* the hash table */ + unsigned int count; /* Number of entries with this hash */ + HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct HashElem { + HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + const char *pKey; /* Key associated with this element */ +}; + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +void sqlite3HashInit(Hash*); +void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +void *sqlite3HashFind(const Hash*, const char *pKey); +void sqlite3HashClear(Hash*); + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Hash h; +** HashElem *p; +** ... +** for(p=sqliteHashFirst(&h); p; p=sqliteHashNext(p)){ +** SomeStructure *pData = sqliteHashData(p); +** // do something with pData +** } +*/ +#define sqliteHashFirst(H) ((H)->first) +#define sqliteHashNext(E) ((E)->next) +#define sqliteHashData(E) ((E)->data) +/* #define sqliteHashKey(E) ((E)->pKey) // NOT USED */ +/* #define sqliteHashKeysize(E) ((E)->nKey) // NOT USED */ + +/* +** Number of entries in a hash table +*/ +/* #define sqliteHashCount(H) ((H)->count) // NOT USED */ + +#endif /* SQLITE_HASH_H */ diff --git a/third_party/sqlite3/hwtime.inc b/third_party/sqlite3/hwtime.inc new file mode 100644 index 000000000..1c315733e --- /dev/null +++ b/third_party/sqlite3/hwtime.inc @@ -0,0 +1,86 @@ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H +/* clang-format off */ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ + sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ diff --git a/third_party/sqlite3/icu.c b/third_party/sqlite3/icu.c new file mode 100644 index 000000000..ced08d6bd --- /dev/null +++ b/third_party/sqlite3/icu.c @@ -0,0 +1,550 @@ +/* +** 2007 May 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. +*/ +/* clang-format off */ + +#if 0 && !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) || \ + defined(SQLITE_ENABLE_ICU_COLLATIONS) + +/* Include ICU headers */ +#include "libc/assert.h" +#include "libc/unicode/unicode.h" + +#ifndef SQLITE_CORE +#include "third_party/sqlite3/sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#else +#include "third_party/sqlite3/sqlite3.h" +#endif + +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); +} + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while( 1 ){ + + /* Read (and consume) the next character from the input pattern. */ + uint32_t uPattern; + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + } + zPattern++; + } + + if( *zPattern==0 ) return 1; + + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ + return 1; + } + SQLITE_ICU_SKIP_UTF8(zString); + } + return 0; + + }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 2. */ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + + }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + uint32_t uString; + SQLITE_ICU_READ_UTF8(zString, uString); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; + } + } + + return *zString==0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + }else{ + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> '\u131' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; + const char *zLocale = 0; + + assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); + return; + } + + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } + + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } + return; + } + assert( 0 ); /* Unreachable */ +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. +*/ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + icuCollationColl, icuCollationDel + ); + if( rc!=SQLITE_OK ){ + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } +} + +/* +** Register the ICU extension functions with database db. +*/ +int sqlite3IcuInit(sqlite3 *db){ +# define SQLITEICU_EXTRAFLAGS (SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) + static const struct IcuScalar { + const char *zName; /* Function name */ + unsigned char nArg; /* Number of arguments */ + unsigned int enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } scalars[] = { + {"icu_load_collation",2,SQLITE_UTF8|SQLITE_DIRECTONLY,1, icuLoadCollation}, +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + {"regexp", 2, SQLITE_ANY|SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + }; + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ + const struct IcuScalar *p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 + ); + } + + return rc; +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +#endif diff --git a/third_party/sqlite3/insert.c b/third_party/sqlite3/insert.c new file mode 100644 index 000000000..783a60125 --- /dev/null +++ b/third_party/sqlite3/insert.c @@ -0,0 +1,3056 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle INSERT statements in SQLite. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** Generate code that will +** +** (1) acquire a lock for table pTab then +** (2) open pTab as cursor iCur. +** +** If pTab is a WITHOUT ROWID table, then it is the PRIMARY KEY index +** for that table that is actually opened. +*/ +void sqlite3OpenTable( + Parse *pParse, /* Generate code into this VDBE */ + int iCur, /* The cursor number of the table */ + int iDb, /* The database index in sqlite3.aDb[] */ + Table *pTab, /* The table to be opened */ + int opcode /* OP_OpenRead or OP_OpenWrite */ +){ + Vdbe *v; + assert( !IsVirtual(pTab) ); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; + assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); + VdbeComment((v, "%s", pTab->zName)); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->tnum==pTab->tnum ); + sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + VdbeComment((v, "%s", pTab->zName)); + } +} + +/* +** Return a pointer to the column affinity string associated with index +** pIdx. A column affinity string has one character for each column in +** the table, according to the affinity of the column: +** +** Character Column affinity +** ------------------------------ +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL +** +** An extra 'D' is appended to the end of the string to cover the +** rowid that appears as the last column in every index. +** +** Memory for the buffer containing the column index affinity string +** is managed along with the rest of the Index structure. It will be +** released when sqlite3DeleteIndex() is called. +*/ +const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); + if( !pIdx->zColAff ){ + sqlite3OomFault(db); + return 0; + } + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; + } + + return pIdx->zColAff; +} + +/* +** Compute the affinity string for table pTab, if it has not already been +** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. +** +** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and +** if iReg>0 then code an OP_Affinity opcode that will set the affinities +** for register iReg and following. Or if affinities exists and iReg==0, +** then just set the P4 operand of the previous opcode (which should be +** an OP_MakeRecord) to the affinity string. +** +** A column affinity string has one character per column: +** +** Character Column affinity +** ------------------------------ +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL +*/ +void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ + int i, j; + char *zColAff = pTab->zColAff; + if( zColAff==0 ){ + sqlite3 *db = sqlite3VdbeDb(v); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + if( !zColAff ){ + sqlite3OomFault(db); + return; + } + + for(i=j=0; inCol; i++){ + assert( pTab->aCol[i].affinity!=0 ); + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } + } + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); + pTab->zColAff = zColAff; + } + assert( zColAff!=0 ); + i = sqlite3Strlen30NN(zColAff); + if( i ){ + if( iReg ){ + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); + }else{ + sqlite3VdbeChangeP4(v, -1, zColAff, i); + } + } +} + +/* +** Return non-zero if the table pTab in database iDb or any of its indices +** have been opened at any point in the VDBE program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using a temporary table for the results of the SELECT. +*/ +static int readsTable(Parse *p, int iDb, Table *pTab){ + Vdbe *v = sqlite3GetVdbe(p); + int i; + int iEnd = sqlite3VdbeCurrentAddr(v); +#ifndef SQLITE_OMIT_VIRTUALTABLE + VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; +#endif + + for(i=1; iopcode==OP_OpenRead && pOp->p3==iDb ){ + Index *pIndex; + Pgno tnum = pOp->p2; + if( tnum==pTab->tnum ){ + return 1; + } + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( tnum==pIndex->tnum ){ + return 1; + } + } + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pVTab ){ + assert( pOp->p4.pVtab!=0 ); + assert( pOp->p4type==P4_VTAB ); + return 1; + } +#endif + } + return 0; +} + +/* This walker callback will compute the union of colFlags flags for all +** referenced columns in a CHECK constraint or generated column expression. +*/ +static int exprColumnFlagUnion(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 ){ + assert( pExpr->iColumn < pWalker->u.pTab->nCol ); + pWalker->eCode |= pWalker->u.pTab->aCol[pExpr->iColumn].colFlags; + } + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** All regular columns for table pTab have been puts into registers +** starting with iRegStore. The registers that correspond to STORED +** or VIRTUAL columns have not yet been initialized. This routine goes +** back and computes the values for those columns based on the previously +** computed normal columns. +*/ +void sqlite3ComputeGeneratedColumns( + Parse *pParse, /* Parsing context */ + int iRegStore, /* Register holding the first column */ + Table *pTab /* The table */ +){ + int i; + Walker w; + Column *pRedo; + int eProgress; + VdbeOp *pOp; + + assert( pTab->tabFlags & TF_HasGenerated ); + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + + /* Before computing generated columns, first go through and make sure + ** that appropriate affinity has been applied to the regular columns + */ + sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); + if( (pTab->tabFlags & TF_HasStored)!=0 + && (pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1))->opcode==OP_Affinity + ){ + /* Change the OP_Affinity argument to '@' (NONE) for all stored + ** columns. '@' is the no-op affinity and those columns have not + ** yet been computed. */ + int ii, jj; + char *zP4 = pOp->p4.z; + assert( zP4!=0 ); + assert( pOp->p4type==P4_DYNAMIC ); + for(ii=jj=0; zP4[jj]; ii++){ + if( pTab->aCol[ii].colFlags & COLFLAG_VIRTUAL ){ + continue; + } + if( pTab->aCol[ii].colFlags & COLFLAG_STORED ){ + zP4[jj] = SQLITE_AFF_NONE; + } + jj++; + } + } + + /* Because there can be multiple generated columns that refer to one another, + ** this is a two-pass algorithm. On the first pass, mark all generated + ** columns as "not available". + */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + pTab->aCol[i].colFlags |= COLFLAG_NOTAVAIL; + } + } + + w.u.pTab = pTab; + w.xExprCallback = exprColumnFlagUnion; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + + /* On the second pass, compute the value of each NOT-AVAILABLE column. + ** Companion code in the TK_COLUMN case of sqlite3ExprCodeTarget() will + ** compute dependencies and mark remove the COLSPAN_NOTAVAIL mark, as + ** they are needed. + */ + pParse->iSelfTab = -iRegStore; + do{ + eProgress = 0; + pRedo = 0; + for(i=0; inCol; i++){ + Column *pCol = pTab->aCol + i; + if( (pCol->colFlags & COLFLAG_NOTAVAIL)!=0 ){ + int x; + pCol->colFlags |= COLFLAG_BUSY; + w.eCode = 0; + sqlite3WalkExpr(&w, pCol->pDflt); + pCol->colFlags &= ~COLFLAG_BUSY; + if( w.eCode & COLFLAG_NOTAVAIL ){ + pRedo = pCol; + continue; + } + eProgress = 1; + assert( pCol->colFlags & COLFLAG_GENERATED ); + x = sqlite3TableColumnToStorage(pTab, i) + iRegStore; + sqlite3ExprCodeGeneratedColumn(pParse, pCol, x); + pCol->colFlags &= ~COLFLAG_NOTAVAIL; + } + } + }while( pRedo && eProgress ); + if( pRedo ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pRedo->zName); + } + pParse->iSelfTab = 0; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* +** Locate or create an AutoincInfo structure associated with table pTab +** which is in database iDb. Return the register number for the register +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) +** +** There is at most one AutoincInfo structure per table even if the +** same table is autoincremented multiple times due to inserts within +** triggers. A new AutoincInfo structure is created if this is the +** first use of table pTab. On 2nd and subsequent uses, the original +** AutoincInfo structure is used. +** +** Four consecutive registers are allocated: +** +** (1) The name of the pTab table. +** (2) The maximum ROWID of pTab. +** (3) The rowid in sqlite_sequence of pTab +** (4) The original value of the max ROWID in pTab, or NULL if none +** +** The 2nd register is the one that is returned. That is all the +** insert routine needs to know about. +*/ +static int autoIncBegin( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database holding pTab */ + Table *pTab /* The table we are writing to */ +){ + int memId = 0; /* Register holding maximum rowid */ + assert( pParse->db->aDb[iDb].pSchema!=0 ); + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->mDbFlags & DBFLAG_Vacuum)==0 + ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + AutoincInfo *pInfo; + Table *pSeqTab = pParse->db->aDb[iDb].pSchema->pSeqTab; + + /* Verify that the sqlite_sequence table exists and is an ordinary + ** rowid table with exactly two columns. + ** Ticket d8dc2b3a58cd5dc2918a1d4acb 2018-05-23 */ + if( pSeqTab==0 + || !HasRowid(pSeqTab) + || IsVirtual(pSeqTab) + || pSeqTab->nCol!=2 + ){ + pParse->nErr++; + pParse->rc = SQLITE_CORRUPT_SEQUENCE; + return 0; + } + + pInfo = pToplevel->pAinc; + while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } + if( pInfo==0 ){ + pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); + sqlite3ParserAddCleanup(pToplevel, sqlite3DbFree, pInfo); + testcase( pParse->earlyCleanup ); + if( pParse->db->mallocFailed ) return 0; + pInfo->pNext = pToplevel->pAinc; + pToplevel->pAinc = pInfo; + pInfo->pTab = pTab; + pInfo->iDb = iDb; + pToplevel->nMem++; /* Register to hold name of table */ + pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ + pToplevel->nMem +=2; /* Rowid in sqlite_sequence + orig max val */ + } + memId = pInfo->regCtr; + } + return memId; +} + +/* +** This routine generates code that will initialize all of the +** register used by the autoincrement tracker. +*/ +void sqlite3AutoincrementBegin(Parse *pParse){ + AutoincInfo *p; /* Information about an AUTOINCREMENT */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* Database only autoinc table */ + int memId; /* Register holding max rowid */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + + /* This routine is never called during trigger-generation. It is + ** only called from the top-level */ + assert( pParse->pTriggerTab==0 ); + assert( sqlite3IsToplevel(pParse) ); + + assert( v ); /* We failed long ago if this is not so */ + for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoInc[] = { + /* 0 */ {OP_Null, 0, 0, 0}, + /* 1 */ {OP_Rewind, 0, 10, 0}, + /* 2 */ {OP_Column, 0, 0, 0}, + /* 3 */ {OP_Ne, 0, 9, 0}, + /* 4 */ {OP_Rowid, 0, 0, 0}, + /* 5 */ {OP_Column, 0, 1, 0}, + /* 6 */ {OP_AddImm, 0, 0, 0}, + /* 7 */ {OP_Copy, 0, 0, 0}, + /* 8 */ {OP_Goto, 0, 11, 0}, + /* 9 */ {OP_Next, 0, 2, 0}, + /* 10 */ {OP_Integer, 0, 0, 0}, + /* 11 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; + pDb = &db->aDb[p->iDb]; + memId = p->regCtr; + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); + if( aOp==0 ) break; + aOp[0].p2 = memId; + aOp[0].p3 = memId+2; + aOp[2].p3 = memId; + aOp[3].p1 = memId-1; + aOp[3].p3 = memId; + aOp[3].p5 = SQLITE_JUMPIFNULL; + aOp[4].p2 = memId+1; + aOp[5].p3 = memId; + aOp[6].p1 = memId; + aOp[7].p2 = memId+2; + aOp[7].p1 = memId; + aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; + } +} + +/* +** Update the maximum rowid for an autoincrement calculation. +** +** This routine should be called when the regRowid register holds a +** new rowid that is about to be inserted. If that new rowid is +** larger than the maximum rowid in the memId memory cell, then the +** memory cell is updated. +*/ +static void autoIncStep(Parse *pParse, int memId, int regRowid){ + if( memId>0 ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); + } +} + +/* +** This routine generates the code needed to write autoincrement +** maximum rowid values back into the sqlite_sequence register. +** Every statement that might do an INSERT into an autoincrement +** table (either directly or through triggers) needs to call this +** routine just before the "exit" code. +*/ +static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ + AutoincInfo *p; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( v ); + for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoIncEnd[] = { + /* 0 */ {OP_NotNull, 0, 2, 0}, + /* 1 */ {OP_NewRowid, 0, 0, 0}, + /* 2 */ {OP_MakeRecord, 0, 2, 0}, + /* 3 */ {OP_Insert, 0, 0, 0}, + /* 4 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; + Db *pDb = &db->aDb[p->iDb]; + int iRec; + int memId = p->regCtr; + + iRec = sqlite3GetTempReg(pParse); + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3VdbeAddOp3(v, OP_Le, memId+2, sqlite3VdbeCurrentAddr(v)+7, memId); + VdbeCoverage(v); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); + if( aOp==0 ) break; + aOp[0].p1 = memId+1; + aOp[1].p2 = memId+1; + aOp[2].p1 = memId-1; + aOp[2].p3 = iRec; + aOp[3].p2 = iRec; + aOp[3].p3 = memId+1; + aOp[3].p5 = OPFLAG_APPEND; + sqlite3ReleaseTempReg(pParse, iRec); + } +} +void sqlite3AutoincrementEnd(Parse *pParse){ + if( pParse->pAinc ) autoIncrementEnd(pParse); +} +#else +/* +** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines +** above are all no-ops +*/ +# define autoIncBegin(A,B,C) (0) +# define autoIncStep(A,B,C) +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + + +/* Forward declaration */ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +); + +/* +** This routine is called to handle SQL of the following forms: +** +** insert into TABLE (IDLIST) values(EXPRLIST),(EXPRLIST),... +** insert into TABLE (IDLIST) select +** insert into TABLE (IDLIST) default values +** +** The IDLIST following the table name is always optional. If omitted, +** then a list of all (non-hidden) columns for the table is substituted. +** The IDLIST appears in the pColumn parameter. pColumn is NULL if IDLIST +** is omitted. +** +** For the pSelect parameter holds the values to be inserted for the +** first two forms shown above. A VALUES clause is really just short-hand +** for a SELECT statement that omits the FROM clause and everything else +** that follows. If the pSelect parameter is NULL, that means that the +** DEFAULT VALUES form of the INSERT statement is intended. +** +** The code generated follows one of four templates. For a simple +** insert with data coming from a single-row VALUES clause, the code executes +** once straight down through. Pseudo-code follows (we call this +** the "1st template"): +** +** open write cursor to
and its indices +** put VALUES clause expressions into registers +** write the resulting record into
+** cleanup +** +** The three remaining templates assume the statement is of the form +** +** INSERT INTO
SELECT ... +** +** If the SELECT clause is of the restricted form "SELECT * FROM " - +** in other words if the SELECT pulls all columns from a single table +** and there is no WHERE or LIMIT or GROUP BY or ORDER BY clauses, and +** if and are distinct tables but have identical +** schemas, including all the same indices, then a special optimization +** is invoked that copies raw records from over to . +** See the xferOptimization() function for the implementation of this +** template. This is the 2nd template. +** +** open a write cursor to
+** open read cursor on +** transfer all records in over to
+** close cursors +** foreach index on
+** open a write cursor on the
index +** open a read cursor on the corresponding index +** transfer all records from the read to the write cursors +** close cursors +** end foreach +** +** The 3rd template is for when the second template does not apply +** and the SELECT clause does not read from
at any time. +** The generated code follows this template: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the rows in the SELECT +** load values into registers R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end-coroutine X +** B: open write cursor to
and its indices +** C: yield X, at EOF goto D +** insert the select result into
from R..R+n +** goto C +** D: cleanup +** +** The 4th template is used if the insert statement takes its +** values from a SELECT but the data is being inserted into a table +** that is also read as part of the SELECT. In the third form, +** we have to use an intermediate table to store the results of +** the select. The template is like this: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the tables in the SELECT +** load value into register R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end co-routine R +** B: open temp table +** L: yield X, at EOF goto M +** insert row from R..R+n into temp table +** goto L +** M: open write cursor to
and its indices +** rewind temp table +** C: loop over rows of intermediate table +** transfer values form intermediate table into
+** end loop +** D: cleanup +*/ +void sqlite3Insert( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* Name of table into which we are inserting */ + Select *pSelect, /* A SELECT statement to use as the data source */ + IdList *pColumn, /* Column names corresponding to IDLIST, or NULL. */ + int onError, /* How to handle constraint errors */ + Upsert *pUpsert /* ON CONFLICT clauses for upsert, or NULL */ +){ + sqlite3 *db; /* The main database structure */ + Table *pTab; /* The table to insert into. aka TABLE */ + int i, j; /* Loop counters */ + Vdbe *v; /* Generate code into this virtual machine */ + Index *pIdx; /* For looping over indices of the table */ + int nColumn; /* Number of columns in the data */ + int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ + int iDataCur = 0; /* VDBE cursor that is the main data repository */ + int iIdxCur = 0; /* First index cursor */ + int ipkColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ + int endOfLoop; /* Label for the end of the insertion loop */ + int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ + int addrInsTop = 0; /* Jump to label "D" */ + int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ + SelectDest dest; /* Destination for SELECT on rhs of INSERT */ + int iDb; /* Index of database holding TABLE */ + u8 useTempTable = 0; /* Store SELECT results in intermediate table */ + u8 appendFlag = 0; /* True if the insert is likely to be an append */ + u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 bIdListInOrder; /* True if IDLIST is in table order */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ + int iRegStore; /* Register in which to store next column */ + + /* Register allocations */ + int regFromSelect = 0;/* Base register for data coming from SELECT */ + int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ + int regRowCount = 0; /* Memory cell used for the row counter */ + int regIns; /* Block of regs holding rowid+data being inserted */ + int regRowid; /* registers holding insert rowid */ + int regData; /* register holding first column to insert */ + int *aRegIdx = 0; /* One register allocated to each index */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to insert into a view */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of trigger times */ +#endif + + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto insert_cleanup; + } + dest.iSDParm = 0; /* Suppress a harmless compiler warning */ + + /* If the Select object is really just a simple VALUES() list with a + ** single row (the common case) then keep that one row of values + ** and discard the other (unused) parts of the pSelect object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } + + /* Locate the table into which we will be inserting new information. + */ + assert( pTabList->nSrc==1 ); + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ){ + goto insert_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ + goto insert_cleanup; + } + withoutRowid = !HasRowid(pTab); + + /* Figure out if we have any triggers and if the table being + ** inserted into is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define tmask 0 +# define isView 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); + + /* If pTab is really a view, make sure it has been initialized. + ** ViewGetColumnNames() is a no-op if pTab is not a view. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto insert_cleanup; + } + + /* Cannot insert into a read-only table. + */ + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto insert_cleanup; + } + + /* Allocate a VDBE + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto insert_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, pSelect || pTrigger, iDb); + +#ifndef SQLITE_OMIT_XFER_OPT + /* If the statement is of the form + ** + ** INSERT INTO SELECT * FROM ; + ** + ** Then special optimizations can be applied that make the transfer + ** very fast and which reduce fragmentation of indices. + ** + ** This is the 2nd template. + */ + if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + assert( !pTrigger ); + assert( pList==0 ); + goto insert_end; + } +#endif /* SQLITE_OMIT_XFER_OPT */ + + /* If this is an AUTOINCREMENT table, look up the sequence number in the + ** sqlite_sequence table and store it in memory cell regAutoinc. + */ + regAutoinc = autoIncBegin(pParse, iDb, pTab); + + /* Allocate a block registers to hold the rowid and the values + ** for all columns of the new row. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the ipkColumn variable + ** the index into IDLIST of the primary key column. ipkColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the INTEGER + ** PRIMARY KEY in the original table is pTab->iPKey.) After this + ** loop, if ipkColumn==(-1), that means that integer primary key + ** is unspecified, and hence the table is either WITHOUT ROWID or + ** it will automatically generated an integer primary key. + ** + ** bIdListInOrder is true if the columns in IDLIST are in storage + ** order. This enables an optimization that avoids shuffling the + ** columns into storage order. False negatives are harmless, + ** but false positives will cause database corruption. + */ + bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; + } + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ + pColumn->a[i].idx = j; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zName); + goto insert_cleanup; + } +#endif + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ + ipkColumn = i; + bIdListInOrder = 0; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList, 0, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } + } + } + + /* Figure out how many columns of data are supplied. If the data + ** is coming from a SELECT statement, then generate a co-routine that + ** produces a single row of the SELECT on each invocation. The + ** co-routine is the common header to the 3rd and 4th templates. + */ + if( pSelect ){ + /* Data is coming from a SELECT or from a multi-row VALUES clause. + ** Generate a co-routine to run the SELECT. */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int rc; /* Result code */ + + regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + + /* Set useTempTable to TRUE if the result of the SELECT statement + ** should be written into a temporary table (template 4). Set to + ** FALSE if each output row of the SELECT can be written directly into + ** the destination table (template 3). + ** + ** A temp table must be used if the table being updated is also one + ** of the tables being read by the SELECT statement. Also use a + ** temp table in the case of row triggers. + */ + if( pTrigger || readsTable(pParse, iDb, pTab) ){ + useTempTable = 1; + } + + if( useTempTable ){ + /* Invoke the coroutine to extract information from the SELECT + ** and add it to a transient table srcTab. The code generated + ** here is from the 4th template: + ** + ** B: open temp table + ** L: yield X, goto M at EOF + ** insert row from R..R+n into temp table + ** goto L + ** M: ... + */ + int regRec; /* Register to hold packed record */ + int regTempRowid; /* Register to hold temp table ROWID */ + int addrL; /* Label "L" */ + + srcTab = pParse->nTab++; + regRec = sqlite3GetTempReg(pParse); + regTempRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); + addrL = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); + sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); + sqlite3VdbeGoto(v, addrL); + sqlite3VdbeJumpHere(v, addrL); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTempRowid); + } + }else{ + /* This is the case if the data for the INSERT is coming from a + ** single-row VALUES clause + */ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + srcTab = -1; + assert( useTempTable==0 ); + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ + goto insert_cleanup; + } + }else{ + nColumn = 0; + } + } + + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the ipkColumn variable to the integer primary key + ** column index in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + ipkColumn = pTab->iPKey; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( ipkColumn>=0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + for(i=ipkColumn-1; i>=0; i--){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + ipkColumn--; + } + } + } +#endif + + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + assert( TF_HasHidden==COLFLAG_HIDDEN ); + assert( TF_HasGenerated==COLFLAG_GENERATED ); + assert( COLFLAG_NOINSERT==(COLFLAG_GENERATED|COLFLAG_HIDDEN) ); + if( (pTab->tabFlags & (TF_HasGenerated|TF_HasHidden))!=0 ){ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; + } + } + if( nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList, 0, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; + } + } + if( pColumn!=0 && nColumn!=pColumn->nId ){ + sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); + goto insert_cleanup; + } + + /* Initialize the count of rows to be inserted + */ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + + /* If this is not a view, open the table and and all indices */ + if( !isView ){ + int nIdx; + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, + &iDataCur, &iIdxCur); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+2)); + if( aRegIdx==0 ){ + goto insert_cleanup; + } + for(i=0, pIdx=pTab->pIndex; ipNext, i++){ + assert( pIdx ); + aRegIdx[i] = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; + } + aRegIdx[i] = ++pParse->nMem; /* Register to store the table record */ + } +#ifndef SQLITE_OMIT_UPSERT + if( pUpsert ){ + Upsert *pNx; + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "UPSERT not implemented for virtual table \"%s\"", + pTab->zName); + goto insert_cleanup; + } + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "cannot UPSERT a view"); + goto insert_cleanup; + } + if( sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget) ){ + goto insert_cleanup; + } + pTabList->a[0].iCursor = iDataCur; + pNx = pUpsert; + do{ + pNx->pUpsertSrc = pTabList; + pNx->regData = regData; + pNx->iDataCur = iDataCur; + pNx->iIdxCur = iIdxCur; + if( pNx->pUpsertTarget ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + goto insert_cleanup; + } + } + pNx = pNx->pNextUpsert; + }while( pNx!=0 ); + } +#endif + + + /* This is the top of the main insertion loop */ + if( useTempTable ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 4): + ** + ** rewind temp table, if empty goto D + ** C: loop over rows of intermediate table + ** transfer values form intermediate table into
+ ** end loop + ** D: ... + */ + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); VdbeCoverage(v); + addrCont = sqlite3VdbeCurrentAddr(v); + }else if( pSelect ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 3): + ** + ** C: yield X, at EOF goto D + ** insert the select result into
from R..R+n + ** goto C + ** D: ... + */ + sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0, 0); + addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + if( ipkColumn>=0 ){ + /* tag-20191021-001: If the INTEGER PRIMARY KEY is being generated by the + ** SELECT, go ahead and copy the value into the rowid slot now, so that + ** the value does not get overwritten by a NULL at tag-20191021-002. */ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + } + } + + /* Compute data for ordinary columns of the new entry. Values + ** are written in storage order into registers starting with regData. + ** Only ordinary columns are computed in this loop. The rowid + ** (if there is one) is computed later and generated columns are + ** computed after the rowid since they might depend on the value + ** of the rowid. + */ + nHidden = 0; + iRegStore = regData; assert( regData==regRowid+1 ); + for(i=0; inCol; i++, iRegStore++){ + int k; + u32 colFlags; + assert( i>=nHidden ); + if( i==pTab->iPKey ){ + /* tag-20191021-002: References to the INTEGER PRIMARY KEY are filled + ** using the rowid. So put a NULL in the IPK slot of the record to avoid + ** using excess space. The file format definition requires this extra + ** NULL - we cannot optimize further by skipping the column completely */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( ((colFlags = pTab->aCol[i].colFlags) & COLFLAG_NOINSERT)!=0 ){ + nHidden++; + if( (colFlags & COLFLAG_VIRTUAL)!=0 ){ + /* Virtual columns do not participate in OP_MakeRecord. So back up + ** iRegStore by one slot to compensate for the iRegStore++ in the + ** outer for() loop */ + iRegStore--; + continue; + }else if( (colFlags & COLFLAG_STORED)!=0 ){ + /* Stored columns are computed later. But if there are BEFORE + ** triggers, the slots used for stored columns will be OP_Copy-ed + ** to a second block of registers, so the register needs to be + ** initialized to NULL to avoid an uninitialized register read */ + if( tmask & TRIGGER_BEFORE ){ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + } + continue; + }else if( pColumn==0 ){ + /* Hidden columns that are not explicitly named in the INSERT + ** get there default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + } + if( pColumn ){ + for(j=0; jnId && pColumn->a[j].idx!=i; j++){} + if( j>=pColumn->nId ){ + /* A column not named in the insert column list gets its + ** default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + k = j; + }else if( nColumn==0 ){ + /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + }else{ + k = i - nHidden; + } + + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); + } + }else{ + sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + } + } + + + /* Run the BEFORE and INSTEAD OF triggers, if there are any + */ + endOfLoop = sqlite3VdbeMakeLabel(pParse); + if( tmask & TRIGGER_BEFORE ){ + int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); + + /* build the NEW.* reference row. Note that if there is an INTEGER + ** PRIMARY KEY into which a NULL is being inserted, that NULL will be + ** translated into a unique ID for the row. But on a BEFORE trigger, + ** we do not know what the unique ID will be (because the insert has + ** not happened yet) so we substitute a rowid of -1 + */ + if( ipkColumn<0 ){ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + }else{ + int addr1; + assert( !withoutRowid ); + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); + } + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); + } + + /* Copy the new data already generated. */ + assert( pTab->nNVCol>0 ); + sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regCols+1, pTab); + } +#endif + + /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, + ** do not attempt any conversions before assembling the record. + ** If this is a real table, attempt conversions as required by the + ** table column affinities. + */ + if( !isView ){ + sqlite3TableAffinity(v, pTab, regCols+1); + } + + /* Fire BEFORE or INSTEAD OF triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + pTab, regCols-pTab->nCol-1, onError, endOfLoop); + + sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); + } + + if( !isView ){ + if( IsVirtual(pTab) ){ + /* The row that the VUpdate opcode will delete: none */ + sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); + } + if( ipkColumn>=0 ){ + /* Compute the new rowid */ + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); + }else if( pSelect ){ + /* Rowid already initialized at tag-20191021-001 */ + }else{ + Expr *pIpk = pList->a[ipkColumn].pExpr; + if( pIpk->op==TK_NULL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + appendFlag = 1; + }else{ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); + } + } + /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid + ** to generate a unique primary key value. + */ + if( !appendFlag ){ + int addr1; + if( !IsVirtual(pTab) ){ + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + sqlite3VdbeJumpHere(v, addr1); + }else{ + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); + } + }else if( IsVirtual(pTab) || withoutRowid ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + appendFlag = 1; + } + autoIncStep(pParse, regAutoinc, regRowid); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** is derived from the INTEGER PRIMARY KEY. */ + if( pTab->tabFlags & TF_HasGenerated ){ + sqlite3ComputeGeneratedColumns(pParse, regRowid+1, pTab); + } +#endif + + /* Generate code to check constraints and generate index keys and + ** do the insertion. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + }else +#endif + { + int isReplace; /* Set to true if constraints may cause a replace */ + int bUseSeek; /* True to use OPFLAG_SEEKRESULT */ + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert + ); + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + + /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE + ** constraints or (b) there are no triggers and this table is not a + ** parent table in a foreign key constraint. It is safe to set the + ** flag in the second case as if any REPLACE constraint is hit, an + ** OP_Delete or OP_IdxDelete instruction will be executed on each + ** cursor that is disturbed. And these instructions both clear the + ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT + ** functionality. */ + bUseSeek = (isReplace==0 || !sqlite3VdbeHasSubProgram(v)); + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regIns, aRegIdx, 0, appendFlag, bUseSeek + ); + } + } + + /* Update the count of rows that are inserted + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } + + if( pTrigger ){ + /* Code AFTER triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + pTab, regData-2-pTab->nCol, onError, endOfLoop); + } + + /* The bottom of the main insertion loop, if the data source + ** is a SELECT statement. + */ + sqlite3VdbeResolveLabel(v, endOfLoop); + if( useTempTable ){ + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrInsTop); + sqlite3VdbeAddOp1(v, OP_Close, srcTab); + }else if( pSelect ){ + sqlite3VdbeGoto(v, addrCont); +#ifdef SQLITE_DEBUG + /* If we are jumping back to an OP_Yield that is preceded by an + ** OP_ReleaseReg, set the p5 flag on the OP_Goto so that the + ** OP_ReleaseReg will be included in the loop. */ + if( sqlite3VdbeGetOp(v, addrCont-1)->opcode==OP_ReleaseReg ){ + assert( sqlite3VdbeGetOp(v, addrCont)->opcode==OP_Yield ); + sqlite3VdbeChangeP5(v, 1); + } +#endif + sqlite3VdbeJumpHere(v, addrInsTop); + } + +#ifndef SQLITE_OMIT_XFER_OPT +insert_end: +#endif /* SQLITE_OMIT_XFER_OPT */ + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows inserted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_ChngCntRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + } + +insert_cleanup: + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pList); + sqlite3UpsertDelete(db, pUpsert); + sqlite3SelectDelete(db, pSelect); + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aRegIdx); +} + +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** they may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif +#ifdef tmask + #undef tmask +#endif + +/* +** Meanings of bits in of pWalker->eCode for +** sqlite3ExprReferencesUpdatedColumn() +*/ +#define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ +#define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ + +/* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). +* Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this +** expression node references any of the +** columns that are being modifed by an UPDATE statement. +*/ +static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + assert( pExpr->iColumn>=0 || pExpr->iColumn==-1 ); + if( pExpr->iColumn>=0 ){ + if( pWalker->u.aiCol[pExpr->iColumn]>=0 ){ + pWalker->eCode |= CKCNSTRNT_COLUMN; + } + }else{ + pWalker->eCode |= CKCNSTRNT_ROWID; + } + } + return WRC_Continue; +} + +/* +** pExpr is a CHECK constraint on a row that is being UPDATE-ed. The +** only columns that are modified by the UPDATE are those for which +** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. +** +** Return true if CHECK constraint pExpr uses any of the +** changing columns (or the rowid if it is changing). In other words, +** return true if this CHECK constraint must be validated for +** the new row in the UPDATE statement. +** +** 2018-09-15: pExpr might also be an expression for an index-on-expressions. +** The operation of this routine is the same - return true if an only if +** the expression uses one or more of columns identified by the second and +** third arguments. +*/ +int sqlite3ExprReferencesUpdatedColumn( + Expr *pExpr, /* The expression to be checked */ + int *aiChng, /* aiChng[x]>=0 if column x changed by the UPDATE */ + int chngRowid /* True if UPDATE changes the rowid */ +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 0; + w.xExprCallback = checkConstraintExprNode; + w.u.aiCol = aiChng; + sqlite3WalkExpr(&w, pExpr); + if( !chngRowid ){ + testcase( (w.eCode & CKCNSTRNT_ROWID)!=0 ); + w.eCode &= ~CKCNSTRNT_ROWID; + } + testcase( w.eCode==0 ); + testcase( w.eCode==CKCNSTRNT_COLUMN ); + testcase( w.eCode==CKCNSTRNT_ROWID ); + testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); + return w.eCode!=0; +} + +/* +** The sqlite3GenerateConstraintChecks() routine usually wants to visit +** the indexes of a table in the order provided in the Table->pIndex list. +** However, sometimes (rarely - when there is an upsert) it wants to visit +** the indexes in a different order. The following data structures accomplish +** this. +** +** The IndexIterator object is used to walk through all of the indexes +** of a table in either Index.pNext order, or in some other order established +** by an array of IndexListTerm objects. +*/ +typedef struct IndexListTerm IndexListTerm; +typedef struct IndexIterator IndexIterator; +struct IndexIterator { + int eType; /* 0 for Index.pNext list. 1 for an array of IndexListTerm */ + int i; /* Index of the current item from the list */ + union { + struct { /* Use this object for eType==0: A Index.pNext list */ + Index *pIdx; /* The current Index */ + } lx; + struct { /* Use this object for eType==1; Array of IndexListTerm */ + int nIdx; /* Size of the array */ + IndexListTerm *aIdx; /* Array of IndexListTerms */ + } ax; + } u; +}; + +/* When IndexIterator.eType==1, then each index is an array of instances +** of the following object +*/ +struct IndexListTerm { + Index *p; /* The index */ + int ix; /* Which entry in the original Table.pIndex list is this index*/ +}; + +/* Return the first index on the list */ +static Index *indexIteratorFirst(IndexIterator *pIter, int *pIx){ + assert( pIter->i==0 ); + if( pIter->eType ){ + *pIx = pIter->u.ax.aIdx[0].ix; + return pIter->u.ax.aIdx[0].p; + }else{ + *pIx = 0; + return pIter->u.lx.pIdx; + } +} + +/* Return the next index from the list. Return NULL when out of indexes */ +static Index *indexIteratorNext(IndexIterator *pIter, int *pIx){ + if( pIter->eType ){ + int i = ++pIter->i; + if( i>=pIter->u.ax.nIdx ){ + *pIx = i; + return 0; + } + *pIx = pIter->u.ax.aIdx[i].ix; + return pIter->u.ax.aIdx[i].p; + }else{ + ++(*pIx); + pIter->u.lx.pIdx = pIter->u.lx.pIdx->pNext; + return pIter->u.lx.pIdx; + } +} + +/* +** Generate code to do constraint checks prior to an INSERT or an UPDATE +** on table pTab. +** +** The regNewData parameter is the first register in a range that contains +** the data to be inserted or the data after the update. There will be +** pTab->nCol+1 registers in this range. The first register (the one +** that regNewData points to) will contain the new rowid, or NULL in the +** case of a WITHOUT ROWID table. The second register in the range will +** contain the content of the first table column. The third register will +** contain the content of the second table column. And so forth. +** +** The regOldData parameter is similar to regNewData except that it contains +** the data prior to an UPDATE rather than afterwards. regOldData is zero +** for an INSERT. This routine can distinguish between UPDATE and INSERT by +** checking regOldData for zero. +** +** For an UPDATE, the pkChng boolean is true if the true primary key (the +** rowid for a normal table or the PRIMARY KEY for a WITHOUT ROWID table) +** might be modified by the UPDATE. If pkChng is false, then the key of +** the iDataCur content table is guaranteed to be unchanged by the UPDATE. +** +** For an INSERT, the pkChng boolean indicates whether or not the rowid +** was explicitly specified as part of the INSERT statement. If pkChng +** is zero, it means that the either rowid is computed automatically or +** that the table is a WITHOUT ROWID table and has no rowid. On an INSERT, +** pkChng will only be true if the INSERT statement provides an integer +** value for either the rowid column or its INTEGER PRIMARY KEY alias. +** +** The code generated by this routine will store new index entries into +** registers identified by aRegIdx[]. No index entry is created for +** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is +** the same as the order of indices on the linked list of indices +** at pTab->pIndex. +** +** (2019-05-07) The generated code also creates a new record for the +** main table, if pTab is a rowid table, and stores that record in the +** register identified by aRegIdx[nIdx] - in other words in the first +** entry of aRegIdx[] past the last index. It is important that the +** record be generated during constraint checks to avoid affinity changes +** to the register content that occur after constraint checks but before +** the new record is inserted. +** +** The caller must have already opened writeable cursors on the main +** table and all applicable indices (that is to say, all indices for which +** aRegIdx[] is not zero). iDataCur is the cursor for the main table when +** inserting or updating a rowid table, or the cursor for the PRIMARY KEY +** index when operating on a WITHOUT ROWID table. iIdxCur is the cursor +** for the first index in the pTab->pIndex list. Cursors for other indices +** are at iIdxCur+N for the N-th element of the pTab->pIndex list. +** +** This routine also generates code to check constraints. NOT NULL, +** CHECK, and UNIQUE constraints are all checked. If a constraint fails, +** then the appropriate action is performed. There are five possible +** actions: ROLLBACK, ABORT, FAIL, REPLACE, and IGNORE. +** +** Constraint type Action What Happens +** --------------- ---------- ---------------------------------------- +** any ROLLBACK The current transaction is rolled back and +** sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. +** +** any ABORT Back out changes from the current command +** only (do not do a complete rollback) then +** cause sqlite3_step() to return immediately +** with SQLITE_CONSTRAINT. +** +** any FAIL Sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. The +** transaction is not rolled back and any +** changes to prior rows are retained. +** +** any IGNORE The attempt in insert or update the current +** row is skipped, without throwing an error. +** Processing continues with the next row. +** (There is an immediate jump to ignoreDest.) +** +** NOT NULL REPLACE The NULL value is replace by the default +** value for that column. If the default value +** is NULL, the action is the same as ABORT. +** +** UNIQUE REPLACE The other row that conflicts with the row +** being inserted is removed. +** +** CHECK REPLACE Illegal. The results in an exception. +** +** Which action to take is determined by the overrideError parameter. +** Or if overrideError==OE_Default, then the pParse->onError parameter +** is used. Or if pParse->onError==OE_Default then the onError value +** for the constraint is used. +*/ +void sqlite3GenerateConstraintChecks( + Parse *pParse, /* The parser context */ + Table *pTab, /* The table being inserted or updated */ + int *aRegIdx, /* Use register aRegIdx[i] for index i. 0 for unused */ + int iDataCur, /* Canonical data cursor (main table or PK index) */ + int iIdxCur, /* First index cursor */ + int regNewData, /* First register in a range holding values to insert */ + int regOldData, /* Previous content. 0 for INSERTs */ + u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ + u8 overrideError, /* Override onError to this if not OE_Default */ + int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ + int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ + int *aiChng, /* column i is unchanged if aiChng[i]<0 */ + Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ +){ + Vdbe *v; /* VDBE under constrution */ + Index *pIdx; /* Pointer to one of the indices */ + Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ + sqlite3 *db; /* Database connection */ + int i; /* loop counter */ + int ix; /* Index loop counter */ + int nCol; /* Number of columns */ + int onError; /* Conflict resolution strategy */ + int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ + int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ + Upsert *pUpsertClause = 0; /* The specific ON CONFLICT clause for pIdx */ + u8 isUpdate; /* True if this is an UPDATE operation */ + u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ + int upsertIpkReturn = 0; /* Address of Goto at end of IPK uniqueness check */ + int upsertIpkDelay = 0; /* Address of Goto to bypass initial IPK check */ + int ipkTop = 0; /* Top of the IPK uniqueness check */ + int ipkBottom = 0; /* OP_Goto at the end of the IPK uniqueness check */ + /* Variables associated with retesting uniqueness constraints after + ** replace triggers fire have run */ + int regTrigCnt; /* Register used to count replace trigger invocations */ + int addrRecheck = 0; /* Jump here to recheck all uniqueness constraints */ + int lblRecheckOk = 0; /* Each recheck jumps to this label if it passes */ + Trigger *pTrigger; /* List of DELETE triggers on the table pTab */ + int nReplaceTrig = 0; /* Number of replace triggers coded */ + IndexIterator sIdxIter; /* Index iterator */ + + isUpdate = regOldData!=0; + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + nCol = pTab->nCol; + + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for + ** normal rowid tables. nPkField is the number of key fields in the + ** pPk index or 1 for a rowid table. In other words, nPkField is the + ** number of fields in the true primary key of the table. */ + if( HasRowid(pTab) ){ + pPk = 0; + nPkField = 1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + nPkField = pPk->nKeyCol; + } + + /* Record that this module has started */ + VdbeModuleComment((v, "BEGIN: GenCnstCks(%d,%d,%d,%d,%d)", + iDataCur, iIdxCur, regNewData, regOldData, pkChng)); + + /* Test all NOT NULL constraints. + */ + if( pTab->tabFlags & TF_HasNotNull ){ + int b2ndPass = 0; /* True if currently running 2nd pass */ + int nSeenReplace = 0; /* Number of ON CONFLICT REPLACE operations */ + int nGenerated = 0; /* Number of generated columns with NOT NULL */ + while(1){ /* Make 2 passes over columns. Exit loop via "break" */ + for(i=0; iaCol[i]; /* The column to check for NOT NULL */ + int isGenerated; /* non-zero if column is generated */ + onError = pCol->notNull; + if( onError==OE_None ) continue; /* No NOT NULL on this column */ + if( i==pTab->iPKey ){ + continue; /* ROWID is never NULL */ + } + isGenerated = pCol->colFlags & COLFLAG_GENERATED; + if( isGenerated && !b2ndPass ){ + nGenerated++; + continue; /* Generated columns processed on 2nd pass */ + } + if( aiChng && aiChng[i]<0 && !isGenerated ){ + /* Do not check NOT NULL on columns that do not change */ + continue; + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace ){ + if( b2ndPass /* REPLACE becomes ABORT on the 2nd pass */ + || pCol->pDflt==0 /* REPLACE is ABORT if no DEFAULT value */ + ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + testcase( pCol->colFlags & COLFLAG_GENERATED ); + onError = OE_Abort; + }else{ + assert( !isGenerated ); + } + }else if( b2ndPass && !isGenerated ){ + continue; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + testcase( i!=sqlite3TableColumnToStorage(pTab, i) ); + iReg = sqlite3TableColumnToStorage(pTab, i) + regNewData + 1; + switch( onError ){ + case OE_Replace: { + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, iReg); + VdbeCoverage(v); + assert( (pCol->colFlags & COLFLAG_GENERATED)==0 ); + nSeenReplace++; + sqlite3ExprCodeCopy(pParse, pCol->pDflt, iReg); + sqlite3VdbeJumpHere(v, addr1); + break; + } + case OE_Abort: + sqlite3MayAbort(pParse); + /* no break */ deliberate_fall_through + case OE_Rollback: + case OE_Fail: { + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pCol->zName); + sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, + onError, iReg); + sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); + break; + } + default: { + assert( onError==OE_Ignore ); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, ignoreDest); + VdbeCoverage(v); + break; + } + } /* end switch(onError) */ + } /* end loop i over columns */ + if( nGenerated==0 && nSeenReplace==0 ){ + /* If there are no generated columns with NOT NULL constraints + ** and no NOT NULL ON CONFLICT REPLACE constraints, then a single + ** pass is sufficient */ + break; + } + if( b2ndPass ) break; /* Never need more than 2 passes */ + b2ndPass = 1; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( nSeenReplace>0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + /* If any NOT NULL ON CONFLICT REPLACE constraints fired on the + ** first pass, recomputed values for all generated columns, as + ** those values might depend on columns affected by the REPLACE. + */ + sqlite3ComputeGeneratedColumns(pParse, regNewData+1, pTab); + } +#endif + } /* end of 2-pass loop */ + } /* end if( has-not-null-constraints ) */ + + /* Test all CHECK constraints + */ +#ifndef SQLITE_OMIT_CHECK + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + pParse->iSelfTab = -(regNewData+1); + onError = overrideError!=OE_Default ? overrideError : OE_Abort; + for(i=0; inExpr; i++){ + int allOk; + Expr *pCopy; + Expr *pExpr = pCheck->a[i].pExpr; + if( aiChng + && !sqlite3ExprReferencesUpdatedColumn(pExpr, aiChng, pkChng) + ){ + /* The check constraints do not reference any of the columns being + ** updated so there is no point it verifying the check constraint */ + continue; + } + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + allOk = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeVerifyAbortable(v, onError); + pCopy = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ){ + sqlite3ExprIfTrue(pParse, pCopy, allOk, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pCopy); + if( onError==OE_Ignore ){ + sqlite3VdbeGoto(v, ignoreDest); + }else{ + char *zName = pCheck->a[i].zEName; + assert( zName!=0 || pParse->db->mallocFailed ); + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */ + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zName, P4_TRANSIENT, + P5_ConstraintCheck); + } + sqlite3VdbeResolveLabel(v, allOk); + } + pParse->iSelfTab = 0; + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ + + /* UNIQUE and PRIMARY KEY constraints should be handled in the following + ** order: + ** + ** (1) OE_Update + ** (2) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore + ** (3) OE_Replace + ** + ** OE_Fail and OE_Ignore must happen before any changes are made. + ** OE_Update guarantees that only a single row will change, so it + ** must happen before OE_Replace. Technically, OE_Abort and OE_Rollback + ** could happen in any order, but they are grouped up front for + ** convenience. + ** + ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** The order of constraints used to have OE_Update as (2) and OE_Abort + ** and so forth as (1). But apparently PostgreSQL checks the OE_Update + ** constraint before any others, so it had to be moved. + ** + ** Constraint checking code is generated in this order: + ** (A) The rowid constraint + ** (B) Unique index constraints that do not have OE_Replace as their + ** default conflict resolution strategy + ** (C) Unique index that do use OE_Replace by default. + ** + ** The ordering of (2) and (3) is accomplished by making sure the linked + ** list of indexes attached to a table puts all OE_Replace indexes last + ** in the list. See sqlite3CreateIndex() for where that happens. + */ + sIdxIter.eType = 0; + sIdxIter.i = 0; + sIdxIter.u.ax.aIdx = 0; /* Silence harmless compiler warning */ + sIdxIter.u.lx.pIdx = pTab->pIndex; + if( pUpsert ){ + if( pUpsert->pUpsertTarget==0 ){ + /* There is just on ON CONFLICT clause and it has no constraint-target */ + assert( pUpsert->pNextUpsert==0 ); + if( pUpsert->isDoUpdate==0 ){ + /* A single ON CONFLICT DO NOTHING clause, without a constraint-target. + ** Make all unique constraint resolution be OE_Ignore */ + overrideError = OE_Ignore; + pUpsert = 0; + }else{ + /* A single ON CONFLICT DO UPDATE. Make all resolutions OE_Update */ + overrideError = OE_Update; + } + }else if( pTab->pIndex!=0 ){ + /* Otherwise, we'll need to run the IndexListTerm array version of the + ** iterator to ensure that all of the ON CONFLICT conditions are + ** checked first and in order. */ + int nIdx, jj; + u64 nByte; + Upsert *pTerm; + u8 *bUsed; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + assert( aRegIdx[nIdx]>0 ); + } + sIdxIter.eType = 1; + sIdxIter.u.ax.nIdx = nIdx; + nByte = (sizeof(IndexListTerm)+1)*nIdx + nIdx; + sIdxIter.u.ax.aIdx = sqlite3DbMallocZero(db, nByte); + if( sIdxIter.u.ax.aIdx==0 ) return; /* OOM */ + bUsed = (u8*)&sIdxIter.u.ax.aIdx[nIdx]; + pUpsert->pToFree = sIdxIter.u.ax.aIdx; + for(i=0, pTerm=pUpsert; pTerm; pTerm=pTerm->pNextUpsert){ + if( pTerm->pUpsertTarget==0 ) break; + if( pTerm->pUpsertIdx==0 ) continue; /* Skip ON CONFLICT for the IPK */ + jj = 0; + pIdx = pTab->pIndex; + while( ALWAYS(pIdx!=0) && pIdx!=pTerm->pUpsertIdx ){ + pIdx = pIdx->pNext; + jj++; + } + if( bUsed[jj] ) continue; /* Duplicate ON CONFLICT clause ignored */ + bUsed[jj] = 1; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + for(jj=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, jj++){ + if( bUsed[jj] ) continue; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + assert( i==nIdx ); + } + } + + /* Determine if it is possible that triggers (either explicitly coded + ** triggers or FK resolution actions) might run as a result of deletes + ** that happen when OE_Replace conflict resolution occurs. (Call these + ** "replace triggers".) If any replace triggers run, we will need to + ** recheck all of the uniqueness constraints after they have all run. + ** But on the recheck, the resolution is OE_Abort instead of OE_Replace. + ** + ** If replace triggers are a possibility, then + ** + ** (1) Allocate register regTrigCnt and initialize it to zero. + ** That register will count the number of replace triggers that + ** fire. Constraint recheck only occurs if the number is positive. + ** (2) Initialize pTrigger to the list of all DELETE triggers on pTab. + ** (3) Initialize addrRecheck and lblRecheckOk + ** + ** The uniqueness rechecking code will create a series of tests to run + ** in a second pass. The addrRecheck and lblRecheckOk variables are + ** used to link together these tests which are separated from each other + ** in the generate bytecode. + */ + if( (db->flags & (SQLITE_RecTriggers|SQLITE_ForeignKeys))==0 ){ + /* There are not DELETE triggers nor FK constraints. No constraint + ** rechecks are needed. */ + pTrigger = 0; + regTrigCnt = 0; + }else{ + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + regTrigCnt = pTrigger!=0 || sqlite3FkRequired(pParse, pTab, 0, 0); + }else{ + pTrigger = 0; + regTrigCnt = sqlite3FkRequired(pParse, pTab, 0, 0); + } + if( regTrigCnt ){ + /* Replace triggers might exist. Allocate the counter and + ** initialize it to zero. */ + regTrigCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regTrigCnt); + VdbeComment((v, "trigger count")); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + addrRecheck = lblRecheckOk; + } + } + + /* If rowid is changing, make sure the new rowid does not previously + ** exist in the table. + */ + if( pkChng && pPk==0 ){ + int addrRowidOk = sqlite3VdbeMakeLabel(pParse); + + /* Figure out what action to take in case of a rowid collision */ + onError = pTab->keyConf; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + /* figure out whether or not upsert applies in this case */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert,0); + if( pUpsertClause!=0 ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + if( pUpsertClause!=pUpsert ){ + /* The first ON CONFLICT clause has a conflict target other than + ** the IPK. We have to jump ahead to that first ON CONFLICT clause + ** and then come back here and deal with the IPK afterwards */ + upsertIpkDelay = sqlite3VdbeAddOp0(v, OP_Goto); + } + } + + /* If the response to a rowid conflict is REPLACE but the response + ** to some other UNIQUE constraint is FAIL or IGNORE, then we need + ** to defer the running of the rowid conflict checking until after + ** the UNIQUE constraints have run. + */ + if( onError==OE_Replace /* IPK rule is REPLACE */ + && onError!=overrideError /* Rules for other constraints are different */ + && pTab->pIndex /* There exist other constraints */ + ){ + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + VdbeComment((v, "defer IPK REPLACE until last")); + } + + if( isUpdate ){ + /* pkChng!=0 does not mean that the rowid has changed, only that + ** it might have changed. Skip the conflict logic below if the rowid + ** is unchanged. */ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + + /* Check to see if the new rowid already exists in the table. Skip + ** the following conflict logic if it does not. */ + VdbeNoopComment((v, "uniqueness check for ROWID")); + sqlite3VdbeVerifyAbortable(v, onError); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); + VdbeCoverage(v); + + switch( onError ){ + default: { + onError = OE_Abort; + /* no break */ deliberate_fall_through + } + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); + sqlite3RowidConstraint(pParse, onError, pTab); + break; + } + case OE_Replace: { + /* If there are DELETE triggers on this table and the + ** recursive-triggers flag is set, call GenerateRowDelete() to + ** remove the conflicting row from the table. This will fire + ** the triggers and remove both the table and index b-tree entries. + ** + ** Otherwise, if there are no triggers or the recursive-triggers + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. + ** + ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, + ** also invoke MultiWrite() to indicate that this VDBE may require + ** statement rollback (if the statement is aborted after the delete + ** takes place). Earlier versions called sqlite3MultiWrite() regardless, + ** but being more selective here allows statements like: + ** + ** REPLACE INTO t(rowid) VALUES($newrowid) + ** + ** to run without a statement journal if there are no indexes on the + ** table. + */ + if( regTrigCnt ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regNewData, 1, 0, OE_Replace, 1, -1); + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + nReplaceTrig++; + }else{ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + assert( HasRowid(pTab) ); + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } + } + seenReplace = 1; + break; + } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur); + /* no break */ deliberate_fall_through + } +#endif + case OE_Ignore: { + testcase( onError==OE_Ignore ); + sqlite3VdbeGoto(v, ignoreDest); + break; + } + } + sqlite3VdbeResolveLabel(v, addrRowidOk); + if( pUpsert && pUpsertClause!=pUpsert ){ + upsertIpkReturn = sqlite3VdbeAddOp0(v, OP_Goto); + }else if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop-1); + } + } + + /* Test all UNIQUE constraints by creating entries for each UNIQUE + ** index and making sure that duplicate entries do not already exist. + ** Compute the revised record entries for indices as we go. + ** + ** This loop also handles the case of the PRIMARY KEY index for a + ** WITHOUT ROWID table. + */ + for(pIdx = indexIteratorFirst(&sIdxIter, &ix); + pIdx; + pIdx = indexIteratorNext(&sIdxIter, &ix) + ){ + int regIdx; /* Range of registers hold conent for pIdx */ + int regR; /* Range of registers holding conflicting PK */ + int iThisCur; /* Cursor for this UNIQUE index */ + int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + int addrConflictCk; /* First opcode in the conflict check logic */ + + if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert, pIdx); + if( upsertIpkDelay && pUpsertClause==pUpsert ){ + sqlite3VdbeJumpHere(v, upsertIpkDelay); + } + } + addrUniqueOk = sqlite3VdbeMakeLabel(pParse); + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + VdbeNoopComment((v, "prep index %s", pIdx->zName)); + iThisCur = iIdxCur+ix; + + + /* Skip partial indices for which the WHERE clause is not true */ + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); + pParse->iSelfTab = -(regNewData+1); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; + } + + /* Create a record for this index entry as it should appear after + ** the insert or update. Store that record in the aRegIdx[ix] register + */ + regIdx = aRegIdx[ix]+1; + for(i=0; inColumn; i++){ + int iField = pIdx->aiColumn[i]; + int x; + if( iField==XN_EXPR ){ + pParse->iSelfTab = -(regNewData+1); + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->iSelfTab = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); + }else if( iField==XN_ROWID || iField==pTab->iPKey ){ + x = regNewData; + sqlite3VdbeAddOp2(v, OP_IntCopy, x, regIdx+i); + VdbeComment((v, "rowid")); + }else{ + testcase( sqlite3TableColumnToStorage(pTab, iField)!=iField ); + x = sqlite3TableColumnToStorage(pTab, iField) + regNewData + 1; + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", pTab->aCol[iField].zName)); + } + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); + VdbeComment((v, "for %s", pIdx->zName)); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + sqlite3SetMakeRecordP5(v, pIdx->pTable); + } +#endif + sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0); + + /* In an UPDATE operation, if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table and there has been no change the + ** primary key, then no collision is possible. The collision detection + ** logic below can all be skipped. */ + if( isUpdate && pPk==pIdx && pkChng==0 ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } + + /* Find out what action to take in case there is a uniqueness conflict */ + onError = pIdx->onError; + if( onError==OE_None ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; /* pIdx is not a UNIQUE index */ + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + /* Figure out if the upsert clause applies to this index */ + if( pUpsertClause ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + + /* Collision detection may be omitted if all of the following are true: + ** (1) The conflict resolution algorithm is REPLACE + ** (2) The table is a WITHOUT ROWID table + ** (3) There are no secondary indexes on the table + ** (4) No delete triggers need to be fired if there is a conflict + ** (5) No FK constraint counters need to be updated if a conflict occurs. + ** + ** This is not possible for ENABLE_PREUPDATE_HOOK builds, as the row + ** must be explicitly deleted in order to ensure any pre-update hook + ** is invoked. */ +#ifndef SQLITE_ENABLE_PREUPDATE_HOOK + if( (ix==0 && pIdx->pNext==0) /* Condition 3 */ + && pPk==pIdx /* Condition 2 */ + && onError==OE_Replace /* Condition 1 */ + && ( 0==(db->flags&SQLITE_RecTriggers) || /* Condition 4 */ + 0==sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0)) + && ( 0==(db->flags&SQLITE_ForeignKeys) || /* Condition 5 */ + (0==pTab->pFKey && 0==sqlite3FkReferences(pTab))) + ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } +#endif /* ifndef SQLITE_ENABLE_PREUPDATE_HOOK */ + + /* Check to see if the new index entry will be unique */ + sqlite3VdbeVerifyAbortable(v, onError); + addrConflictCk = + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); + + /* Generate code to handle collisions */ + regR = pIdx==pPk ? regIdx : sqlite3GetTempRange(pParse, nPkField); + if( isUpdate || onError==OE_Replace ){ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iThisCur, regR); + /* Conflict only if the rowid of the existing index entry + ** is different from old-rowid */ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regR, addrUniqueOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + }else{ + int x; + /* Extract the PRIMARY KEY from the end of the index entry and + ** store it in registers regR..regR+nPk-1 */ + if( pIdx!=pPk ){ + for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); + x = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); + VdbeComment((v, "%s.%s", pTab->zName, + pTab->aCol[pPk->aiColumn[i]].zName)); + } + } + if( isUpdate ){ + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + ** table, only conflict if the new PRIMARY KEY values are actually + ** different from the old. + ** + ** For a UNIQUE index, only conflict if the PRIMARY KEY values + ** of the matched index row are different from the original PRIMARY + ** KEY values of this row before the update. */ + int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; + int op = OP_Ne; + int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); + + for(i=0; inKeyCol; i++){ + char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); + x = pPk->aiColumn[i]; + assert( x>=0 ); + if( i==(pPk->nKeyCol-1) ){ + addrJump = addrUniqueOk; + op = OP_Eq; + } + x = sqlite3TableColumnToStorage(pTab, x); + sqlite3VdbeAddOp4(v, op, + regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ + ); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverageIf(v, op==OP_Eq); + VdbeCoverageIf(v, op==OP_Ne); + } + } + } + } + + /* Generate code that executes if the new index entry is not unique */ + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update ); + switch( onError ){ + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); + sqlite3UniqueConstraint(pParse, onError, pIdx); + break; + } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix); + /* no break */ deliberate_fall_through + } +#endif + case OE_Ignore: { + testcase( onError==OE_Ignore ); + sqlite3VdbeGoto(v, ignoreDest); + break; + } + default: { + int nConflictCk; /* Number of opcodes in conflict check logic */ + + assert( onError==OE_Replace ); + nConflictCk = sqlite3VdbeCurrentAddr(v) - addrConflictCk; + assert( nConflictCk>0 ); + testcase( nConflictCk>1 ); + if( regTrigCnt ){ + sqlite3MultiWrite(pParse); + nReplaceTrig++; + } + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorLock, iDataCur); + } + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorUnlock, iDataCur); + } + if( regTrigCnt ){ + int addrBypass; /* Jump destination to bypass recheck logic */ + + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + addrBypass = sqlite3VdbeAddOp0(v, OP_Goto); /* Bypass recheck */ + VdbeComment((v, "bypass recheck")); + + /* Here we insert code that will be invoked after all constraint + ** checks have run, if and only if one or more replace triggers + ** fired. */ + sqlite3VdbeResolveLabel(v, lblRecheckOk); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + if( pIdx->pPartIdxWhere ){ + /* Bypass the recheck if this partial index is not defined + ** for the current row */ + sqlite3VdbeAddOp2(v, OP_IsNull, regIdx-1, lblRecheckOk); + VdbeCoverage(v); + } + /* Copy the constraint check code from above, except change + ** the constraint-ok jump destination to be the address of + ** the next retest block */ + while( nConflictCk>0 ){ + VdbeOp x; /* Conflict check opcode to copy */ + /* The sqlite3VdbeAddOp4() call might reallocate the opcode array. + ** Hence, make a complete copy of the opcode, rather than using + ** a pointer to the opcode. */ + x = *sqlite3VdbeGetOp(v, addrConflictCk); + if( x.opcode!=OP_IdxRowid ){ + int p2; /* New P2 value for copied conflict check opcode */ + const char *zP4; + if( sqlite3OpcodeProperty[x.opcode]&OPFLG_JUMP ){ + p2 = lblRecheckOk; + }else{ + p2 = x.p2; + } + zP4 = x.p4type==P4_INT32 ? SQLITE_INT_TO_PTR(x.p4.i) : x.p4.z; + sqlite3VdbeAddOp4(v, x.opcode, x.p1, p2, x.p3, zP4, x.p4type); + sqlite3VdbeChangeP5(v, x.p5); + VdbeCoverageIf(v, p2!=x.p2); + } + nConflictCk--; + addrConflictCk++; + } + /* If the retest fails, issue an abort */ + sqlite3UniqueConstraint(pParse, OE_Abort, pIdx); + + sqlite3VdbeJumpHere(v, addrBypass); /* Terminate the recheck bypass */ + } + seenReplace = 1; + break; + } + } + sqlite3VdbeResolveLabel(v, addrUniqueOk); + if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + if( pUpsertClause + && upsertIpkReturn + && sqlite3UpsertNextIsIPK(pUpsertClause) + ){ + sqlite3VdbeGoto(v, upsertIpkDelay+1); + sqlite3VdbeJumpHere(v, upsertIpkReturn); + upsertIpkReturn = 0; + } + } + + /* If the IPK constraint is a REPLACE, run it last */ + if( ipkTop ){ + sqlite3VdbeGoto(v, ipkTop); + VdbeComment((v, "Do IPK REPLACE")); + sqlite3VdbeJumpHere(v, ipkBottom); + } + + /* Recheck all uniqueness constraints after replace triggers have run */ + testcase( regTrigCnt!=0 && nReplaceTrig==0 ); + assert( regTrigCnt!=0 || nReplaceTrig==0 ); + if( nReplaceTrig ){ + sqlite3VdbeAddOp2(v, OP_IfNot, regTrigCnt, lblRecheckOk);VdbeCoverage(v); + if( !pPk ){ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRecheck, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRecheck, regNewData); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, OE_Abort, pTab); + }else{ + sqlite3VdbeGoto(v, addrRecheck); + } + sqlite3VdbeResolveLabel(v, lblRecheckOk); + } + + /* Generate the table record */ + if( HasRowid(pTab) ){ + int regRec = aRegIdx[ix]; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nNVCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + } + } + + *pbMayReplace = seenReplace; + VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); +} + +#ifdef SQLITE_ENABLE_NULL_TRIM +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].pDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); +} +#endif + +/* +** Table pTab is a WITHOUT ROWID table that is being written to. The cursor +** number is iCur, and register regData contains the new record for the +** PK index. This function adds code to invoke the pre-update hook, +** if one is registered. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void codeWithoutRowidPreupdate( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated */ + int iCur, /* Cursor number for table */ + int regData /* Data containing new record */ +){ + Vdbe *v = pParse->pVdbe; + int r = sqlite3GetTempReg(pParse); + assert( !HasRowid(pTab) ); + assert( 0==(pParse->db->mDbFlags & DBFLAG_Vacuum) || CORRUPT_DB ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, iCur, regData, r, (char*)pTab, P4_TABLE); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); +} +#else +# define codeWithoutRowidPreupdate(a,b,c,d) +#endif + +/* +** This routine generates code to finish the INSERT or UPDATE operation +** that was started by a prior call to sqlite3GenerateConstraintChecks. +** A consecutive range of registers starting at regNewData contains the +** rowid and the content to be inserted. +** +** The arguments to this routine should be the same as the first six +** arguments to sqlite3GenerateConstraintChecks. +*/ +void sqlite3CompleteInsertion( + Parse *pParse, /* The parser context */ + Table *pTab, /* the table into which we are inserting */ + int iDataCur, /* Cursor of the canonical data source */ + int iIdxCur, /* First index cursor */ + int regNewData, /* Range of content */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int update_flags, /* True for UPDATE, False for INSERT */ + int appendBias, /* True if this is likely to be an append */ + int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ +){ + Vdbe *v; /* Prepared statements under construction */ + Index *pIdx; /* An index being inserted or updated */ + u8 pik_flags; /* flag values passed to the btree insert */ + int i; /* Loop counter */ + + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + + v = pParse->pVdbe; + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + /* All REPLACE indexes are at the end of the list */ + assert( pIdx->onError!=OE_Replace + || pIdx->pNext==0 + || pIdx->pNext->onError==OE_Replace ); + if( aRegIdx[i]==0 ) continue; + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + } + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + assert( pParse->nested==0 ); + pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); + if( update_flags==0 ){ + codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i]); + } + } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); + sqlite3VdbeChangeP5(v, pik_flags); + } + if( !HasRowid(pTab) ) return; + if( pParse->nested ){ + pik_flags = 0; + }else{ + pik_flags = OPFLAG_NCHANGE; + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); + } + if( appendBias ){ + pik_flags |= OPFLAG_APPEND; + } + if( useSeekResult ){ + pik_flags |= OPFLAG_USESEEKRESULT; + } + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, aRegIdx[i], regNewData); + if( !pParse->nested ){ + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } + sqlite3VdbeChangeP5(v, pik_flags); +} + +/* +** Allocate cursors for the pTab table and all its indices and generate +** code to open and initialized those cursors. +** +** The cursor for the object that contains the complete data (normally +** the table itself, but the PRIMARY KEY index in the case of a WITHOUT +** ROWID table) is returned in *piDataCur. The first index cursor is +** returned in *piIdxCur. The number of indices is returned. +** +** Use iBase as the first cursor (either the *piDataCur for rowid tables +** or the first index for WITHOUT ROWID tables) if it is non-negative. +** If iBase is negative, then allocate the next available cursor. +** +** For a rowid table, *piDataCur will be exactly one less than *piIdxCur. +** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range +** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the +** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. +*/ +int sqlite3OpenTableAndIndices( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table to be opened */ + int op, /* OP_OpenRead or OP_OpenWrite */ + u8 p5, /* P5 value for OP_Open* opcodes (except on WITHOUT ROWID) */ + int iBase, /* Use this for the table cursor, if there is one */ + u8 *aToOpen, /* If not NULL: boolean for each table and index */ + int *piDataCur, /* Write the database source cursor number here */ + int *piIdxCur /* Write the first index cursor number here */ +){ + int i; + int iDb; + int iDataCur; + Index *pIdx; + Vdbe *v; + + assert( op==OP_OpenRead || op==OP_OpenWrite ); + assert( op==OP_OpenWrite || p5==0 ); + if( IsVirtual(pTab) ){ + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ + return 0; + } + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + v = pParse->pVdbe; + assert( v!=0 ); + if( iBase<0 ) iBase = pParse->nTab; + iDataCur = iBase++; + if( piDataCur ) *piDataCur = iDataCur; + if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ + sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); + } + if( piIdxCur ) *piIdxCur = iBase; + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + int iIdxCur = iBase++; + assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + p5 = 0; + } + if( aToOpen==0 || aToOpen[i+1] ){ + sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + sqlite3VdbeChangeP5(v, p5); + VdbeComment((v, "%s", pIdx->zName)); + } + } + if( iBase>pParse->nTab ) pParse->nTab = iBase; + return i; +} + + +#ifdef SQLITE_TEST +/* +** The following global variable is incremented whenever the +** transfer optimization is used. This is used for testing +** purposes only - to make sure the transfer optimization really +** is happening when it is supposed to. +*/ +int sqlite3_xferopt_count; +#endif /* SQLITE_TEST */ + + +#ifndef SQLITE_OMIT_XFER_OPT +/* +** Check to see if index pSrc is compatible as a source of data +** for index pDest in an insert transfer optimization. The rules +** for a compatible index: +** +** * The index is over the same set of columns +** * The same DESC and ASC markings occurs on all columns +** * The same onError processing (OE_Abort, OE_Ignore, etc) +** * The same collating sequence on each column +** * The index has the exact same WHERE clause +*/ +static int xferCompatibleIndex(Index *pDest, Index *pSrc){ + int i; + assert( pDest && pSrc ); + assert( pDest->pTable!=pSrc->pTable ); + if( pDest->nKeyCol!=pSrc->nKeyCol || pDest->nColumn!=pSrc->nColumn ){ + return 0; /* Different number of columns */ + } + if( pDest->onError!=pSrc->onError ){ + return 0; /* Different conflict resolution strategies */ + } + for(i=0; inKeyCol; i++){ + if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ + return 0; /* Different columns indexed */ + } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(0, pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } + if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ + return 0; /* Different sort orders */ + } + if( sqlite3_stricmp(pSrc->azColl[i],pDest->azColl[i])!=0 ){ + return 0; /* Different collating sequences */ + } + } + if( sqlite3ExprCompare(0, pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ + return 0; /* Different WHERE clauses */ + } + + /* If no test above fails then the indices must be compatible */ + return 1; +} + +/* +** Attempt the transfer optimization on INSERTs of the form +** +** INSERT INTO tab1 SELECT * FROM tab2; +** +** The xfer optimization transfers raw records from tab2 over to tab1. +** Columns are not decoded and reassembled, which greatly improves +** performance. Raw index records are transferred in the same way. +** +** The xfer optimization is only attempted if tab1 and tab2 are compatible. +** There are lots of rules for determining compatibility - see comments +** embedded in the code for details. +** +** This routine returns TRUE if the optimization is guaranteed to be used. +** Sometimes the xfer optimization will only work if the destination table +** is empty - a factor that can only be determined at run-time. In that +** case, this routine generates code for the xfer optimization but also +** does a test to see if the destination table is empty and jumps over the +** xfer optimization code if the test fails. In that case, this routine +** returns FALSE so that the caller will know to go ahead and generate +** an unoptimized transfer. This routine also returns FALSE if there +** is no chance that the xfer optimization can be applied. +** +** This optimization is particularly useful at making VACUUM run faster. +*/ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +){ + sqlite3 *db = pParse->db; + ExprList *pEList; /* The result set of the SELECT */ + Table *pSrc; /* The table in the FROM clause of SELECT */ + Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ + SrcItem *pItem; /* An element of pSelect->pSrc */ + int i; /* Loop counter */ + int iDbSrc; /* The database of pSrc */ + int iSrc, iDest; /* Cursors from source and destination */ + int addr1, addr2; /* Loop addresses */ + int emptyDestTest = 0; /* Address of test for empty pDest */ + int emptySrcTest = 0; /* Address of test for empty pSrc */ + Vdbe *v; /* The VDBE we are building */ + int regAutoinc; /* Memory register used by AUTOINC */ + int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ + int regData, regRowid; /* Registers holding data and rowid */ + + if( pSelect==0 ){ + return 0; /* Must be of the form INSERT INTO ... SELECT ... */ + } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; + } + if( sqlite3TriggerList(pParse, pDest) ){ + return 0; /* tab1 must not have triggers */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pDest) ){ + return 0; /* tab1 must not be a virtual table */ + } +#endif + if( onError==OE_Default ){ + if( pDest->iPKey>=0 ) onError = pDest->keyConf; + if( onError==OE_Default ) onError = OE_Abort; + } + assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ + if( pSelect->pSrc->nSrc!=1 ){ + return 0; /* FROM clause must have exactly one term */ + } + if( pSelect->pSrc->a[0].pSelect ){ + return 0; /* FROM clause cannot contain a subquery */ + } + if( pSelect->pWhere ){ + return 0; /* SELECT may not have a WHERE clause */ + } + if( pSelect->pOrderBy ){ + return 0; /* SELECT may not have an ORDER BY clause */ + } + /* Do not need to test for a HAVING clause. If HAVING is present but + ** there is no ORDER BY, we will get an error. */ + if( pSelect->pGroupBy ){ + return 0; /* SELECT may not have a GROUP BY clause */ + } + if( pSelect->pLimit ){ + return 0; /* SELECT may not have a LIMIT clause */ + } + if( pSelect->pPrior ){ + return 0; /* SELECT may not be a compound query */ + } + if( pSelect->selFlags & SF_Distinct ){ + return 0; /* SELECT may not be DISTINCT */ + } + pEList = pSelect->pEList; + assert( pEList!=0 ); + if( pEList->nExpr!=1 ){ + return 0; /* The result set must have exactly one column */ + } + assert( pEList->a[0].pExpr ); + if( pEList->a[0].pExpr->op!=TK_ASTERISK ){ + return 0; /* The result set must be the special operator "*" */ + } + + /* At this point we have established that the statement is of the + ** correct syntactic form to participate in this optimization. Now + ** we have to check the semantics. + */ + pItem = pSelect->pSrc->a; + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); + if( pSrc==0 ){ + return 0; /* FROM clause does not contain a real table */ + } + if( pSrc->tnum==pDest->tnum && pSrc->pSchema==pDest->pSchema ){ + testcase( pSrc!=pDest ); /* Possible due to bad sqlite_schema.rootpage */ + return 0; /* tab1 and tab2 may not be the same table */ + } + if( HasRowid(pDest)!=HasRowid(pSrc) ){ + return 0; /* source and destination must both be WITHOUT ROWID or not */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pSrc) ){ + return 0; /* tab2 must not be a virtual table */ + } +#endif + if( pSrc->pSelect ){ + return 0; /* tab2 may not be a view */ + } + if( pDest->nCol!=pSrc->nCol ){ + return 0; /* Number of columns must be the same in tab1 and tab2 */ + } + if( pDest->iPKey!=pSrc->iPKey ){ + return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ + } + for(i=0; inCol; i++){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + ){ + return 0; /* Neither table may have __hidden__ columns */ + } +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Even if tables t1 and t2 have identical schemas, if they contain + ** generated columns, then this statement is semantically incorrect: + ** + ** INSERT INTO t2 SELECT * FROM t1; + ** + ** The reason is that generated column values are returned by the + ** the SELECT statement on the right but the INSERT statement on the + ** left wants them to be omitted. + ** + ** Nevertheless, this is a useful notational shorthand to tell SQLite + ** to do a bulk transfer all of the content from t1 over to t2. + ** + ** We could, in theory, disable this (except for internal use by the + ** VACUUM command where it is actually needed). But why do that? It + ** seems harmless enough, and provides a useful service. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED) != + (pSrcCol->colFlags & COLFLAG_GENERATED) ){ + return 0; /* Both columns have the same generated-column type */ + } + /* But the transfer is only allowed if both the source and destination + ** tables have the exact same expressions for generated columns. + ** This requirement could be relaxed for VIRTUAL columns, I suppose. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)!=0 ){ + if( sqlite3ExprCompare(0, pSrcCol->pDflt, pDestCol->pDflt, -1)!=0 ){ + testcase( pDestCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pDestCol->colFlags & COLFLAG_STORED ); + return 0; /* Different generator expressions */ + } + } +#endif + if( pDestCol->affinity!=pSrcCol->affinity ){ + return 0; /* Affinity must be the same on all columns */ + } + if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ + return 0; /* Collating sequence must be the same on all columns */ + } + if( pDestCol->notNull && !pSrcCol->notNull ){ + return 0; /* tab2 must be NOT NULL if tab1 is */ + } + /* Default values for second and subsequent columns need to match. */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)==0 && i>0 ){ + assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); + assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); + if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) + || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, + pSrcCol->pDflt->u.zToken)!=0) + ){ + return 0; /* Default values must be the same for all columns */ + } + } + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + if( IsUniqueIndex(pDestIdx) ){ + destHasUniqueIdx = 1; + } + for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + if( pSrcIdx==0 ){ + return 0; /* pDestIdx has no corresponding index in pSrc */ + } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ + } + } +#ifndef SQLITE_OMIT_CHECK + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + return 0; /* Tables have different CHECK constraints. Ticket #2252 */ + } +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + return 0; + } +#endif + if( (db->flags & SQLITE_CountRows)!=0 ){ + return 0; /* xfer opt does not play well with PRAGMA count_changes */ + } + + /* If we get this far, it means that the xfer optimization is at + ** least a possibility, though it might only work if the destination + ** table (tab1) is initially empty. + */ +#ifdef SQLITE_TEST + sqlite3_xferopt_count++; +#endif + iDbSrc = sqlite3SchemaToIndex(db, pSrc->pSchema); + v = sqlite3GetVdbe(pParse); + sqlite3CodeVerifySchema(pParse, iDbSrc); + iSrc = pParse->nTab++; + iDest = pParse->nTab++; + regAutoinc = autoIncBegin(pParse, iDbDest, pDest); + regData = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, regData); + regRowid = sqlite3GetTempReg(pParse); + sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); + assert( HasRowid(pDest) || destHasUniqueIdx ); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 && ( + (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + || destHasUniqueIdx /* (2) */ + || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ + )){ + /* In some circumstances, we are able to run the xfer optimization + ** only if the destination table is initially empty. Unless the + ** DBFLAG_Vacuum flag is set, this block generates code to make + ** that determination. If DBFLAG_Vacuum is set, then the destination + ** table is always empty. + ** + ** Conditions under which the destination must be empty: + ** + ** (1) There is no INTEGER PRIMARY KEY but there are indices. + ** (If the destination is not initially empty, the rowid fields + ** of index entries might need to change.) + ** + ** (2) The destination has a unique index. (The xfer optimization + ** is unable to test uniqueness.) + ** + ** (3) onError is something other than OE_Abort and OE_Rollback. + */ + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, addr1); + } + if( HasRowid(pSrc) ){ + u8 insFlags; + sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( pDest->iPKey>=0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeVerifyAbortable(v, onError); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, onError, pDest); + sqlite3VdbeJumpHere(v, addr2); + } + autoIncStep(pParse, regAutoinc, regRowid); + }else if( pDest->pIndex==0 && !(db->mDbFlags & DBFLAG_VacuumInto) ){ + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + }else{ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + } + + if( db->mDbFlags & DBFLAG_Vacuum ){ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + insFlags = OPFLAG_APPEND|OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + }else{ + insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND|OPFLAG_PREFORMAT; + } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + insFlags &= ~OPFLAG_PREFORMAT; + }else +#endif + { + sqlite3VdbeAddOp3(v, OP_RowCell, iDest, iSrc, regRowid); + } + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeChangeP4(v, -1, (char*)pDest, P4_TABLE); + } + sqlite3VdbeChangeP5(v, insFlags); + + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + }else{ + sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); + sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + u8 idxInsFlags = 0; + for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + assert( pSrcIdx ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc); + sqlite3VdbeSetP4KeyInfo(pParse, pSrcIdx); + VdbeComment((v, "%s", pSrcIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest); + sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); + VdbeComment((v, "%s", pDestIdx->zName)); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( db->mDbFlags & DBFLAG_Vacuum ){ + /* This INSERT command is part of a VACUUM operation, which guarantees + ** that the destination table is empty. If all indexed columns use + ** collation sequence BINARY, then it can also be assumed that the + ** index will be populated by inserting keys in strictly sorted + ** order. In this case, instead of seeking within the b-tree as part + ** of every OP_IdxInsert opcode, an OP_SeekEnd is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key + ** should be inserted. This is faster. + ** + ** If any of the indexed columns use a collation sequence other than + ** BINARY, this optimization is disabled. This is because the user + ** might change the definition of a collation sequence and then run + ** a VACUUM command. In that case keys may not be written in strictly + ** sorted order. */ + for(i=0; inColumn; i++){ + const char *zColl = pSrcIdx->azColl[i]; + if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; + } + if( i==pSrcIdx->nColumn ){ + idxInsFlags = OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + sqlite3VdbeAddOp2(v, OP_RowCell, iDest, iSrc); + } + }else if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + idxInsFlags |= OPFLAG_NCHANGE; + } + if( idxInsFlags!=(OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT) ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && !HasRowid(pDest) + && IsPrimaryKeyIndex(pDestIdx) + ){ + codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); + } + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); + sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + } + if( emptySrcTest ) sqlite3VdbeJumpHere(v, emptySrcTest); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regData); + if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); + sqlite3VdbeJumpHere(v, emptyDestTest); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + return 0; + }else{ + return 1; + } +} +#endif /* SQLITE_OMIT_XFER_OPT */ diff --git a/third_party/sqlite3/inttypes.inc b/third_party/sqlite3/inttypes.inc new file mode 100644 index 000000000..a8462e8fa --- /dev/null +++ b/third_party/sqlite3/inttypes.inc @@ -0,0 +1,62 @@ +#ifndef COSMOPOLITAN_THIRD_PARTY_SQLITE3_INTTYPES_H_ +#define COSMOPOLITAN_THIRD_PARTY_SQLITE3_INTTYPES_H_ +#include "third_party/sqlite3/sqlite3.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* +** Integers of known sizes. These typedefs might change for architectures +** where the sizes very. Preprocessor macros are available so that the +** types can be conveniently redefined at compile-type. Like this: +** +** cc '-DUINTPTR_TYPE=long long int' ... +*/ +#ifndef UINT32_TYPE +#ifdef HAVE_UINT32_T +#define UINT32_TYPE uint32_t +#else +#define UINT32_TYPE unsigned int +#endif +#endif +#ifndef UINT16_TYPE +#ifdef HAVE_UINT16_T +#define UINT16_TYPE uint16_t +#else +#define UINT16_TYPE unsigned short int +#endif +#endif +#ifndef INT16_TYPE +#ifdef HAVE_INT16_T +#define INT16_TYPE int16_t +#else +#define INT16_TYPE short int +#endif +#endif +#ifndef UINT8_TYPE +#ifdef HAVE_UINT8_T +#define UINT8_TYPE uint8_t +#else +#define UINT8_TYPE unsigned char +#endif +#endif +#ifndef INT8_TYPE +#ifdef HAVE_INT8_T +#define INT8_TYPE int8_t +#else +#define INT8_TYPE signed char +#endif +#endif +#ifndef LONGDOUBLE_TYPE +#define LONGDOUBLE_TYPE long double +#endif +typedef sqlite_int64 i64; /* 8-byte signed integer */ +typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ +typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ +typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ +typedef INT16_TYPE i16; /* 2-byte signed integer */ +typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ +typedef INT8_TYPE i8; /* 1-byte signed integer */ + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_THIRD_PARTY_SQLITE3_INTTYPES_H_ */ diff --git a/third_party/sqlite3/json1.c b/third_party/sqlite3/json1.c new file mode 100644 index 000000000..17183d782 --- /dev/null +++ b/third_party/sqlite3/json1.c @@ -0,0 +1,2632 @@ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) +#if !defined(SQLITEINT_H) +#include "third_party/sqlite3/sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/str/str.h" +#include "third_party/gdtoa/gdtoa.h" +/* clang-format off */ + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +#ifndef deliberate_fall_through +# define deliberate_fall_through +#endif + +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else +/* Use the standard library for separate compilation */ +#include "libc/str/str.h" /* amalgamator: keep */ +#define safe_isdigit(x) isdigit((unsigned char)(x)) +#define safe_isalnum(x) isalnum((unsigned char)(x)) +#define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) + +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned short int u16; + typedef unsigned char u8; +#endif + +/* Objects */ +typedef struct JsonString JsonString; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. +*/ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 bErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* JSON type values +*/ +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ + + +/* A single node of parsed JSON +*/ +struct JsonNode { + u8 eType; /* One of the JSON_ type values */ + u8 jnFlags; /* JNODE flags */ + u32 n; /* Bytes of content, or number of sub-nodes */ + union { + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ + } u; +}; + +/* A completely parsed JSON string +*/ +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ + u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ + u16 iDepth; /* Nesting depth */ + int nJson; /* Length of the zJson string in bytes */ + u32 iHold; /* Replace cache line with the lowest iHold value */ +}; + +/* +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 2000 is far deeper than any sane JSON +** should go. +*/ +#define JSON_MAX_DEPTH 2000 + +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ + +/* Set the JsonString object to an empty string +*/ +static void jsonZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the JsonString object +*/ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->bErr = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the JsonString object back to its +** initial state. +*/ +static void jsonReset(JsonString *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(JsonString *p){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(JsonString *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->bErr ) return 1; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the JsonString string. +*/ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + +/* Append a single character +*/ +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; + } + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); +} + +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 2; + jsonReset(p); + } + break; + } + } +} + + +/* Make the JSON in p the result of the SQL function. +*/ +static void jsonResult(JsonString *p){ + if( p->bErr==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + +/* +** Free a JsonParse object that was obtained from sqlite3_malloc(). +*/ +static void jsonParseFree(JsonParse *pParse){ + jsonParseReset(pParse); + sqlite3_free(pParse); +} + +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. +*/ +static void jsonRenderNode( + JsonNode *pNode, /* The node to render */ + JsonString *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + break; + } + /* no break */ deliberate_fall_through + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, '}'); + break; + } + } +} + +/* +** Return a JsonNode and all its descendents as a JSON string. +*/ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); +} + +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +static u8 jsonHexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#else + h += 9*(1&(h>>6)); +#endif + return (u8)(h & 0xf); +} + +/* +** Convert a 4-byte hex string into an integer +*/ +static u32 jsonHexToInt4(const char *z){ + u32 v; + assert( safe_isxdigit(z[0]) ); + assert( safe_isxdigit(z[1]) ); + assert( safe_isxdigit(z[2]) ); + assert( safe_isxdigit(z[3]) ); + v = (jsonHexToInt(z[0])<<12) + + (jsonHexToInt(z[1])<<8) + + (jsonHexToInt(z[2])<<4) + + jsonHexToInt(z[3]); + return v; +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; + } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); + int_done: + break; + int_as_real: i=0; /* no break */ deliberate_fall_through + } + case JSON_REAL: { + double r; +#ifdef SQLITE_AMALGAMATION + const char *z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); +#else + r = strtod(pNode->u.zJContent, 0); +#endif + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { +#if 0 /* Never happens because JNODE_RAW is only set by json_set(), + ** json_insert() and json_replace() and those routines do not + ** call jsonReturn() */ + if( pNode->jnFlags & JNODE_RAW ){ + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else +#endif + assert( (pNode->jnFlags & JNODE_RAW)==0 ); + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->u.zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pNode, pCtx, aReplace); + break; + } + } +} + +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +#define JSON_NOINLINE __attribute__((__noinline__)) +#elif defined(_MSC_VER) && _MSC_VER >= 1310 +#define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif + + +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNodenAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); +} + +/* +** Create a new JsonNode instance based on the arguments and append that +** instance to the JsonParse. Return the index in pParse->aNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + return jsonParseAddNodeExpand(pParse, eType, n, zContent); + } + p = &pParse->aNode[pParse->nNode]; + p->eType = (u8)eType; + p->jnFlags = 0; + p->n = n; + p->u.zJContent = zContent; + return pParse->nNode++; +} + +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + JsonNode *pNode; + const char *z = pParse->zJson; + while( safe_isspace(z[i]) ){ i++; } + if( (c = z[i])=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + if( x<0 ){ + pParse->iDepth--; + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + if( pParse->oom ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; + j = x; + while( safe_isspace(z[j]) ){ j++; } + if( z[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ) return -1; + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ){ + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + u8 jnFlags = 0; + j = i+1; + for(;;){ + c = z[j]; + if( (c & ~0x1f)==0 ){ + /* Control characters are not allowed in strings */ + return -1; + } + if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(z+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + return j+1; + }else if( c=='n' + && strncmp(z+i,"null",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(z+i,"true",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(z+i,"false",5)==0 + && !safe_isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + assert( '-' < '0' ); + if( c<='0' ){ + j = c=='-' ? i+1 : i; + if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + } + j = i+1; + for(;; j++){ + c = z[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return -1; + continue; + } + break; + } + if( z[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &z[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ +){ + int i; + memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( safe_isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* First cache entry */ +#define JSON_CACHE_SZ 4 /* Max number of cache entries */ + +/* +** Obtain a complete parse of the JSON found in the first argument +** of the argv array. Use the sqlite3_get_auxdata() cache for this +** parse if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse, +** and also register the new parse so that it will be available for +** future sqlite3_get_auxdata() calls. +*/ +static JsonParse *jsonParseCached( + sqlite3_context *pCtx, + sqlite3_value **argv, + sqlite3_context *pErrCtx +){ + const char *zJson = (const char*)sqlite3_value_text(argv[0]); + int nJson = sqlite3_value_bytes(argv[0]); + JsonParse *p; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + if( zJson==0 ) return 0; + for(iKey=0; iKeynJson==nJson + && memcmp(p->zJson,zJson,nJson)==0 + ){ + p->nErr = 0; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + if( pMatch ){ + pMatch->nErr = 0; + pMatch->iHold = iMaxHold+1; + return pMatch; + } + p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, sizeof(*p)); + p->zJson = (char*)&p[1]; + memcpy((char*)p->zJson, zJson, nJson+1); + if( jsonParse(p, pErrCtx, p->zJson) ){ + sqlite3_free(p); + return 0; + } + p->nJson = nJson; + p->iHold = iMaxHold+1; + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, + (void(*)(void*))jsonParseFree); + return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); +} + +/* +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. +*/ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; + }else{ + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + } +} + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. +*/ +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot = &pParse->aNode[iRoot]; + if( zPath[0]==0 ) return pRoot; + if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' ){ + i = 0; + j = 1; + while( safe_isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( j<2 || zPath[j]!=']' ){ + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + j += jsonNodeSize(&pBase[j]); + } + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + iBase += pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && safe_isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( safe_isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + }else{ + *pzErr = zPath; + return 0; + } + } + if( pRoot->eType!=JSON_ARRAY ) return 0; + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} + +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} + +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; + +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#ifdef SQLITE_DEBUG +/* +** The json_parse(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + + assert( argc==1 ); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); + jsonInit(&s, ctx); + for(i=0; inNode ); + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode==0 ){ + return; + } + if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + } + sqlite3_result_int64(ctx, n); +} + +/* +** json_extract(JSON, PATH, ...) +** +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + JsonNode *pNode; + const char *zPath; + JsonString jx; + int i; + + if( argc<2 ) return; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; inErr ) break; + if( argc>2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); + } + } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); +} + +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + u32 nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ + + UNUSED_PARAM(argc); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } + jsonParseReset(&x); + jsonParseReset(&y); +} + + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(ctx, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '{'); + for(i=0; ijnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(x.aNode, ctx, 0); + } +remove_done: + jsonParseReset(&x); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto replace_err; + if( pNode ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +replace_err: + jsonParseReset(&x); +} + +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(ctx); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + bApnd = 0; + pNode = jsonLookup(&x, zPath, &bApnd, ctx); + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +jsonSetDone: + jsonParseReset(&x); +} + +/* +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. +*/ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + const char *zPath; + JsonNode *pNode; + + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + } +} + +/* +** json_valid(JSON) +** +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAM(argc); + p = jsonParseCached(ctx, argv, 0); + sqlite3_result_int(ctx, p!=0); +} + + +/**************************************************************************** +** Aggregate SQL function implementations +****************************************************************************/ +/* +** json_group_array(VALUE) +** +** Return a JSON array composed of all values in the aggregate. +*/ +static void jsonArrayStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + jsonAppendValue(pStr, argv[0]); + } +} +static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonArrayValue(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 0); +} +static void jsonArrayFinal(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 1); +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** This method works for both json_group_array() and json_group_object(). +** It works by removing the first element of the group by searching forward +** to the first comma (",") that is not within a string and deleting all +** text through that comma. +*/ +static void jsonGroupInverse( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + unsigned int i; + int inStr = 0; + int nNest = 0; + char *z; + char c; + JsonString *pStr; + UNUSED_PARAM(argc); + UNUSED_PARAM(argv); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); +#ifdef NEVER + /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will + ** always have been called to initalize it */ + if( NEVER(!pStr) ) return; +#endif + z = pStr->zBuf; + for(i=1; (c = z[i])!=',' || inStr || nNest; i++){ + if( i>=pStr->nUsed ){ + pStr->nUsed = 1; + return; + } + if( c=='"' ){ + inStr = !inStr; + }else if( c=='\\' ){ + i++; + }else if( !inStr ){ + if( c=='{' || c=='[' ) nNest++; + if( c=='}' || c==']' ) nNest--; + } + } + pStr->nUsed -= i; + memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1); +} +#else +# define jsonGroupInverse 0 +#endif + + +/* +** json_group_obj(NAME,VALUE) +** +** Return a JSON object composed of all names and values in the aggregate. +*/ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + const char *z; + u32 n; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + z = (const char*)sqlite3_value_text(argv[0]); + n = (u32)sqlite3_value_bytes(argv[0]); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendValue(pStr, argv[1]); + } +} +static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + jsonAppendChar(pStr, '}'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonObjectValue(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 0); +} +static void jsonObjectFinal(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 1); +} + + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zRoot; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +/* The xBestIndex method assumes that the JSON and ROOT columns are +** the last two columns in the table. Should this ever changes, be +** sure to update the xBestIndex method. */ +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAM(pzErr); + UNUSED_PARAM(argv); + UNUSED_PARAM(argc); + UNUSED_PARAM(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + + UNUSED_PARAM(p); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zRoot); + jsonParseReset(&p->sParse); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->zJson = 0; + p->zRoot = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->iiEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } + } + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + } + return SQLITE_OK; +} + +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); + } +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( i ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + /* no break */ deliberate_fall_through + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + assert( i==JEACH_JSON ); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop counter or computed array index */ + int aIdx[2]; /* Index of constraints for JSON and ROOT */ + int unusableMask = 0; /* Mask of unusable JSON and ROOT constraints */ + int idxMask = 0; /* Mask of usable == constraints JSON and ROOT */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that JSON and ROOT are the last two + ** columns in the table */ + assert( JEACH_ROOT == JEACH_JSON+1 ); + UNUSED_PARAM(tab); + aIdx[0] = aIdx[1] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; + int iMask; + if( pConstraint->iColumn < JEACH_JSON ) continue; + iCol = pConstraint->iColumn - JEACH_JSON; + assert( iCol==0 || iCol==1 ); + iMask = 1 << iCol; + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + aIdx[iCol] = i; + idxMask |= iMask; + } + } + if( (unusableMask & ~idxMask)!=0 ){ + /* If there are any unusable constraints on JSON or ROOT, then reject + ** this entire plan */ + return SQLITE_CONSTRAINT; + } + if( aIdx[0]<0 ){ + /* No JSON input. Leave estimatedCost at the huge value that it was + ** initialized to to discourage the query planner from selecting this + ** plan. */ + pIdxInfo->idxNum = 0; + }else{ + pIdxInfo->estimatedCost = 1.0; + i = aIdx[0]; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + if( aIdx[1]<0 ){ + pIdxInfo->idxNum = 1; /* Only JSON supplied. Plan 1 */ + }else{ + i = aIdx[1]; + pIdxInfo->aConstraintUsage[i].argvIndex = 2; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 3; /* Both JSON and ROOT are supplied. Plan 3 */ + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zRoot = 0; + sqlite3_int64 n; + + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc64( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, (size_t)n+1); + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } + jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; + }else{ + JsonNode *pNode = 0; + if( idxNum==3 ){ + const char *zErr = 0; + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + zErr = zRoot; + }else{ + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + } + if( zErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; + if( p->bRecursive ){ + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/**************************************************************************** +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ + +int sqlite3Json1Init(sqlite3 *db){ + int rc = SQLITE_OK; + unsigned int i; + static const struct { + const char *zName; + int nArg; + int flag; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, + +#if SQLITE_DEBUG + /* DEBUG and TESTING functions */ + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, +#endif + }; + static const struct { + const char *zName; + int nArg; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + void (*xValue)(sqlite3_context*); + } aAgg[] = { + { "json_group_array", 1, + jsonArrayStep, jsonArrayFinal, jsonArrayValue }, + { "json_group_object", 2, + jsonObjectStep, jsonObjectFinal, jsonObjectValue }, + }; +#ifndef SQLITE_OMIT_VIRTUALTABLE + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; +#endif + static const int enc = + SQLITE_UTF8 | + SQLITE_DETERMINISTIC | + SQLITE_INNOCUOUS; + for(i=0; i=2 ){ + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; + for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; +#ifdef SQLITE_ASCII + if( (z[0]&~0x20)!=zKW[0] ) continue; + if( (z[1]&~0x20)!=zKW[1] ) continue; + j = 2; + while( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + *pzName = zKWText + aKWOffset[i]; + *pnName = aKWLen[i]; + return SQLITE_OK; +} +int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; } +int sqlite3_keyword_check(const char *zName, int nName){ + return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName); +} diff --git a/third_party/sqlite3/legacy.c b/third_party/sqlite3/legacy.c new file mode 100644 index 000000000..70d9dc062 --- /dev/null +++ b/third_party/sqlite3/legacy.c @@ -0,0 +1,142 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +/* clang-format off */ + +#include "third_party/sqlite3/sqliteInt.inc" + +/* +** Execute SQL code. Return one of the SQLITE_ success/failure +** codes. Also write an error message into memory obtained from +** malloc() and make *pzErrMsg point to that message. +** +** If the SQL is a query, then for each row in the query result +** the xCallback() function is called. pArg becomes the first +** argument to xCallback(). If xCallback=NULL then no callback +** is invoked, even for queries. +*/ +int sqlite3_exec( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + sqlite3_callback xCallback, /* Invoke this callback routine */ + void *pArg, /* First argument to xCallback() */ + char **pzErrMsg /* Write error messages here */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3_stmt *pStmt = 0; /* The current SQL statement */ + char **azCols = 0; /* Names of result columns */ + int callbackIsInit; /* True if callback data is initialized */ + + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; + if( zSql==0 ) zSql = ""; + + sqlite3_mutex_enter(db->mutex); + sqlite3Error(db, SQLITE_OK); + while( rc==SQLITE_OK && zSql[0] ){ + int nCol = 0; + char **azVals = 0; + + pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc!=SQLITE_OK ){ + continue; + } + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + continue; + } + callbackIsInit = 0; + + while( 1 ){ + int i; + rc = sqlite3_step(pStmt); + + /* Invoke the callback function if required */ + if( xCallback && (SQLITE_ROW==rc || + (SQLITE_DONE==rc && !callbackIsInit + && db->flags&SQLITE_NullCallback)) ){ + if( !callbackIsInit ){ + nCol = sqlite3_column_count(pStmt); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); + if( azCols==0 ){ + goto exec_out; + } + for(i=0; ierrMask)==rc ); + sqlite3_mutex_leave(db->mutex); + return rc; +} diff --git a/third_party/sqlite3/loadext.c b/third_party/sqlite3/loadext.c new file mode 100644 index 000000000..794217714 --- /dev/null +++ b/third_party/sqlite3/loadext.c @@ -0,0 +1,855 @@ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to dynamically load extensions into +** the SQLite library. +*/ +/* clang-format off */ + +#ifndef SQLITE_CORE + #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ +#endif +#include "third_party/sqlite3/sqlite3ext.h" +#include "third_party/sqlite3/sqliteInt.inc" + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Some API routines are omitted when various features are +** excluded from a build of SQLite. Substitute a NULL pointer +** for any missing APIs. +*/ +#ifndef SQLITE_ENABLE_COLUMN_METADATA +# define sqlite3_column_database_name 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name 0 +# define sqlite3_column_origin_name16 0 +#endif + +#ifdef SQLITE_OMIT_AUTHORIZATION +# define sqlite3_set_authorizer 0 +#endif + +#ifdef SQLITE_OMIT_UTF16 +# define sqlite3_bind_text16 0 +# define sqlite3_collation_needed16 0 +# define sqlite3_column_decltype16 0 +# define sqlite3_column_name16 0 +# define sqlite3_column_text16 0 +# define sqlite3_complete16 0 +# define sqlite3_create_collation16 0 +# define sqlite3_create_function16 0 +# define sqlite3_errmsg16 0 +# define sqlite3_open16 0 +# define sqlite3_prepare16 0 +# define sqlite3_prepare16_v2 0 +# define sqlite3_prepare16_v3 0 +# define sqlite3_result_error16 0 +# define sqlite3_result_text16 0 +# define sqlite3_result_text16be 0 +# define sqlite3_result_text16le 0 +# define sqlite3_value_text16 0 +# define sqlite3_value_text16be 0 +# define sqlite3_value_text16le 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name16 0 +#endif + +#ifdef SQLITE_OMIT_COMPLETE +# define sqlite3_complete 0 +# define sqlite3_complete16 0 +#endif + +#ifdef SQLITE_OMIT_DECLTYPE +# define sqlite3_column_decltype16 0 +# define sqlite3_column_decltype 0 +#endif + +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +# define sqlite3_progress_handler 0 +#endif + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define sqlite3_create_module 0 +# define sqlite3_create_module_v2 0 +# define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 +# define sqlite3_vtab_collation 0 +#endif + +#ifdef SQLITE_OMIT_SHARED_CACHE +# define sqlite3_enable_shared_cache 0 +#endif + +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) +# define sqlite3_profile 0 +# define sqlite3_trace 0 +#endif + +#ifdef SQLITE_OMIT_GET_TABLE +# define sqlite3_free_table 0 +# define sqlite3_get_table 0 +#endif + +#ifdef SQLITE_OMIT_INCRBLOB +#define sqlite3_bind_zeroblob 0 +#define sqlite3_blob_bytes 0 +#define sqlite3_blob_close 0 +#define sqlite3_blob_open 0 +#define sqlite3_blob_read 0 +#define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 +#endif + +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + +/* +** The following structure contains pointers to all SQLite API routines. +** A pointer to this structure is passed into extensions when they are +** loaded so that the extension can make calls back into the SQLite +** library. +** +** When adding new APIs, add them to the bottom of this structure +** in order to preserve backwards compatibility. +** +** Extensions that use newer APIs should first call the +** sqlite3_libversion_number() to make sure that the API they +** intend to use is supported by the library. Extensions should +** also check to make sure that the pointer to the function is +** not NULL before calling it. +*/ +static const sqlite3_api_routines sqlite3Apis = { + sqlite3_aggregate_context, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_aggregate_count, +#else + 0, +#endif + sqlite3_bind_blob, + sqlite3_bind_double, + sqlite3_bind_int, + sqlite3_bind_int64, + sqlite3_bind_null, + sqlite3_bind_parameter_count, + sqlite3_bind_parameter_index, + sqlite3_bind_parameter_name, + sqlite3_bind_text, + sqlite3_bind_text16, + sqlite3_bind_value, + sqlite3_busy_handler, + sqlite3_busy_timeout, + sqlite3_changes, + sqlite3_close, + sqlite3_collation_needed, + sqlite3_collation_needed16, + sqlite3_column_blob, + sqlite3_column_bytes, + sqlite3_column_bytes16, + sqlite3_column_count, + sqlite3_column_database_name, + sqlite3_column_database_name16, + sqlite3_column_decltype, + sqlite3_column_decltype16, + sqlite3_column_double, + sqlite3_column_int, + sqlite3_column_int64, + sqlite3_column_name, + sqlite3_column_name16, + sqlite3_column_origin_name, + sqlite3_column_origin_name16, + sqlite3_column_table_name, + sqlite3_column_table_name16, + sqlite3_column_text, + sqlite3_column_text16, + sqlite3_column_type, + sqlite3_column_value, + sqlite3_commit_hook, + sqlite3_complete, + sqlite3_complete16, + sqlite3_create_collation, + sqlite3_create_collation16, + sqlite3_create_function, + sqlite3_create_function16, + sqlite3_create_module, + sqlite3_data_count, + sqlite3_db_handle, + sqlite3_declare_vtab, + sqlite3_enable_shared_cache, + sqlite3_errcode, + sqlite3_errmsg, + sqlite3_errmsg16, + sqlite3_exec, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_expired, +#else + 0, +#endif + sqlite3_finalize, + sqlite3_free, + sqlite3_free_table, + sqlite3_get_autocommit, + sqlite3_get_auxdata, + sqlite3_get_table, + 0, /* Was sqlite3_global_recover(), but that function is deprecated */ + sqlite3_interrupt, + sqlite3_last_insert_rowid, + sqlite3_libversion, + sqlite3_libversion_number, + sqlite3_malloc, + sqlite3_mprintf, + sqlite3_open, + sqlite3_open16, + sqlite3_prepare, + sqlite3_prepare16, + sqlite3_profile, + sqlite3_progress_handler, + sqlite3_realloc, + sqlite3_reset, + sqlite3_result_blob, + sqlite3_result_double, + sqlite3_result_error, + sqlite3_result_error16, + sqlite3_result_int, + sqlite3_result_int64, + sqlite3_result_null, + sqlite3_result_text, + sqlite3_result_text16, + sqlite3_result_text16be, + sqlite3_result_text16le, + sqlite3_result_value, + sqlite3_rollback_hook, + sqlite3_set_authorizer, + sqlite3_set_auxdata, + sqlite3_snprintf, + sqlite3_step, + sqlite3_table_column_metadata, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_thread_cleanup, +#else + 0, +#endif + sqlite3_total_changes, + sqlite3_trace, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_transfer_bindings, +#else + 0, +#endif + sqlite3_update_hook, + sqlite3_user_data, + sqlite3_value_blob, + sqlite3_value_bytes, + sqlite3_value_bytes16, + sqlite3_value_double, + sqlite3_value_int, + sqlite3_value_int64, + sqlite3_value_numeric_type, + sqlite3_value_text, + sqlite3_value_text16, + sqlite3_value_text16be, + sqlite3_value_text16le, + sqlite3_value_type, + sqlite3_vmprintf, + /* + ** The original API set ends here. All extensions can call any + ** of the APIs above provided that the pointer is not NULL. But + ** before calling APIs that follow, extension should check the + ** sqlite3_libversion_number() to make sure they are dealing with + ** a library that is new enough to support that API. + ************************************************************************* + */ + sqlite3_overload_function, + + /* + ** Added after 3.3.13 + */ + sqlite3_prepare_v2, + sqlite3_prepare16_v2, + sqlite3_clear_bindings, + + /* + ** Added for 3.4.1 + */ + sqlite3_create_module_v2, + + /* + ** Added for 3.5.0 + */ + sqlite3_bind_zeroblob, + sqlite3_blob_bytes, + sqlite3_blob_close, + sqlite3_blob_open, + sqlite3_blob_read, + sqlite3_blob_write, + sqlite3_create_collation_v2, + sqlite3_file_control, + sqlite3_memory_highwater, + sqlite3_memory_used, +#ifdef SQLITE_MUTEX_OMIT + 0, + 0, + 0, + 0, + 0, +#else + sqlite3_mutex_alloc, + sqlite3_mutex_enter, + sqlite3_mutex_free, + sqlite3_mutex_leave, + sqlite3_mutex_try, +#endif + sqlite3_open_v2, + sqlite3_release_memory, + sqlite3_result_error_nomem, + sqlite3_result_error_toobig, + sqlite3_sleep, + sqlite3_soft_heap_limit, + sqlite3_vfs_find, + sqlite3_vfs_register, + sqlite3_vfs_unregister, + + /* + ** Added for 3.5.8 + */ + sqlite3_threadsafe, + sqlite3_result_zeroblob, + sqlite3_result_error_code, + sqlite3_test_control, + sqlite3_randomness, + sqlite3_context_db_handle, + + /* + ** Added for 3.6.0 + */ + sqlite3_extended_result_codes, + sqlite3_limit, + sqlite3_next_stmt, + sqlite3_sql, + sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_text64, + sqlite3_strglob, + /* Version 3.8.11 and later */ + (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, + sqlite3_value_free, + sqlite3_result_zeroblob64, + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype, + /* Version 3.10.0 and later */ + sqlite3_status64, + sqlite3_strlike, + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql, + /* Version 3.18.0 and later */ + sqlite3_set_last_insert_rowid, + /* Version 3.20.0 and later */ + sqlite3_prepare_v3, + sqlite3_prepare16_v3, + sqlite3_bind_pointer, + sqlite3_result_pointer, + sqlite3_value_pointer, + /* Version 3.22.0 and later */ + sqlite3_vtab_nochange, + sqlite3_value_nochange, + sqlite3_vtab_collation, + /* Version 3.24.0 and later */ + sqlite3_keyword_count, + sqlite3_keyword_name, + sqlite3_keyword_check, + sqlite3_str_new, + sqlite3_str_finish, + sqlite3_str_appendf, + sqlite3_str_vappendf, + sqlite3_str_append, + sqlite3_str_appendall, + sqlite3_str_appendchar, + sqlite3_str_reset, + sqlite3_str_errcode, + sqlite3_str_length, + sqlite3_str_value, + /* Version 3.25.0 and later */ + sqlite3_create_window_function, + /* Version 3.26.0 and later */ +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3_normalized_sql, +#else + 0, +#endif + /* Version 3.28.0 and later */ + sqlite3_stmt_isexplain, + sqlite3_value_frombind, + /* Version 3.30.0 and later */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_drop_modules, +#else + 0, +#endif + /* Version 3.31.0 and later */ + sqlite3_hard_heap_limit64, + sqlite3_uri_key, + sqlite3_filename_database, + sqlite3_filename_journal, + sqlite3_filename_wal, + /* Version 3.32.0 and later */ + sqlite3_create_filename, + sqlite3_free_filename, + sqlite3_database_file_object, + /* Version 3.34.0 and later */ + sqlite3_txn_state, +}; + +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + +/* +** Attempt to load an SQLite extension library contained in the file +** zFile. The entry point is zProc. zProc may be 0 in which case a +** default entry point name (sqlite3_extension_init) is used. Use +** of the default name is recommended. +** +** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. +** +** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with +** error message text. The calling function should free this memory +** by calling sqlite3DbFree(db, ). +*/ +static int sqlite3LoadExtension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + sqlite3_vfs *pVfs = db->pVfs; + void *handle; + sqlite3_loadext_entry xInit; + char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; + void **aHandle; + u64 nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + int rc; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + + + if( pzErrMsg ) *pzErrMsg = 0; + + /* Ticket #1863. To avoid a creating security problems for older + ** applications that relink against newer versions of SQLite, the + ** ability to run load_extension is turned off by default. One + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("not authorized"); + } + return SQLITE_ERROR; + } + + zEntry = zProc ? zProc : "sqlite3_extension_init"; + + handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc64(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM_BKPT; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + } + if( xInit==0 ){ + if( pzErrMsg ){ + nMsg += sqlite3Strlen30(zEntry); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); + if( zErrmsg ){ + sqlite3_snprintf(nMsg, zErrmsg, + "no entry point [%s] in shared library [%s]", zEntry, zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); + return SQLITE_ERROR; + } + sqlite3_free(zAltEntry); + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); + } + sqlite3_free(zErrmsg); + sqlite3OsDlClose(pVfs, handle); + return SQLITE_ERROR; + } + + /* Append the new shared library handle to the db->aExtension array. */ + aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); + if( aHandle==0 ){ + return SQLITE_NOMEM_BKPT; + } + if( db->nExtension>0 ){ + memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); + } + sqlite3DbFree(db, db->aExtension); + db->aExtension = aHandle; + + db->aExtension[db->nExtension++] = handle; + return SQLITE_OK; +} +int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + int rc; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Call this routine when the database connection is closing in order +** to clean up loaded extensions +*/ +void sqlite3CloseExtensions(sqlite3 *db){ + int i; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inExtension; i++){ + sqlite3OsDlClose(db->pVfs, db->aExtension[i]); + } + sqlite3DbFree(db, db->aExtension); +} + +/* +** Enable or disable extension loading. Extension loading is disabled by +** default so as not to open security holes in older applications. +*/ +int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); + if( onoff ){ + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; + }else{ + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + } + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +/* +** The following object holds the list of automatically loaded +** extensions. +** +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MAIN +** mutex must be held while accessing this list. +*/ +typedef struct sqlite3AutoExtList sqlite3AutoExtList; +static SQLITE_WSD struct sqlite3AutoExtList { + u32 nExt; /* Number of entries in aExt[] */ + void (**aExt)(void); /* Pointers to the extension init functions */ +} sqlite3Autoext = { 0, 0 }; + +/* The "wsdAutoext" macro will resolve to the autoextension +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Autoext" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdAutoextInit \ + sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) +# define wsdAutoext x[0] +#else +# define wsdAutoextInit +# define wsdAutoext sqlite3Autoext +#endif + + +/* +** Register a statically linked extension that is automatically +** loaded by every new database connection. +*/ +int sqlite3_auto_extension( + void (*xInit)(void) +){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ){ + return rc; + }else +#endif + { + u32 i; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + for(i=0; i=0; i--){ + if( wsdAutoext.aExt[i]==xInit ){ + wsdAutoext.nExt--; + wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; + n++; + break; + } + } + sqlite3_mutex_leave(mutex); + return n; +} + +/* +** Reset the automatic extension loading mechanism. +*/ +void sqlite3_reset_auto_extension(void){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize()==SQLITE_OK ) +#endif + { +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + sqlite3_free(wsdAutoext.aExt); + wsdAutoext.aExt = 0; + wsdAutoext.nExt = 0; + sqlite3_mutex_leave(mutex); + } +} + +/* +** Load all automatic extensions. +** +** If anything goes wrong, set an error in the database connection. +*/ +void sqlite3AutoLoadExtensions(sqlite3 *db){ + u32 i; + int go = 1; + int rc; + sqlite3_loadext_entry xInit; + + wsdAutoextInit; + if( wsdAutoext.nExt==0 ){ + /* Common case: early out without every having to acquire a mutex */ + return; + } + for(i=0; go; i++){ + char *zErrmsg; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; +#endif + sqlite3_mutex_enter(mutex); + if( i>=wsdAutoext.nExt ){ + xInit = 0; + go = 0; + }else{ + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; + } + sqlite3_mutex_leave(mutex); + zErrmsg = 0; + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ + sqlite3ErrorWithMsg(db, rc, + "automatic extension loading failed: %s", zErrmsg); + go = 0; + } + sqlite3_free(zErrmsg); + } +} diff --git a/third_party/sqlite3/main.c b/third_party/sqlite3/main.c new file mode 100644 index 000000000..31673e193 --- /dev/null +++ b/third_party/sqlite3/main.c @@ -0,0 +1,4677 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#ifdef SQLITE_ENABLE_FTS3 +#include "third_party/sqlite3/fts3.inc" +#endif +#ifdef SQLITE_ENABLE_RTREE +#include "third_party/sqlite3/rtree.inc" +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +#include "third_party/sqlite3/sqliteicu.inc" +#endif + +/* +** This is an extension initializer that is a no-op and always +** succeeds, except that it fails if the fault-simulation is set +** to 500. +*/ +static int sqlite3TestExtInit(sqlite3 *db){ + (void)db; + return sqlite3FaultSim(500); +} + + +/* +** Forward declarations of external module initializer functions +** for modules that need them. +*/ +#ifdef SQLITE_ENABLE_FTS1 +int sqlite3Fts1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS2 +int sqlite3Fts2Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +int sqlite3Fts5Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_JSON1 +int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_STMTVTAB +int sqlite3StmtVtabInit(sqlite3*); +#endif + +/* +** An array of pointers to extension initializer functions for +** built-in extensions. +*/ +static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { +#ifdef SQLITE_ENABLE_FTS1 + sqlite3Fts1Init, +#endif +#ifdef SQLITE_ENABLE_FTS2 + sqlite3Fts2Init, +#endif +#ifdef SQLITE_ENABLE_FTS3 + sqlite3Fts3Init, +#endif +#ifdef SQLITE_ENABLE_FTS5 + sqlite3Fts5Init, +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + sqlite3IcuInit, +#endif +#ifdef SQLITE_ENABLE_RTREE + sqlite3RtreeInit, +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + sqlite3DbpageRegister, +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + sqlite3DbstatRegister, +#endif + sqlite3TestExtInit, +#ifdef SQLITE_ENABLE_JSON1 + sqlite3Json1Init, +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + sqlite3StmtVtabInit, +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + sqlite3VdbeBytecodeVtabInit, +#endif +}; + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using +** an edited copy of the amalgamation, then the last four characters of +** the hash might be different from SQLITE_SOURCE_ID. +*/ +const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +int sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ){ + sqlite3MemoryBarrier(); + return SQLITE_OK; + } + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MAIN mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMainMtx); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + sqlite3GlobalConfig.inProgress = 1; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); + } +#ifdef SQLITE_ENABLE_DESERIALIZE + if( rc==SQLITE_OK ){ + rc = sqlite3MemdbInit(); + } +#endif + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3MemoryBarrier(); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMainMtx); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMainMtx); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SMALL_MALLOC: { + sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-18761-36601 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), + ** the size of each page cache line (sz), and the number of cache lines + ** (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; + } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; + break; + } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + +#ifdef SQLITE_ENABLE_DESERIALIZE + case SQLITE_CONFIG_MEMDB_MAXSIZE: { + sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64); + break; + } +#endif /* SQLITE_ENABLE_DESERIALIZE */ + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE + void *pStart; + sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + + if( sqlite3LookasideUsed(db,0)>0 ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) szAlloc = sqlite3MallocSize(pStart); + }else{ + pStart = pBuf; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( sz>=LOOKASIDE_SMALL*3 ){ + nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else if( sz>=LOOKASIDE_SMALL*2 ){ + nBig = szAlloc/(LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( sz>0 ){ + nBig = szAlloc/sz; + nSm = 0; + }else{ + nBig = nSm = 0; + } + db->lookaside.pStart = pStart; + db->lookaside.pInit = 0; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + db->lookaside.szTrue = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=0; ipNext = db->lookaside.pInit; + db->lookaside.pInit = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = p; + for(i=0; ipNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = p; + p = (LookasideSlot*)&((u8*)p)[LOOKASIDE_SMALL]; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + assert( ((uptr)p)<=szAlloc + (uptr)pStart ); + db->lookaside.pEnd = p; + db->lookaside.bDisable = 0; + db->lookaside.bMalloced = pBuf==0 ?1:0; + db->lookaside.nSlot = nBig+nSm; + }else{ + db->lookaside.pStart = db; +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = db; +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + db->lookaside.pEnd = db; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; + db->lookaside.bMalloced = 0; + db->lookaside.nSlot = 0; + } + assert( sqlite3LookasideUsed(db,0)==0 ); +#endif /* SQLITE_OMIT_LOOKASIDE */ + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +int sqlite3_db_release_memory(sqlite3 *db){ + int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + +/* +** Configuration settings for an individual database connection +*/ +int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + /* IMP: R-06824-28531 */ + /* IMP: R-36257-52125 */ + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_VIEW, SQLITE_EnableView }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, + { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, + { SQLITE_DBCONFIG_ENABLE_QPSG, SQLITE_EnableQPSG }, + { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, + { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, + { SQLITE_DBCONFIG_DEFENSIVE, SQLITE_Defensive }, + { SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_WriteSchema| + SQLITE_NoSchemaError }, + { SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_LegacyAlter }, + { SQLITE_DBCONFIG_DQS_DDL, SQLITE_DqsDDL }, + { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, + { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, + { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~(u64)aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db, 0); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +*/ +static int binCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + UNUSED_PARAMETER(NotUsed); + n = nKey1xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==binCollFunc; +} + +/* +** Another built-in collating sequence: NOCASE. +** +** This collating sequence is intended to be used for "case independent +** comparison". SQLite's knowledge of upper and lower case equivalents +** extends only to the 26 characters used in the English language. +** +** At the moment there is only a UTF-8 implementation. +*/ +static int nocaseCollatingFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int r = sqlite3StrNICmp( + (const char *)pKey1, (const char *)pKey2, (nKey1lastRowid; +} + +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->u.pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + HashElem *p; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( pSchema ){ + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->trace.xV2(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Return the transaction state for a single databse, or the maximum +** transaction state over all attached databases if zSchema is null. +*/ +int sqlite3_txn_state(sqlite3 *db, const char *zSchema){ + int iDb, nDb; + int iTxn = -1; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( zSchema ){ + nDb = iDb = sqlite3FindDbName(db, zSchema); + if( iDb<0 ) nDb--; + }else{ + iDb = 0; + nDb = db->nDb-1; + } + for(; iDb<=nDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + int x = pBt!=0 ? sqlite3BtreeTxnState(pBt) : SQLITE_TXN_NONE; + if( x>iTxn ) iTxn = x; + } + sqlite3_mutex_leave(db->mutex); + return iTxn; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); + } + sqlite3HashClear(&db->aFunc); + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); + } + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + sqlite3VtabEponymousTableClear(db, pMod); + sqlite3VtabModuleUnref(db, pMod); + } + sqlite3HashClear(&db->aModule); +#endif + + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif + + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( sqlite3LookasideUsed(db,0)==0 ); + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. +*/ +void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeTxnState(p)==SQLITE_TXN_WRITE ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( schemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~(u64)SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_NEED_ERR_NAME) +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "query aborted", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ 0, + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "bad parameter or other API misuse", +#ifdef SQLITE_DISABLE_LFS + /* SQLITE_NOLFS */ "large file support is disabled", +#else + /* SQLITE_NOLFS */ 0, +#endif + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ 0, + /* SQLITE_RANGE */ "column index out of range", + /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > tmout ){ + delay = tmout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + /* This case for unix systems that lack usleep() support. Sleeping + ** must be done in increments of whole seconds */ + sqlite3 *db = (sqlite3 *)ptr; + int tmout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > tmout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif +} + +/* +** Invoke the given busy handler. +** +** This routine is called when an operation failed to acquire a +** lock on VFS file pFile. +** +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. +*/ +int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( p->xBusyHandler==0 || p->nBusy<0 ) return 0; + rc = p->xBusyHandler(p->pBusyArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} + +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. +*/ +int sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xBusyHandler = xBusy; + db->busyHandler.pBusyArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +void sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif + + +/* +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. +*/ +int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms>0 ){ + sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, + (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} + +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) && (db==0 || db->magic!=SQLITE_MAGIC_ZOMBIE) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + AtomicStore(&db->u1.isInterrupted, 1); +} + + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + assert( xValue==0 || xSFunc==0 ); + if( zFunctionName==0 /* Must have a valid name */ + || (xSFunc!=0 && xFinal!=0) /* Not both xSFunc and xFinal */ + || ((xFinal==0)!=(xStep==0)) /* Both or neither of xFinal and xStep */ + || ((xValue==0)!=(xInverse==0)) /* Both or neither of xValue, xInverse */ + || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) + || (255<(nName = sqlite3Strlen30( zFunctionName))) + ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); + extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| + SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + + /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But + ** the meaning is inverted. So flip the bit. */ + assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); + extraFlags ^= SQLITE_FUNC_UNSAFE; + + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db, 0); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->u.pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + testcase( p->funcFlags & SQLITE_DIRECTONLY ); + p->xSFunc = xSFunc ? xSFunc : xStep; + p->xFinalize = xFinal; + p->xValue = xValue; + p->xInverse = xInverse; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Worker function used by utf-8 APIs that create new functions: +** +** sqlite3_create_function() +** sqlite3_create_function_v2() +** sqlite3_create_window_function() +*/ +static int createFunctionApi( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3Malloc(sizeof(FuncDestructor)); + if( !pArg ){ + sqlite3OomFault(db); + xDestroy(p); + goto out; + } + pArg->nRef = 0; + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + xSFunc, xStep, xFinal, xValue, xInverse, pArg + ); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3_free(pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Create new user functions. +*/ +int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, 0); +} +int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, xDestroy); +} +int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, 0, xStep, + xFinal, xValue, xInverse, xDestroy); +} + +#ifndef SQLITE_OMIT_UTF16 +int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0,0,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + + +/* +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. +*/ +static void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = (const char*)sqlite3_user_data(context); + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); +} + +/* +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. +*/ +int sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int rc; + char *zCopy; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; + sqlite3_mutex_leave(db->mutex); + if( rc ) return SQLITE_OK; + zCopy = sqlite3_mprintf(zName); + if( zCopy==0 ) return SQLITE_NOMEM; + return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, + zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Register a trace function. The pArg from the previously registered trace +** is returned. +** +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->trace.xLegacy = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->trace.xV2 = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Register a profile function. The pArg from the previously registered +** profile function is returned. +** +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. +*/ +void *sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ +#endif /* SQLITE_OMIT_TRACE */ + +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +void *sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} + +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +void *sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +void *sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } +#endif + return SQLITE_OK; +} + +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +void *sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif +} + +/* +** Checkpoint database zDb. +*/ +int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb; /* Schema to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + }else{ + iDb = SQLITE_MAX_DB; /* This means process all schemas */ + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + + /* If there are no active statements, clear the interrupt flag at this + ** point. */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_DB then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART +** or TRUNCATE. +*/ +int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + testcase( iDb==SQLITE_MAX_ATTACHED ); /* See forum post a006d86f72 */ + testcase( iDb==SQLITE_MAX_DB ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_DB ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } + } + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); + return 0; +#endif +} + +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +const char *sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); + }else{ + testcase( db->pErr==0 ); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +const void *sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'b', 'a', 'd', ' ', 'p', 'a', 'r', 'a', 'm', 'e', 't', 'e', 'r', ' ', + 'o', 'r', ' ', 'o', 't', 'h', 'e', 'r', ' ', 'A', 'P', 'I', ' ', + 'm', 'i', 's', 'u', 's', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + sqlite3OomClear(db); + } + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +int sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode & db->errMask; +} +int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode; +} +int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} + +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db, 0); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } + } + } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM_BKPT; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); + return SQLITE_OK; +} + + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; /* IMP: R-53341-35419 */ +} + +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. The value +** stored in *pzFile is a database name acceptable to sqlite3_uri_parameter() +** and is in the same format as names created using sqlite3_create_filename(). +** The caller must invoke sqlite3_free_filename() (not sqlite3_free()!) on +** the value returned in *pzFile to avoid a memory leak. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+8; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ +#ifndef SQLITE_ENABLE_URI_00_ERROR + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; +#else + /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ + *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); + rc = SQLITE_ERROR; + goto parse_uri_out; +#endif + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + memset(zFile+iOut, 0, 4); /* end-of-options + empty journal filenames */ + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc64(nUri+8); + if( !zFile ) return SQLITE_NOMEM_BKPT; + memset(zFile, 0, 4); + zFile += 4; + if( nUri ){ + memcpy(zFile, zUri, nUri); + } + memset(zFile+nUri, 0, 4); + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free_filename(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + +/* +** This routine does the core work of extracting URI parameters from a +** database filename for the sqlite3_uri_parameter() interface. +*/ +static const char *uriParameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + + + +/* +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. +*/ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + int i; /* Loop counter */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_SUPER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + || sqlite3GlobalConfig.bCoreMutex +#endif + ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + if( isThreadsafe==0 ){ + sqlite3MutexWarnOnContention(db->mutex); + } + } + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames + | SQLITE_EnableTrigger + | SQLITE_EnableView + | SQLITE_CacheSpill +#if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 + | SQLITE_TrustedSchema +#endif +/* The SQLITE_DQS compile-time option determines the default settings +** for SQLITE_DBCONFIG_DQS_DDL and SQLITE_DBCONFIG_DQS_DML. +** +** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML +** ---------- ----------------------- ----------------------- +** undefined on on +** 3 on on +** 2 on off +** 1 off on +** 0 off off +** +** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) +** and so that is the default. But developers are encouranged to use +** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. +*/ +#if !defined(SQLITE_DQS) +# define SQLITE_DQS 3 +#endif +#if (SQLITE_DQS&1)==1 + | SQLITE_DqsDML +#endif +#if (SQLITE_DQS&2)==2 + | SQLITE_DqsDDL +#endif + +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif +#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) + | SQLITE_Fts3Tokenizer +#endif +#if defined(SQLITE_ENABLE_QPSG) + | SQLITE_EnableQPSG +#endif +#if defined(SQLITE_DEFAULT_DEFENSIVE) + | SQLITE_Defensive +#endif +#if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) + | SQLITE_LegacyAlter +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Parse the filename/URI argument + ** + ** Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + db->openFlags = flags; + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ + }else{ + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + } + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3Error(db, rc); + goto opendb_out; + } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ){ + sqlite3SetTextEncoding(db, SCHEMA_ENC(db)); + } + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. + */ + db->aDb[0].zDbSName = "main"; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + db->aDb[1].zDbSName = "temp"; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + + + /* Load compiled-in extensions */ + for(i=0; rc==SQLITE_OK && imDbFlags |= DBFLAG_InternalFunc; +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + sqlite3_free_filename(zOpen); + return rc & 0xff; +} + + +/* +** Open a new database handle. +*/ +int sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Open a new database handle. +*/ +int sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( zFilename==0 ) zFilename = "\000\000"; + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3ValueFree(pVal); + + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a new collation sequence with the database handle db. +*/ +int sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); +} + +/* +** Register a new collation sequence with the database handle db. +*/ +int sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a new collation sequence with the database handle db. +*/ +int sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +int sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +int sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. +*/ +int sqlite3_global_recover(void){ + return SQLITE_OK; +} +#endif + +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->autoCommit; +} + +/* +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error +** constants. They serve two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +int sqlite3ReportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} +int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); +} +int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); +} +int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); +} +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ + char zMsg[100]; + sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); +} +#endif +#ifdef SQLITE_DEBUG +int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); +} +int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); +} +#endif + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +void sqlite3_thread_cleanup(void){ +} +#endif + +/* +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. +*/ +int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } + } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = sqlite3ColumnType(pCol,0); + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = sqlite3StrBINARY; + } + +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +int sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} + +/* +** Enable or disable the extended result codes. +*/ +int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Invoke the xFileControl method on a particular database. +*/ +int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_VFS_POINTER ){ + *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_DATA_VERSION ){ + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=255 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; + }else{ + int nSave = db->busyHandler.nBusy; + rc = sqlite3OsFileControl(fd, op, pArg); + db->busyHandler.nBusy = nSave; + } + sqlite3BtreeLeave(pBtree); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifdef SQLITE_UNTESTABLE + UNUSED_PARAMETER(op); +#else + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, int x, sqlite3 *db); + ** + ** Control the seed for the pseudo-random number generator (PRNG) that + ** is built into SQLite. Cases: + ** + ** x!=0 && db!=0 Seed the PRNG to the current value of the + ** schema cookie in the main database for db, or + ** x if the schema cookie is zero. This case + ** is convenient to use with database fuzzers + ** as it allows the fuzzer some control over the + ** the PRNG seed. + ** + ** x!=0 && db==0 Seed the PRNG to the value of x. + ** + ** x==0 && db==0 Revert to default behavior of using the + ** xRandomness method on the primary VFS. + ** + ** This test-control also resets the PRNG so that the new seed will + ** be used for the next call to sqlite3_randomness(). + */ +#ifndef SQLITE_OMIT_WSD + case SQLITE_TESTCTRL_PRNG_SEED: { + int x = va_arg(ap, int); + int y; + sqlite3 *db = va_arg(ap, sqlite3*); + assert( db==0 || db->aDb[0].pSchema!=0 ); + if( db && (y = db->aDb[0].pSchema->schema_cookie)!=0 ){ x = y; } + sqlite3Config.iPrngSeed = x; + sqlite3_randomness(0,0); + break; + } +#endif + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X) if X is true, or 0 if X is false. + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = x ? ALWAYS(x) : 0; + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = va_arg(ap, u32); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, subsequent calls to localtime() + ** and its variants fail. If onoff is zero, undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, sqlite3*); + ** + ** Toggle the ability to use internal functions on or off for + ** the database connection given in the argument. + */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->mDbFlags ^= DBFLAG_InternalFunc; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS, int); + ** + ** Set or clear a flag that causes SQLite to verify that type, name, + ** and tbl_name fields of the sqlite_schema table. This is normally + ** on, but it is sometimes useful to turn it off for testing. + ** + ** 2020-07-22: Disabling EXTRA_SCHEMA_CHECKS also disables the + ** verification of rootpage numbers when parsing the schema. This + ** is useful to make it easier to reach strange internal error states + ** during testing. The EXTRA_SCHEMA_CHECKS setting is always enabled + ** in production. + */ + case SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS: { + sqlite3GlobalConfig.bExtraSchemaChecks = va_arg(ap, int); + break; + } + + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,unsigned int, + unsigned char,unsigned char); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + ** + ** This test control is used to create imposter tables. "db" is a pointer + ** to the database connection. dbName is the database name (ex: "main" or + ** "temp") which will receive the imposter. "onOff" turns imposter mode on + ** or off. "tnum" is the root page of the b-tree to which the imposter + ** table should connect. + ** + ** Enable imposter mode only when the schema has already been parsed. Then + ** run a single CREATE TABLE statement to construct the imposter table in + ** the parsed schema. Then turn imposter mode back off again. + ** + ** If onOff==0 and tnum>0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3_mutex_leave(db->mutex); + break; + } + +#if defined(YYCOVERAGE) + /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) + ** + ** This test control (only available when SQLite is compiled with + ** -DYYCOVERAGE) writes a report onto "out" that shows all + ** state/lookahead combinations in the parser state machine + ** which are never exercised. If any state is missed, make the + ** return code SQLITE_ERROR. + */ + case SQLITE_TESTCTRL_PARSER_COVERAGE: { + FILE *out = va_arg(ap, FILE*); + if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; + break; + } +#endif /* defined(YYCOVERAGE) */ + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, sqlite3_context*); + ** + ** This test-control causes the most recent sqlite3_result_int64() value + ** to be interpreted as a MEM_IntReal instead of as an MEM_Int. Normally, + ** MEM_IntReal values only arise during an INSERT operation of integer + ** values into a REAL column, so they can be challenging to test. This + ** test-control enables us to write an intreal() SQL function that can + ** inject an intreal() value at arbitrary places in an SQL statement, + ** for testing purposes. + */ + case SQLITE_TESTCTRL_RESULT_INTREAL: { + sqlite3_context *pCtx = va_arg(ap, sqlite3_context*); + sqlite3ResultIntReal(pCtx); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SEEK_COUNT, + ** sqlite3 *db, // Database connection + ** u64 *pnSeek // Write seek count here + ** ); + ** + ** This test-control queries the seek-counter on the "main" database + ** file. The seek-counter is written into *pnSeek and is then reset. + ** The seek-count is only available if compiled with SQLITE_DEBUG. + */ + case SQLITE_TESTCTRL_SEEK_COUNT: { + sqlite3 *db = va_arg(ap, sqlite3*); + u64 *pn = va_arg(ap, sqlite3_uint64*); + *pn = sqlite3BtreeSeekCount(db->aDb->pBt); + (void)db; /* Silence harmless unused variable warning */ + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, op, ptr) + ** + ** "ptr" is a pointer to a u32. + ** + ** op==0 Store the current sqlite3SelectTrace in *ptr + ** op==1 Set sqlite3SelectTrace to the value *ptr + ** op==3 Store the current sqlite3WhereTrace in *ptr + ** op==3 Set sqlite3WhereTrace to the value *ptr + */ + case SQLITE_TESTCTRL_TRACEFLAGS: { + int opTrace = va_arg(ap, int); + u32 *ptr = va_arg(ap, u32*); + switch( opTrace ){ + case 0: *ptr = sqlite3SelectTrace; break; + case 1: sqlite3SelectTrace = *ptr; break; + case 2: *ptr = sqlite3WhereTrace; break; + case 3: sqlite3WhereTrace = *ptr; break; + } + break; + } + } + va_end(ap); +#endif /* SQLITE_UNTESTABLE */ + return rc; +} + +/* +** The Pager stores the Database filename, Journal filename, and WAL filename +** consecutively in memory, in that order. The database filename is prefixed +** by four zero bytes. Locate the start of the database filename by searching +** backwards for the first byte following four consecutive zero bytes. +** +** This only works if the filename passed in was obtained from the Pager. +*/ +static const char *databaseName(const char *zName){ + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + return zName; +} + +/* +** Append text z[] to the end of p[]. Return a pointer to the first +** character after then zero terminator on the new text in p[]. +*/ +static char *appendText(char *p, const char *z){ + size_t n = strlen(z); + memcpy(p, z, n+1); + return p+n+1; +} + +/* +** Allocate memory to hold names for a database, journal file, WAL file, +** and query parameters. The pointer returned is valid for use by +** sqlite3_filename_database() and sqlite3_uri_parameter() and related +** functions. +** +** Memory layout must be compatible with that generated by the pager +** and expected by sqlite3_uri_parameter() and databaseName(). +*/ +char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +){ + sqlite3_int64 nByte; + int i; + char *pResult, *p; + nByte = strlen(zDatabase) + strlen(zJournal) + strlen(zWal) + 10; + for(i=0; i0 ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return zFilename[0] ? zFilename : 0; +} + +/* +** Return a boolean value for a query parameter. +*/ +int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==0 ){ + bDflt = v; + } + return bDflt; +} + +/* +** Translate a filename that was handed to a VFS routine into the corresponding +** database, journal, or WAL file. +** +** It is an error to pass this routine a filename string that was not +** passed into the VFS from the SQLite core. Doing so is similar to +** passing free() a pointer that was not obtained from malloc() - it is +** an error that we cannot easily detect but that will likely cause memory +** corruption. +*/ +const char *sqlite3_filename_database(const char *zFilename){ + return databaseName(zFilename); +} +const char *sqlite3_filename_journal(const char *zFilename){ + zFilename = databaseName(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return zFilename + 1; +} +const char *sqlite3_filename_wal(const char *zFilename){ +#ifdef SQLITE_OMIT_WAL + return 0; +#else + zFilename = sqlite3_filename_journal(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + return zFilename; +#endif +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; + return iDb<0 ? 0 : db->aDb[iDb].pBt; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Obtain a snapshot handle for the snapshot of database zDb currently +** being read by handle db. +*/ +int sqlite3_snapshot_get( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot **ppSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Open a read-transaction on the snapshot idendified by pSnapshot. +*/ +int sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( db->autoCommit==0 ){ + int iDb; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ){ + Pager *pPager = sqlite3BtreePager(pBt); + int bUnlock = 0; + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_NONE ){ + if( db->nVdbeActive==0 ){ + rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot); + if( rc==SQLITE_OK ){ + bUnlock = 1; + rc = sqlite3BtreeCommit(pBt); + } + } + }else{ + rc = SQLITE_OK; + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotOpen(pPager, pSnapshot); + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + } + if( bUnlock ){ + sqlite3PagerSnapshotUnlock(pPager); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. +*/ +int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; + int iDb; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_NONE==sqlite3BtreeTxnState(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } + } + } + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Free a snapshot handle obtained from sqlite3_snapshot_get(). +*/ +void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ + sqlite3_free(pSnapshot); +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* +** Given the name of a compile-time option, return true if that option +** was used and false if not. +** +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. +*/ +int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + int nOpt; + const char **azCompileOpt; + +#if SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + azCompileOpt = sqlite3CompileOptions(&nOpt); + + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since nOpt is normally in single digits, a linear search is + ** adequate. No need for a binary search. */ + for(i=0; i=0 && N0 && (n>mem0.hardLimit || n==0) ){ + n = mem0.hardLimit; + } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; +} +void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); +} + +/* +** Set the hard heap-size limit for the library. An argument of zero +** disables the hard heap limit. A negative argument is a no-op used +** to obtain the return value without affecting the hard heap limit. +** +** The return value is the value of the hard heap limit just prior to +** calling this interface. +** +** Setting the hard heap limit will also activate the soft heap limit +** and constrain the soft heap limit to be no more than the hard heap +** limit. +*/ +sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.hardLimit; + if( n>=0 ){ + mem0.hardLimit = n; + if( n0 ); + + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ + nFull = sqlite3GlobalConfig.m.xRoundup(n); + + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ + AtomicStore(&mem0.nearlyFull, 1); + sqlite3MallocAlarm(nFull); + if( mem0.hardLimit ){ + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.hardLimit - nFull ){ + *pp = 0; + return; + } + } + }else{ + AtomicStore(&mem0.nearlyFull, 0); + } + } + p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( p==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm(nFull); + p = sqlite3GlobalConfig.m.xMalloc(nFull); + } +#endif + if( p ){ + nFull = sqlite3MallocSize(p); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); + } + *pp = p; +} + +/* +** Allocate memory. This routine is like sqlite3_malloc() except that it +** assumes the memory subsystem has already been initialized. +*/ +void *sqlite3Malloc(u64 n){ + void *p; + if( n==0 || n>=0x7fffff00 ){ + /* A memory allocation of a number of bytes which is near the maximum + ** signed integer value might cause an integer overflow inside of the + ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving + ** 255 bytes of overhead. SQLite itself will never use anything near + ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + p = 0; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + mallocWithAlarm((int)n, &p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + p = sqlite3GlobalConfig.m.xMalloc((int)n); + } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-11148-40995 */ + return p; +} + +/* +** This version of the memory allocation is for use by the application. +** First make sure the memory subsystem is initialized, then do the +** allocation. +*/ +void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return n<=0 ? 0 : sqlite3Malloc(n); +} +void *sqlite3_malloc64(sqlite3_uint64 n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Malloc(n); +} + +/* +** TRUE if p is a lookaside memory allocation from db +*/ +#ifndef SQLITE_OMIT_LOOKASIDE +static int isLookaside(sqlite3 *db, void *p){ + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); +} +#else +#define isLookaside(A,B) 0 +#endif + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3Malloc() or sqlite3_malloc(). +*/ +int sqlite3MallocSize(void *p){ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return sqlite3GlobalConfig.m.xSize(p); +} +static int lookasideMallocSize(sqlite3 *db, void *p){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + return plookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL; +#else + return db->lookaside.szTrue; +#endif +} +int sqlite3DbMallocSize(sqlite3 *db, void *p){ + assert( p!=0 ); +#ifdef SQLITE_DEBUG + if( db==0 || !isLookaside(db,p) ){ + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + } + } +#endif + if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return LOOKASIDE_SMALL; + } +#endif + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.szTrue; + } + } + } + return sqlite3GlobalConfig.m.xSize(p); +} +sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return p ? sqlite3GlobalConfig.m.xSize(p) : 0; +} + +/* +** Free memory previously obtained from sqlite3Malloc(). +*/ +void sqlite3_free(void *p){ + if( p==0 ) return; /* IMP: R-49053-54554 */ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, sqlite3MallocSize(p)); + sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } +} + +/* +** Add the size of memory allocation "p" to the count in +** *db->pnBytesFreed. +*/ +static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); +} + +/* +** Free memory that might be associated with a particular database +** connection. Calling sqlite3DbFree(D,X) for X==0 is a harmless no-op. +** The sqlite3DbFreeNN(D,X) version requires that X be non-NULL. +*/ +void sqlite3DbFreeNN(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( db ){ + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} +void sqlite3DbFree(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + if( p ) sqlite3DbFreeNN(db, p); +} + +/* +** Change the size of an existing memory allocation +*/ +void *sqlite3Realloc(void *pOld, u64 nBytes){ + int nOld, nNew, nDiff; + void *pNew; + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, (u8)~MEMTYPE_HEAP) ); + if( pOld==0 ){ + return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ + } + if( nBytes==0 ){ + sqlite3_free(pOld); /* IMP: R-26507-47431 */ + return 0; + } + if( nBytes>=0x7fffff00 ){ + /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ + return 0; + } + nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ + nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); + if( nOld==nNew ){ + pNew = pOld; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_int64 nUsed; + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); + nDiff = nNew - nOld; + if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); + if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ + sqlite3_mutex_leave(mem0.mutex); + return 0; + } + } + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } +#endif + if( pNew ){ + nNew = sqlite3MallocSize(pNew); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); + } + sqlite3_mutex_leave(mem0.mutex); + }else{ + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */ + return pNew; +} + +/* +** The public interface to sqlite3Realloc. Make sure that the memory +** subsystem is initialized prior to invoking sqliteRealloc. +*/ +void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + if( n<0 ) n = 0; /* IMP: R-26507-47431 */ + return sqlite3Realloc(pOld, n); +} +void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Realloc(pOld, n); +} + + +/* +** Allocate and zero memory. +*/ +void *sqlite3MallocZero(u64 n){ + void *p = sqlite3Malloc(n); + if( p ){ + memset(p, 0, (size_t)n); + } + return p; +} + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +*/ +void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ + void *p; + testcase( db==0 ); + p = sqlite3DbMallocRaw(db, n); + if( p ) memset(p, 0, (size_t)n); + return p; +} + + +/* Finish the work of sqlite3DbMallocRawNN for the unusual and +** slower case when the allocation cannot be fulfilled using lookaside. +*/ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p; + assert( db!=0 ); + p = sqlite3Malloc(n); + if( !p ) sqlite3OomFault(db); + sqlite3MemdebugSetType(p, + (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); + return p; +} + +/* +** Allocate memory, either lookaside (if possible) or heap. +** If the allocation fails, set the mallocFailed flag in +** the connection pointer. +** +** If db!=0 and db->mallocFailed is true (indicating a prior malloc +** failure on the same database connection) then always return 0. +** Hence for a particular database connection, once malloc starts +** failing, it fails consistently until mallocFailed is reset. +** This is an important assumption. There are many places in the +** code that do things like this: +** +** int *a = (int*)sqlite3DbMallocRaw(db, 100); +** int *b = (int*)sqlite3DbMallocRaw(db, 200); +** if( b ) a[10] = 9; +** +** In other words, if a subsequent malloc (ex: "b") worked, it is assumed +** that all prior mallocs (ex: "a") worked too. +** +** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is +** not a NULL pointer. +*/ +void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ + void *p; + if( db ) return sqlite3DbMallocRawNN(db, n); + p = sqlite3Malloc(n); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + return p; +} +void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ +#ifndef SQLITE_OMIT_LOOKASIDE + LookasideSlot *pBuf; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( n>db->lookaside.sz ){ + if( !db->lookaside.bDisable ){ + db->lookaside.anStat[1]++; + }else if( db->mallocFailed ){ + return 0; + } + return dbMallocRawFinish(db, n); + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( n<=LOOKASIDE_SMALL ){ + if( (pBuf = db->lookaside.pSmallFree)!=0 ){ + db->lookaside.pSmallFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){ + db->lookaside.pSmallInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + } + } +#endif + if( (pBuf = db->lookaside.pFree)!=0 ){ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pInit)!=0 ){ + db->lookaside.pInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else{ + db->lookaside.anStat[2]++; + } +#else + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->mallocFailed ){ + return 0; + } +#endif + return dbMallocRawFinish(db, n); +} + +/* Forward declaration */ +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); + +/* +** Resize the block of memory pointed to by p to n bytes. If the +** resize fails, set the mallocFailed flag in the connection object. +*/ +void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ + assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); + assert( sqlite3_mutex_held(db->mutex) ); + if( ((uptr)p)<(uptr)db->lookaside.pEnd ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ + if( n<=LOOKASIDE_SMALL ) return p; + }else +#endif + if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ + if( n<=db->lookaside.szTrue ) return p; + } + } + return dbReallocFinish(db, p, n); +} +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ + void *pNew = 0; + assert( db!=0 ); + assert( p!=0 ); + if( db->mallocFailed==0 ){ + if( isLookaside(db, p) ){ + pNew = sqlite3DbMallocRawNN(db, n); + if( pNew ){ + memcpy(pNew, p, lookasideMallocSize(db, p)); + sqlite3DbFree(db, p); + } + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + pNew = sqlite3Realloc(p, n); + if( !pNew ){ + sqlite3OomFault(db); + } + sqlite3MemdebugSetType(pNew, + (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + } + } + return pNew; +} + +/* +** Attempt to reallocate p. If the reallocation fails, then free p +** and set the mallocFailed flag in the database connection. +*/ +void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ + void *pNew; + pNew = sqlite3DbRealloc(db, p, n); + if( !pNew ){ + sqlite3DbFree(db, p); + } + return pNew; +} + +/* +** Make a copy of a string in memory obtained from sqliteMalloc(). These +** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This +** is because when memory debugging is turned on, these two functions are +** called via macros that record the current file and line number in the +** ThreadData structure. +*/ +char *sqlite3DbStrDup(sqlite3 *db, const char *z){ + char *zNew; + size_t n; + if( z==0 ){ + return 0; + } + n = strlen(z) + 1; + zNew = sqlite3DbMallocRaw(db, n); + if( zNew ){ + memcpy(zNew, z, n); + } + return zNew; +} +char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ + char *zNew; + assert( db!=0 ); + assert( z!=0 || n==0 ); + assert( (n&0x7fffffff)==n ); + zNew = z ? sqlite3DbMallocRawNN(db, n+1) : 0; + if( zNew ){ + memcpy(zNew, z, (size_t)n); + zNew[n] = 0; + } + return zNew; +} + +/* +** The text between zStart and zEnd represents a phrase within a larger +** SQL statement. Make a copy of this phrase in space obtained form +** sqlite3DbMalloc(). Omit leading and trailing whitespace. +*/ +char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ + int n; + while( sqlite3Isspace(zStart[0]) ) zStart++; + n = (int)(zEnd - zStart); + while( ALWAYS(n>0) && sqlite3Isspace(zStart[n-1]) ) n--; + return sqlite3DbStrNDup(db, zStart, n); +} + +/* +** Free any prior content in *pz and replace it with a copy of zNew. +*/ +void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ + sqlite3DbFree(db, *pz); + *pz = sqlite3DbStrDup(db, zNew); +} + +/* +** Call this routine to record the fact that an OOM (out-of-memory) error +** has happened. This routine will set db->mallocFailed, and also +** temporarily disable the lookaside memory allocator and interrupt +** any running VDBEs. +*/ +void sqlite3OomFault(sqlite3 *db){ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ + AtomicStore(&db->u1.isInterrupted, 1); + } + DisableLookaside; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; + } + } +} + +/* +** This routine reactivates the memory allocator and clears the +** db->mallocFailed flag as necessary. +** +** The memory allocator is not restarted if there are running +** VDBEs. +*/ +void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; + AtomicStore(&db->u1.isInterrupted, 0); + assert( db->lookaside.bDisable>0 ); + EnableLookaside; + } +} + +/* +** Take actions at the end of an API call to deal with error codes. +*/ +static SQLITE_NOINLINE int apiHandleError(sqlite3 *db, int rc){ + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomClear(db); + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM_BKPT; + } + return rc & db->errMask; +} + +/* +** This function must be called before exiting any API function (i.e. +** returning control to the user) that has called sqlite3_malloc or +** sqlite3_realloc. +** +** The returned value is normally a copy of the second argument to this +** function. However, if a malloc() failure has occurred since the previous +** invocation SQLITE_NOMEM is returned instead. +** +** If an OOM as occurred, then the connection error-code (the value +** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. +*/ +int sqlite3ApiExit(sqlite3* db, int rc){ + /* If the db handle must hold the connection handle mutex here. + ** Otherwise the read (and possible write) of db->mallocFailed + ** is unsafe, as is the call to sqlite3Error(). + */ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + if( db->mallocFailed || rc ){ + return apiHandleError(db, rc); + } + return rc & db->errMask; +} diff --git a/third_party/sqlite3/mem0.c b/third_party/sqlite3/mem0.c new file mode 100644 index 000000000..52ac7fc02 --- /dev/null +++ b/third_party/sqlite3/mem0.c @@ -0,0 +1,60 @@ +/* +** 2008 October 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a no-op memory allocation drivers for use when +** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented +** here always fail. SQLite will not operate with these drivers. These +** are merely placeholders. Real drivers must be substituted using +** sqlite3_config() before SQLite will operate. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_ZERO_MALLOC + +/* +** No-op versions of all memory allocation routines +*/ +static void *sqlite3MemMalloc(int nByte){ return 0; } +static void sqlite3MemFree(void *pPrior){ return; } +static void *sqlite3MemRealloc(void *pPrior, int nByte){ return 0; } +static int sqlite3MemSize(void *pPrior){ return 0; } +static int sqlite3MemRoundup(int n){ return n; } +static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } +static void sqlite3MemShutdown(void *NotUsed){ return; } + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_ZERO_MALLOC */ diff --git a/third_party/sqlite3/mem1.c b/third_party/sqlite3/mem1.c new file mode 100644 index 000000000..4c7dd182a --- /dev/null +++ b/third_party/sqlite3/mem1.c @@ -0,0 +1,292 @@ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. The content of +** this file is only used if SQLITE_SYSTEM_MALLOC is defined. The +** SQLITE_SYSTEM_MALLOC macro is defined automatically if neither the +** SQLITE_MEMDEBUG nor the SQLITE_WIN32_MALLOC macros are defined. The +** default configuration is to use memory allocation routines in this +** file. +** +** C-preprocessor macro summary: +** +** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if +** the malloc_usable_size() interface exists +** on the target platform. Or, this symbol +** can be set manually, if desired. +** If an equivalent interface exists by +** a different name, using a separate -D +** option to rename it. +** +** SQLITE_WITHOUT_ZONEMALLOC Some older macs lack support for the zone +** memory allocator. Set this symbol to enable +** building on older macs. +** +** SQLITE_WITHOUT_MSIZE Set this symbol to disable the use of +** _msize() on windows systems. This might +** be necessary when compiling for Delphi, +** for example. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_SYSTEM_MALLOC +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + +/* +** Use the zone allocator available on apple products unless the +** SQLITE_WITHOUT_ZONEMALLOC symbol is defined. +*/ +#include +#include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE +#include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ +static malloc_zone_t* _sqliteZone_; +#define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) +#define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); +#define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) \ + (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) + +#else /* if not __APPLE__ */ + +/* +** Use standard C library malloc and free on non-Apple systems. +** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. +*/ +#define SQLITE_MALLOC(x) malloc(x) +#define SQLITE_FREE(x) free(x) +#define SQLITE_REALLOC(x,y) realloc((x),(y)) + +/* +** The malloc.h header file is needed for malloc_usable_size() function +** on some systems (e.g. Linux). +*/ +#if HAVE_MALLOC_H && HAVE_MALLOC_USABLE_SIZE +# define SQLITE_USE_MALLOC_H 1 +# define SQLITE_USE_MALLOC_USABLE_SIZE 1 +/* +** The MSVCRT has malloc_usable_size(), but it is called _msize(). The +** use of _msize() is automatic, but can be disabled by compiling with +** -DSQLITE_WITHOUT_MSIZE. Using the _msize() function also requires +** the malloc.h header file. +*/ +#elif defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MSIZE +#endif + +/* +** Include the malloc.h header file, if necessary. Also set define macro +** SQLITE_MALLOCSIZE to the appropriate function name, which is _msize() +** for MSVC and malloc_usable_size() for most other systems (e.g. Linux). +** The memory size function can always be overridden manually by defining +** the macro SQLITE_MALLOCSIZE to the desired function name. +*/ +#if defined(SQLITE_USE_MALLOC_H) +#include "libc/mem/mem.h" +#if defined(SQLITE_USE_MALLOC_USABLE_SIZE) +#if !defined(SQLITE_MALLOCSIZE) +#define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) +#endif +#elif defined(SQLITE_USE_MSIZE) +#if !defined(SQLITE_MALLOCSIZE) +#define SQLITE_MALLOCSIZE _msize +#endif +#endif +#endif /* defined(SQLITE_USE_MALLOC_H) */ + +#endif /* __APPLE__ or not __APPLE__ */ + +/* +** Like malloc(), but remember the size of the allocation +** so that we can find it later using sqlite3MemSize(). +** +** For this low-level routine, we are guaranteed that nByte>0 because +** cases of nByte<=0 will be intercepted and dealt with by higher level +** routines. +*/ +static void *sqlite3MemMalloc(int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return p; +#else + sqlite3_int64 *p; + assert( nByte>0 ); + testcase( ROUND8(nByte)!=nByte ); + p = SQLITE_MALLOC( nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return (void *)p; +#endif +} + +/* +** Like free() but works for allocations obtained from sqlite3MemMalloc() +** or sqlite3MemRealloc(). +** +** For this low-level routine, we already know that pPrior!=0 since +** cases where pPrior==0 will have been intecepted and dealt with +** by higher-level routines. +*/ +static void sqlite3MemFree(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + SQLITE_FREE(pPrior); +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 ); + p--; + SQLITE_FREE(p); +#endif +} + +/* +** Report the allocated size of a prior return from xMalloc() +** or xRealloc(). +*/ +static int sqlite3MemSize(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + assert( pPrior!=0 ); + return (int)SQLITE_MALLOCSIZE(pPrior); +#else + sqlite3_int64 *p; + assert( pPrior!=0 ); + p = (sqlite3_int64*)pPrior; + p--; + return (int)p[0]; +#endif +} + +/* +** Like realloc(). Resize an allocation previously obtained from +** sqlite3MemMalloc(). +** +** For this low-level interface, we know that pPrior!=0. Cases where +** pPrior==0 while have been intercepted by higher-level routine and +** redirected to xMalloc. Similarly, we know that nByte>0 because +** cases where nByte<=0 will have been intercepted by higher-level +** routines and redirected to xFree. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_REALLOC(pPrior, nByte); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + SQLITE_MALLOCSIZE(pPrior), nByte); + } + return p; +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 && nByte>0 ); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ + p--; + p = SQLITE_REALLOC(p, nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + sqlite3MemSize(pPrior), nByte); + } + return (void*)p; +#endif +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Initialize this module. +*/ +static int sqlite3MemInit(void *NotUsed){ +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + int cpuCount; + size_t len; + if( _sqliteZone_ ){ + return SQLITE_OK; + } + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); + } +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ + UNUSED_PARAMETER(NotUsed); + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return; +} + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_SYSTEM_MALLOC */ diff --git a/third_party/sqlite3/mem2.c b/third_party/sqlite3/mem2.c new file mode 100644 index 000000000..3249f81fc --- /dev/null +++ b/third_party/sqlite3/mem2.c @@ -0,0 +1,529 @@ +/* +** 2007 August 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs while adding lots of additional debugging +** information to each allocation in order to help detect and fix memory +** leaks and memory usage errors. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** This version of the memory allocator is used only if the +** SQLITE_MEMDEBUG macro is defined +*/ +#ifdef SQLITE_MEMDEBUG + +/* +** The backtrace functionality is only available with GLIBC +*/ +#ifdef __GLIBC__ + extern int backtrace(void**,int); + extern void backtrace_symbols_fd(void*const*,int,int); +#else +# define backtrace(A,B) 1 +# define backtrace_symbols_fd(A,B,C) +#endif +#include "libc/stdio/stdio.h" + +/* +** Each memory allocation looks like this: +** +** ------------------------------------------------------------------------ +** | Title | backtrace pointers | MemBlockHdr | allocation | EndGuard | +** ------------------------------------------------------------------------ +** +** The application code sees only a pointer to the allocation. We have +** to back up from the allocation pointer to find the MemBlockHdr. The +** MemBlockHdr tells us the size of the allocation and the number of +** backtrace pointers. There is also a guard word at the end of the +** MemBlockHdr. +*/ +struct MemBlockHdr { + i64 iSize; /* Size of this allocation */ + struct MemBlockHdr *pNext, *pPrev; /* Linked list of all unfreed memory */ + char nBacktrace; /* Number of backtraces on this alloc */ + char nBacktraceSlots; /* Available backtrace slots */ + u8 nTitle; /* Bytes of title; includes '\0' */ + u8 eType; /* Allocation type code */ + int iForeGuard; /* Guard word for sanity */ +}; + +/* +** Guard words +*/ +#define FOREGUARD 0x80F5E153 +#define REARGUARD 0xE4676B53 + +/* +** Number of malloc size increments to track. +*/ +#define NCSIZE 1000 + +/* +** All of the static variables used by this module are collected +** into a single structure named "mem". This is to keep the +** static variables organized and to reduce namespace pollution +** when this module is combined with other in the amalgamation. +*/ +static struct { + + /* + ** Mutex to control access to the memory allocation subsystem. + */ + sqlite3_mutex *mutex; + + /* + ** Head and tail of a linked list of all outstanding allocations + */ + struct MemBlockHdr *pFirst; + struct MemBlockHdr *pLast; + + /* + ** The number of levels of backtrace to save in new allocations. + */ + int nBacktrace; + void (*xBacktrace)(int, int, void **); + + /* + ** Title text to insert in front of each block + */ + int nTitle; /* Bytes of zTitle to save. Includes '\0' and padding */ + char zTitle[100]; /* The title text */ + + /* + ** sqlite3MallocDisallow() increments the following counter. + ** sqlite3MallocAllow() decrements it. + */ + int disallow; /* Do not allow memory allocation */ + + /* + ** Gather statistics on the sizes of memory allocations. + ** nAlloc[i] is the number of allocation attempts of i*8 + ** bytes. i==NCSIZE is the number of allocation attempts for + ** sizes more than NCSIZE*8 bytes. + */ + int nAlloc[NCSIZE]; /* Total number of allocations */ + int nCurrent[NCSIZE]; /* Current number of allocations */ + int mxCurrent[NCSIZE]; /* Highwater mark for nCurrent */ + +} mem; + + +/* +** Adjust memory usage statistics +*/ +static void adjustStats(int iSize, int increment){ + int i = ROUND8(iSize)/8; + if( i>NCSIZE-1 ){ + i = NCSIZE - 1; + } + if( increment>0 ){ + mem.nAlloc[i]++; + mem.nCurrent[i]++; + if( mem.nCurrent[i]>mem.mxCurrent[i] ){ + mem.mxCurrent[i] = mem.nCurrent[i]; + } + }else{ + mem.nCurrent[i]--; + assert( mem.nCurrent[i]>=0 ); + } +} + +/* +** Given an allocation, find the MemBlockHdr for that allocation. +** +** This routine checks the guards at either end of the allocation and +** if they are incorrect it asserts. +*/ +static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ + struct MemBlockHdr *p; + int *pInt; + u8 *pU8; + int nReserve; + + p = (struct MemBlockHdr*)pAllocation; + p--; + assert( p->iForeGuard==(int)FOREGUARD ); + nReserve = ROUND8(p->iSize); + pInt = (int*)pAllocation; + pU8 = (u8*)pAllocation; + assert( pInt[nReserve/sizeof(int)]==(int)REARGUARD ); + /* This checks any of the "extra" bytes allocated due + ** to rounding up to an 8 byte boundary to ensure + ** they haven't been overwritten. + */ + while( nReserve-- > p->iSize ) assert( pU8[nReserve]==0x65 ); + return p; +} + +/* +** Return the number of bytes currently allocated at address p. +*/ +static int sqlite3MemSize(void *p){ + struct MemBlockHdr *pHdr; + if( !p ){ + return 0; + } + pHdr = sqlite3MemsysGetHeader(p); + return (int)pHdr->iSize; +} + +/* +** Initialize the memory allocation subsystem. +*/ +static int sqlite3MemInit(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( (sizeof(struct MemBlockHdr)&7) == 0 ); + if( !sqlite3GlobalConfig.bMemstat ){ + /* If memory status is enabled, then the malloc.c wrapper will already + ** hold the STATIC_MEM mutex when the routines here are invoked. */ + mem.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + return SQLITE_OK; +} + +/* +** Deinitialize the memory allocation subsystem. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem.mutex = 0; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Fill a buffer with pseudo-random bytes. This is used to preset +** the content of a new memory allocation to unpredictable values and +** to clear the content of a freed allocation to unpredictable values. +*/ +static void randomFill(char *pBuf, int nByte){ + unsigned int x, y, r; + x = SQLITE_PTR_TO_INT(pBuf); + y = nByte | 1; + while( nByte >= 4 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(int*)pBuf = r; + pBuf += 4; + nByte -= 4; + } + while( nByte-- > 0 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(pBuf++) = r & 0xff; + } +} + +/* +** Allocate nByte bytes of memory. +*/ +static void *sqlite3MemMalloc(int nByte){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + int *pInt; + void *p = 0; + int totalSize; + int nReserve; + sqlite3_mutex_enter(mem.mutex); + assert( mem.disallow==0 ); + nReserve = ROUND8(nByte); + totalSize = nReserve + sizeof(*pHdr) + sizeof(int) + + mem.nBacktrace*sizeof(void*) + mem.nTitle; + p = malloc(totalSize); + if( p ){ + z = p; + pBt = (void**)&z[mem.nTitle]; + pHdr = (struct MemBlockHdr*)&pBt[mem.nBacktrace]; + pHdr->pNext = 0; + pHdr->pPrev = mem.pLast; + if( mem.pLast ){ + mem.pLast->pNext = pHdr; + }else{ + mem.pFirst = pHdr; + } + mem.pLast = pHdr; + pHdr->iForeGuard = FOREGUARD; + pHdr->eType = MEMTYPE_HEAP; + pHdr->nBacktraceSlots = mem.nBacktrace; + pHdr->nTitle = mem.nTitle; + if( mem.nBacktrace ){ + void *aAddr[40]; + pHdr->nBacktrace = backtrace(aAddr, mem.nBacktrace+1)-1; + memcpy(pBt, &aAddr[1], pHdr->nBacktrace*sizeof(void*)); + assert(pBt[0]); + if( mem.xBacktrace ){ + mem.xBacktrace(nByte, pHdr->nBacktrace-1, &aAddr[1]); + } + }else{ + pHdr->nBacktrace = 0; + } + if( mem.nTitle ){ + memcpy(z, mem.zTitle, mem.nTitle); + } + pHdr->iSize = nByte; + adjustStats(nByte, +1); + pInt = (int*)&pHdr[1]; + pInt[nReserve/sizeof(int)] = REARGUARD; + randomFill((char*)pInt, nByte); + memset(((char*)pInt)+nByte, 0x65, nReserve-nByte); + p = (void*)pInt; + } + sqlite3_mutex_leave(mem.mutex); + return p; +} + +/* +** Free memory. +*/ +static void sqlite3MemFree(void *pPrior){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 + || mem.mutex!=0 ); + pHdr = sqlite3MemsysGetHeader(pPrior); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + sqlite3_mutex_enter(mem.mutex); + if( pHdr->pPrev ){ + assert( pHdr->pPrev->pNext==pHdr ); + pHdr->pPrev->pNext = pHdr->pNext; + }else{ + assert( mem.pFirst==pHdr ); + mem.pFirst = pHdr->pNext; + } + if( pHdr->pNext ){ + assert( pHdr->pNext->pPrev==pHdr ); + pHdr->pNext->pPrev = pHdr->pPrev; + }else{ + assert( mem.pLast==pHdr ); + mem.pLast = pHdr->pPrev; + } + z = (char*)pBt; + z -= pHdr->nTitle; + adjustStats((int)pHdr->iSize, -1); + randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + + (int)pHdr->iSize + sizeof(int) + pHdr->nTitle); + free(z); + sqlite3_mutex_leave(mem.mutex); +} + +/* +** Change the size of an existing memory allocation. +** +** For this debugging implementation, we *always* make a copy of the +** allocation into a new place in memory. In this way, if the +** higher level code is using pointer to the old allocation, it is +** much more likely to break and we are much more liking to find +** the error. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ + struct MemBlockHdr *pOldHdr; + void *pNew; + assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ + pOldHdr = sqlite3MemsysGetHeader(pPrior); + pNew = sqlite3MemMalloc(nByte); + if( pNew ){ + memcpy(pNew, pPrior, (int)(nByteiSize ? nByte : pOldHdr->iSize)); + if( nByte>pOldHdr->iSize ){ + randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - (int)pOldHdr->iSize); + } + sqlite3MemFree(pPrior); + } + return pNew; +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +/* +** Set the "type" of an allocation. +*/ +void sqlite3MemdebugSetType(void *p, u8 eType){ + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); + pHdr->eType = eType; + } +} + +/* +** Return TRUE if the mask of type in eType matches the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +*/ +int sqlite3MemdebugHasType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)==0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); +*/ +int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)!=0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Set the number of backtrace levels kept for each allocation. +** A value of zero turns off backtracing. The number is always rounded +** up to a multiple of 2. +*/ +void sqlite3MemdebugBacktrace(int depth){ + if( depth<0 ){ depth = 0; } + if( depth>20 ){ depth = 20; } + depth = (depth+1)&0xfe; + mem.nBacktrace = depth; +} + +void sqlite3MemdebugBacktraceCallback(void (*xBacktrace)(int, int, void **)){ + mem.xBacktrace = xBacktrace; +} + +/* +** Set the title string for subsequent allocations. +*/ +void sqlite3MemdebugSettitle(const char *zTitle){ + unsigned int n = sqlite3Strlen30(zTitle) + 1; + sqlite3_mutex_enter(mem.mutex); + if( n>=sizeof(mem.zTitle) ) n = sizeof(mem.zTitle)-1; + memcpy(mem.zTitle, zTitle, n); + mem.zTitle[n] = 0; + mem.nTitle = ROUND8(n); + sqlite3_mutex_leave(mem.mutex); +} + +void sqlite3MemdebugSync(){ + struct MemBlockHdr *pHdr; + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + void **pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + mem.xBacktrace((int)pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); + } +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +void sqlite3MemdebugDump(const char *zFilename){ + FILE *out; + struct MemBlockHdr *pHdr; + void **pBt; + int i; + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + char *z = (char*)pHdr; + z -= pHdr->nBacktraceSlots*sizeof(void*) + pHdr->nTitle; + fprintf(out, "**** %lld bytes at %p from %s ****\n", + pHdr->iSize, &pHdr[1], pHdr->nTitle ? z : "???"); + if( pHdr->nBacktrace ){ + fflush(out); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out)); + fprintf(out, "\n"); + } + } + fprintf(out, "COUNTS:\n"); + for(i=0; i=1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + } +} + +/* +** Link the chunk at mem3.aPool[i] so that is on the list rooted +** at *pRoot. +*/ +static void memsys3LinkIntoList(u32 i, u32 *pRoot){ + assert( sqlite3_mutex_held(mem3.mutex) ); + mem3.aPool[i].u.list.next = *pRoot; + mem3.aPool[i].u.list.prev = 0; + if( *pRoot ){ + mem3.aPool[*pRoot].u.list.prev = i; + } + *pRoot = i; +} + +/* +** Link the chunk at index i into either the appropriate +** small chunk list, or into the large chunk hash table. +*/ +static void memsys3Link(u32 i){ + u32 size, hash; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3LinkIntoList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3LinkIntoList(i, &mem3.aiHash[hash]); + } +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys3Enter(void){ + if( sqlite3GlobalConfig.bMemstat==0 && mem3.mutex==0 ){ + mem3.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + sqlite3_mutex_enter(mem3.mutex); +} +static void memsys3Leave(void){ + sqlite3_mutex_leave(mem3.mutex); +} + +/* +** Called when we are unable to satisfy an allocation of nBytes. +*/ +static void memsys3OutOfMemory(int nByte){ + if( !mem3.alarmBusy ){ + mem3.alarmBusy = 1; + assert( sqlite3_mutex_held(mem3.mutex) ); + sqlite3_mutex_leave(mem3.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem3.mutex); + mem3.alarmBusy = 0; + } +} + + +/* +** Chunk i is a free chunk that has been unlinked. Adjust its +** size parameters for check-out and return a pointer to the +** user portion of the chunk. +*/ +static void *memsys3Checkout(u32 i, u32 nBlock){ + u32 x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ); + assert( mem3.aPool[i+nBlock-1].u.hdr.prevSize==nBlock ); + x = mem3.aPool[i-1].u.hdr.size4x; + mem3.aPool[i-1].u.hdr.size4x = nBlock*4 | 1 | (x&2); + mem3.aPool[i+nBlock-1].u.hdr.prevSize = nBlock; + mem3.aPool[i+nBlock-1].u.hdr.size4x |= 2; + return &mem3.aPool[i]; +} + +/* +** Carve a piece off of the end of the mem3.iKeyBlk free chunk. +** Return a pointer to the new allocation. Or, if the key chunk +** is not large enough, return 0. +*/ +static void *memsys3FromKeyBlk(u32 nBlock){ + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( mem3.szKeyBlk>=nBlock ); + if( nBlock>=mem3.szKeyBlk-1 ){ + /* Use the entire key chunk */ + void *p = memsys3Checkout(mem3.iKeyBlk, mem3.szKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; + mem3.mnKeyBlk = 0; + return p; + }else{ + /* Split the key block. Return the tail. */ + u32 newi, x; + newi = mem3.iKeyBlk + mem3.szKeyBlk - nBlock; + assert( newi > mem3.iKeyBlk+1 ); + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = nBlock; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x |= 2; + mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1; + mem3.szKeyBlk -= nBlock; + mem3.aPool[newi-1].u.hdr.prevSize = mem3.szKeyBlk; + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + if( mem3.szKeyBlk < mem3.mnKeyBlk ){ + mem3.mnKeyBlk = mem3.szKeyBlk; + } + return (void*)&mem3.aPool[newi]; + } +} + +/* +** *pRoot is the head of a list of free chunks of the same size +** or same size hash. In other words, *pRoot is an entry in either +** mem3.aiSmall[] or mem3.aiHash[]. +** +** This routine examines all entries on the given list and tries +** to coalesce each entries with adjacent free chunks. +** +** If it sees a chunk that is larger than mem3.iKeyBlk, it replaces +** the current mem3.iKeyBlk with the new larger chunk. In order for +** this mem3.iKeyBlk replacement to work, the key chunk must be +** linked into the hash tables. That is not the normal state of +** affairs, of course. The calling routine must link the key +** chunk before invoking this routine, then must unlink the (possibly +** changed) key chunk once this routine has finished. +*/ +static void memsys3Merge(u32 *pRoot){ + u32 iNext, prev, size, i, x; + + assert( sqlite3_mutex_held(mem3.mutex) ); + for(i=*pRoot; i>0; i=iNext){ + iNext = mem3.aPool[i].u.list.next; + size = mem3.aPool[i-1].u.hdr.size4x; + assert( (size&1)==0 ); + if( (size&2)==0 ){ + memsys3UnlinkFromList(i, pRoot); + assert( i > mem3.aPool[i-1].u.hdr.prevSize ); + prev = i - mem3.aPool[i-1].u.hdr.prevSize; + if( prev==iNext ){ + iNext = mem3.aPool[prev].u.list.next; + } + memsys3Unlink(prev); + size = i + size/4 - prev; + x = mem3.aPool[prev-1].u.hdr.size4x & 2; + mem3.aPool[prev-1].u.hdr.size4x = size*4 | x; + mem3.aPool[prev+size-1].u.hdr.prevSize = size; + memsys3Link(prev); + i = prev; + }else{ + size /= 4; + } + if( size>mem3.szKeyBlk ){ + mem3.iKeyBlk = i; + mem3.szKeyBlk = size; + } + } +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void *memsys3MallocUnsafe(int nByte){ + u32 i; + u32 nBlock; + u32 toFree; + + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( sizeof(Mem3Block)==8 ); + if( nByte<=12 ){ + nBlock = 2; + }else{ + nBlock = (nByte + 11)/8; + } + assert( nBlock>=2 ); + + /* STEP 1: + ** Look for an entry of the correct size in either the small + ** chunk table or in the large chunk hash table. This is + ** successful most of the time (about 9 times out of 10). + */ + if( nBlock <= MX_SMALL ){ + i = mem3.aiSmall[nBlock-2]; + if( i>0 ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[nBlock-2]); + return memsys3Checkout(i, nBlock); + } + }else{ + int hash = nBlock % N_HASH; + for(i=mem3.aiHash[hash]; i>0; i=mem3.aPool[i].u.list.next){ + if( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ){ + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + return memsys3Checkout(i, nBlock); + } + } + } + + /* STEP 2: + ** Try to satisfy the allocation by carving a piece off of the end + ** of the key chunk. This step usually works if step 1 fails. + */ + if( mem3.szKeyBlk>=nBlock ){ + return memsys3FromKeyBlk(nBlock); + } + + + /* STEP 3: + ** Loop through the entire memory pool. Coalesce adjacent free + ** chunks. Recompute the key chunk as the largest free chunk. + ** Then try again to satisfy the allocation by carving a piece off + ** of the end of the key chunk. This step happens very + ** rarely (we hope!) + */ + for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){ + memsys3OutOfMemory(toFree); + if( mem3.iKeyBlk ){ + memsys3Link(mem3.iKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; + } + for(i=0; i=nBlock ){ + return memsys3FromKeyBlk(nBlock); + } + } + } + + /* If none of the above worked, then we fail. */ + return 0; +} + +/* +** Free an outstanding memory allocation. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void memsys3FreeUnsafe(void *pOld){ + Mem3Block *p = (Mem3Block*)pOld; + int i; + u32 size, x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] ); + i = p - mem3.aPool; + assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( i+size<=mem3.nPool+1 ); + mem3.aPool[i-1].u.hdr.size4x &= ~1; + mem3.aPool[i+size-1].u.hdr.prevSize = size; + mem3.aPool[i+size-1].u.hdr.size4x &= ~2; + memsys3Link(i); + + /* Try to expand the key using the newly freed chunk */ + if( mem3.iKeyBlk ){ + while( (mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x&2)==0 ){ + size = mem3.aPool[mem3.iKeyBlk-1].u.hdr.prevSize; + mem3.iKeyBlk -= size; + mem3.szKeyBlk += size; + memsys3Unlink(mem3.iKeyBlk); + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; + } + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + while( (mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x&1)==0 ){ + memsys3Unlink(mem3.iKeyBlk+mem3.szKeyBlk); + mem3.szKeyBlk += mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x/4; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; + } + } +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys3Size(void *p){ + Mem3Block *pBlock; + assert( p!=0 ); + pBlock = (Mem3Block*)p; + assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); + return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int memsys3Roundup(int n){ + if( n<=12 ){ + return 12; + }else{ + return ((n+11)&~7) - 4; + } +} + +/* +** Allocate nBytes of memory. +*/ +static void *memsys3Malloc(int nBytes){ + sqlite3_int64 *p; + assert( nBytes>0 ); /* malloc.c filters out 0 byte requests */ + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + memsys3Leave(); + return (void*)p; +} + +/* +** Free memory. +*/ +static void memsys3Free(void *pPrior){ + assert( pPrior ); + memsys3Enter(); + memsys3FreeUnsafe(pPrior); + memsys3Leave(); +} + +/* +** Change the size of an existing memory allocation +*/ +static void *memsys3Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return sqlite3_malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pPrior); + return 0; + } + nOld = memsys3Size(pPrior); + if( nBytes<=nOld && nBytes>=nOld-128 ){ + return pPrior; + } + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + if( p ){ + if( nOld>1)!=(size&1) ){ + fprintf(out, "%p tail checkout bit is incorrect\n", &mem3.aPool[i]); + assert( 0 ); + break; + } + if( size&1 ){ + fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8); + }else{ + fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8, + i==mem3.iKeyBlk ? " **key**" : ""); + } + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + fprintf(out, "key=%d\n", mem3.iKeyBlk); + fprintf(out, "nowUsed=%d\n", mem3.nPool*8 - mem3.szKeyBlk*8); + fprintf(out, "mxUsed=%d\n", mem3.nPool*8 - mem3.mnKeyBlk*8); + sqlite3_mutex_leave(mem3.mutex); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#else + UNUSED_PARAMETER(zFilename); +#endif +} + +/* +** This routine is the only routine in this file with external +** linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ + static const sqlite3_mem_methods mempoolMethods = { + memsys3Malloc, + memsys3Free, + memsys3Realloc, + memsys3Size, + memsys3Roundup, + memsys3Init, + memsys3Shutdown, + 0 + }; + return &mempoolMethods; +} + +#endif /* SQLITE_ENABLE_MEMSYS3 */ diff --git a/third_party/sqlite3/mem5.c b/third_party/sqlite3/mem5.c new file mode 100644 index 000000000..cb754e5c1 --- /dev/null +++ b/third_party/sqlite3/mem5.c @@ -0,0 +1,577 @@ +/* +** 2007 October 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement a memory +** allocation subsystem for use by SQLite. +** +** This version of the memory allocation subsystem omits all +** use of malloc(). The application gives SQLite a block of memory +** before calling sqlite3_initialize() from which allocations +** are made and returned by the xMalloc() and xRealloc() +** implementations. Once sqlite3_initialize() has been called, +** the amount of memory available to SQLite is fixed and cannot +** be changed. +** +** This version of the memory allocation subsystem is included +** in the build only if SQLITE_ENABLE_MEMSYS5 is defined. +** +** This memory allocator uses the following algorithm: +** +** 1. All memory allocation sizes are rounded up to a power of 2. +** +** 2. If two adjacent free blocks are the halves of a larger block, +** then the two blocks are coalesced into the single larger block. +** +** 3. New memory is allocated from the first available free block. +** +** This algorithm is described in: J. M. Robson. "Bounds for Some Functions +** Concerning Dynamic Storage Allocation". Journal of the Association for +** Computing Machinery, Volume 21, Number 8, July 1974, pages 491-499. +** +** Let n be the size of the largest allocation divided by the minimum +** allocation size (after rounding all sizes up to a power of 2.) Let M +** be the maximum amount of memory ever outstanding at one time. Let +** N be the total amount of memory available for allocation. Robson +** proved that this memory allocator will never breakdown due to +** fragmentation as long as the following constraint holds: +** +** N >= M*(1 + log2(n)/2) - n + 1 +** +** The sqlite3_status() logic tracks the maximum values of n and M so +** that an application can, at any time, verify this constraint. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** This version of the memory allocator is used only when +** SQLITE_ENABLE_MEMSYS5 is defined. +*/ +#ifdef SQLITE_ENABLE_MEMSYS5 + +/* +** A minimum allocation is an instance of the following structure. +** Larger allocations are an array of these structures where the +** size of the array is a power of 2. +** +** The size of this object must be a power of two. That fact is +** verified in memsys5Init(). +*/ +typedef struct Mem5Link Mem5Link; +struct Mem5Link { + int next; /* Index of next free chunk */ + int prev; /* Index of previous free chunk */ +}; + +/* +** Maximum size of any allocation is ((1<=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + next = MEM5LINK(i)->next; + prev = MEM5LINK(i)->prev; + if( prev<0 ){ + mem5.aiFreelist[iLogsize] = next; + }else{ + MEM5LINK(prev)->next = next; + } + if( next>=0 ){ + MEM5LINK(next)->prev = prev; + } +} + +/* +** Link the chunk at mem5.aPool[i] so that is on the iLogsize +** free list. +*/ +static void memsys5Link(int i, int iLogsize){ + int x; + assert( sqlite3_mutex_held(mem5.mutex) ); + assert( i>=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + x = MEM5LINK(i)->next = mem5.aiFreelist[iLogsize]; + MEM5LINK(i)->prev = -1; + if( x>=0 ){ + assert( xprev = i; + } + mem5.aiFreelist[iLogsize] = i; +} + +/* +** Obtain or release the mutex needed to access global data structures. +*/ +static void memsys5Enter(void){ + sqlite3_mutex_enter(mem5.mutex); +} +static void memsys5Leave(void){ + sqlite3_mutex_leave(mem5.mutex); +} + +/* +** Return the size of an outstanding allocation, in bytes. +** This only works for chunks that are currently checked out. +*/ +static int memsys5Size(void *p){ + int iSize, i; + assert( p!=0 ); + i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); + assert( i>=0 && i0 ); + + /* No more than 1GiB per allocation */ + if( nByte > 0x40000000 ) return 0; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* Keep track of the maximum allocation request. Even unfulfilled + ** requests are counted */ + if( (u32)nByte>mem5.maxRequest ){ + mem5.maxRequest = nByte; + } +#endif + + + /* Round nByte up to the next valid power of two */ + for(iFullSz=mem5.szAtom,iLogsize=0; iFullSzLOGMAX ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); + return 0; + } + i = mem5.aiFreelist[iBin]; + memsys5Unlink(i, iBin); + while( iBin>iLogsize ){ + int newSize; + + iBin--; + newSize = 1 << iBin; + mem5.aCtrl[i+newSize] = CTRL_FREE | iBin; + memsys5Link(i+newSize, iBin); + } + mem5.aCtrl[i] = iLogsize; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* Update allocator performance statistics. */ + mem5.nAlloc++; + mem5.totalAlloc += iFullSz; + mem5.totalExcess += iFullSz - nByte; + mem5.currentCount++; + mem5.currentOut += iFullSz; + if( mem5.maxCount=0 && iBlock0 ); + assert( mem5.currentOut>=(size*mem5.szAtom) ); + mem5.currentCount--; + mem5.currentOut -= size*mem5.szAtom; + assert( mem5.currentOut>0 || mem5.currentCount==0 ); + assert( mem5.currentCount>0 || mem5.currentOut==0 ); +#endif + + mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; + while( ALWAYS(iLogsize>iLogsize) & 1 ){ + iBuddy = iBlock - size; + assert( iBuddy>=0 ); + }else{ + iBuddy = iBlock + size; + if( iBuddy>=mem5.nBlock ) break; + } + if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; + memsys5Unlink(iBuddy, iLogsize); + iLogsize++; + if( iBuddy0 ){ + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + memsys5Leave(); + } + return (void*)p; +} + +/* +** Free memory. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +*/ +static void memsys5Free(void *pPrior){ + assert( pPrior!=0 ); + memsys5Enter(); + memsys5FreeUnsafe(pPrior); + memsys5Leave(); +} + +/* +** Change the size of an existing memory allocation. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +** +** nBytes is always a value obtained from a prior call to +** memsys5Round(). Hence nBytes is always a non-negative power +** of two. If nBytes==0 that means that an oversize allocation +** (an allocation larger than 0x40000000) was requested and this +** routine should return 0 without freeing pPrior. +*/ +static void *memsys5Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + assert( pPrior!=0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ + assert( nBytes>=0 ); + if( nBytes==0 ){ + return 0; + } + nOld = memsys5Size(pPrior); + if( nBytes<=nOld ){ + return pPrior; + } + p = memsys5Malloc(nBytes); + if( p ){ + memcpy(p, pPrior, nOld); + memsys5Free(pPrior); + } + return p; +} + +/* +** Round up a request size to the next valid allocation size. If +** the allocation is too large to be handled by this allocation system, +** return 0. +** +** All allocations must be a power of two and must be expressed by a +** 32-bit signed integer. Hence the largest allocation is 0x40000000 +** or 1073741824 bytes. +*/ +static int memsys5Roundup(int n){ + int iFullSz; + if( n > 0x40000000 ) return 0; + for(iFullSz=mem5.szAtom; iFullSz 0 +** memsys5Log(2) -> 1 +** memsys5Log(4) -> 2 +** memsys5Log(5) -> 3 +** memsys5Log(8) -> 3 +** memsys5Log(9) -> 4 +*/ +static int memsys5Log(int iValue){ + int iLog; + for(iLog=0; (iLog<(int)((sizeof(int)*8)-1)) && (1<mem5.szAtom ){ + mem5.szAtom = mem5.szAtom << 1; + } + + mem5.nBlock = (nByte / (mem5.szAtom+sizeof(u8))); + mem5.zPool = zByte; + mem5.aCtrl = (u8 *)&mem5.zPool[mem5.nBlock*mem5.szAtom]; + + for(ii=0; ii<=LOGMAX; ii++){ + mem5.aiFreelist[ii] = -1; + } + + iOffset = 0; + for(ii=LOGMAX; ii>=0; ii--){ + int nAlloc = (1<mem5.nBlock); + } + + /* If a mutex is required for normal operation, allocate one */ + if( sqlite3GlobalConfig.bMemstat==0 ){ + mem5.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void memsys5Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem5.mutex = 0; + return; +} + +#ifdef SQLITE_TEST +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +void sqlite3Memsys5Dump(const char *zFilename){ + FILE *out; + int i, j, n; + int nMinLog; + + if( zFilename==0 || zFilename[0]==0 ){ + out = stdout; + }else{ + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + } + memsys5Enter(); + nMinLog = memsys5Log(mem5.szAtom); + for(i=0; i<=LOGMAX && i+nMinLog<32; i++){ + for(n=0, j=mem5.aiFreelist[i]; j>=0; j = MEM5LINK(j)->next, n++){} + fprintf(out, "freelist items of size %d: %d\n", mem5.szAtom << i, n); + } + fprintf(out, "mem5.nAlloc = %llu\n", mem5.nAlloc); + fprintf(out, "mem5.totalAlloc = %llu\n", mem5.totalAlloc); + fprintf(out, "mem5.totalExcess = %llu\n", mem5.totalExcess); + fprintf(out, "mem5.currentOut = %u\n", mem5.currentOut); + fprintf(out, "mem5.currentCount = %u\n", mem5.currentCount); + fprintf(out, "mem5.maxOut = %u\n", mem5.maxOut); + fprintf(out, "mem5.maxCount = %u\n", mem5.maxCount); + fprintf(out, "mem5.maxRequest = %u\n", mem5.maxRequest); + memsys5Leave(); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +} +#endif + +/* +** This routine is the only routine in this file with external +** linkage. It returns a pointer to a static sqlite3_mem_methods +** struct populated with the memsys5 methods. +*/ +const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ + static const sqlite3_mem_methods memsys5Methods = { + memsys5Malloc, + memsys5Free, + memsys5Realloc, + memsys5Size, + memsys5Roundup, + memsys5Init, + memsys5Shutdown, + 0 + }; + return &memsys5Methods; +} + +#endif /* SQLITE_ENABLE_MEMSYS5 */ diff --git a/third_party/sqlite3/memdb.c b/third_party/sqlite3/memdb.c new file mode 100644 index 000000000..0b3939acb --- /dev/null +++ b/third_party/sqlite3/memdb.c @@ -0,0 +1,636 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an in-memory VFS. A database is held as a contiguous +** block of memory. +** +** This file also implements interface sqlite3_serialize() and +** sqlite3_deserialize(). +*/ +#include "third_party/sqlite3/sqliteInt.inc" +#ifdef SQLITE_ENABLE_DESERIALIZE +/* clang-format off */ + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ + unsigned char *aData; /* content of the file */ + int nMmap; /* Number of memory mapped pages */ + unsigned mFlags; /* Flags */ + int eLock; /* Most recent lock against this file */ +}; + +/* +** Methods for MemFile +*/ +static int memdbClose(sqlite3_file*); +static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); +static int memdbSync(sqlite3_file*, int flags); +static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memdbLock(sqlite3_file*, int); +/* static int memdbCheckReservedLock(sqlite3_file*, int *pResOut);// not used */ +static int memdbFileControl(sqlite3_file*, int op, void *pArg); +/* static int memdbSectorSize(sqlite3_file*); // not used */ +static int memdbDeviceCharacteristics(sqlite3_file*); +static int memdbFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memdbUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memdbOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +/* static int memdbDelete(sqlite3_vfs*, const char *zName, int syncDir); */ +static int memdbAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memdbFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memdbDlOpen(sqlite3_vfs*, const char *zFilename); +static void memdbDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memdbDlClose(sqlite3_vfs*, void*); +static int memdbRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memdbSleep(sqlite3_vfs*, int microseconds); +/* static int memdbCurrentTime(sqlite3_vfs*, double*); */ +static int memdbGetLastError(sqlite3_vfs*, int, char *); +static int memdbCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs memdb_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memdb", /* zName */ + 0, /* pAppData (set when registered) */ + memdbOpen, /* xOpen */ + 0, /* memdbDelete, */ /* xDelete */ + memdbAccess, /* xAccess */ + memdbFullPathname, /* xFullPathname */ + memdbDlOpen, /* xDlOpen */ + memdbDlError, /* xDlError */ + memdbDlSym, /* xDlSym */ + memdbDlClose, /* xDlClose */ + memdbRandomness, /* xRandomness */ + memdbSleep, /* xSleep */ + 0, /* memdbCurrentTime, */ /* xCurrentTime */ + memdbGetLastError, /* xGetLastError */ + memdbCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods memdb_io_methods = { + 3, /* iVersion */ + memdbClose, /* xClose */ + memdbRead, /* xRead */ + memdbWrite, /* xWrite */ + memdbTruncate, /* xTruncate */ + memdbSync, /* xSync */ + memdbFileSize, /* xFileSize */ + memdbLock, /* xLock */ + memdbLock, /* xUnlock - same as xLock in this case */ + 0, /* memdbCheckReservedLock, */ /* xCheckReservedLock */ + memdbFileControl, /* xFileControl */ + 0, /* memdbSectorSize,*/ /* xSectorSize */ + memdbDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + memdbFetch, /* xFetch */ + memdbUnfetch /* xUnfetch */ +}; + + + +/* +** Close an memdb-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. Unless the SQLITE_DESERIALIZE_FREEONCLOSE flag is set, +** in which case we own the pData pointer and need to free it. +*/ +static int memdbClose(sqlite3_file *pFile){ + MemFile *p = (MemFile *)pFile; + if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){ + sqlite3_free(p->aData); + } + return SQLITE_OK; +} + +/* +** Read data from an memdb-file. +*/ +static int memdbRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + memset(zBuf, 0, iAmt); + if( iOfstsz ) memcpy(zBuf, p->aData+iOfst, p->sz - iOfst); + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Try to enlarge the memory allocation to hold at least sz bytes +*/ +static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){ + unsigned char *pNew; + if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){ + return SQLITE_FULL; + } + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; + pNew = sqlite3Realloc(p->aData, newSz); + if( pNew==0 ) return SQLITE_NOMEM; + p->aData = pNew; + p->szAlloc = newSz; + return SQLITE_OK; +} + +/* +** Write data to an memdb-file. +*/ +static int memdbWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ) return SQLITE_READONLY; + if( iOfst+iAmt>p->sz ){ + int rc; + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK + ){ + return rc; + } + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + return SQLITE_OK; +} + +/* +** Truncate an memdb-file. +** +** In rollback mode (which is always the case for memdb, as it does not +** support WAL mode) the truncate() method is only used to reduce +** the size of a file, never to increase the size. +*/ +static int memdbTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemFile *p = (MemFile *)pFile; + if( NEVER(size>p->sz) ) return SQLITE_FULL; + p->sz = size; + return SQLITE_OK; +} + +/* +** Sync an memdb-file. +*/ +static int memdbSync(sqlite3_file *pFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an memdb-file. +*/ +static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an memdb-file. +*/ +static int memdbLock(sqlite3_file *pFile, int eLock){ + MemFile *p = (MemFile *)pFile; + if( eLock>SQLITE_LOCK_SHARED + && (p->mFlags & SQLITE_DESERIALIZE_READONLY)!=0 + ){ + return SQLITE_READONLY; + } + p->eLock = eLock; + return SQLITE_OK; +} + +#if 0 /* Never used because memdbAccess() always returns false */ +/* +** Check if another file-handle holds a RESERVED lock on an memdb-file. +*/ +static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} +#endif + +/* +** File control method. For custom operations on an memdb-file. +*/ +static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimitsz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } + return rc; +} + +#if 0 /* Not used because of SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +/* +** Return the sector-size in bytes for an memdb-file. +*/ +static int memdbSectorSize(sqlite3_file *pFile){ + return 1024; +} +#endif + +/* +** Return the device characteristic flags supported by an memdb-file. +*/ +static int memdbDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Fetch a page of a memory-mapped file */ +static int memdbFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + *pp = 0; + }else{ + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + } + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + MemFile *p = (MemFile *)pFile; + p->nMmap--; + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memdbOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags); + } + memset(p, 0, sizeof(*p)); + p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE; + assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */ + *pOutFlags = flags | SQLITE_OPEN_MEMORY; + pFile->pMethods = &memdb_io_methods; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + return SQLITE_OK; +} + +#if 0 /* Only used to delete rollback journals, super-journals, and WAL + ** files, none of which exist in memdb. So this routine is never used */ +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memdbDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} +#endif + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +** +** With memdb, no files ever exist on disk. So always return false. +*/ +static int memdbAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memdbFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memdbDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memdbDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memdbSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +#if 0 /* Never used. Modern cores only call xCurrentTimeInt64() */ +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memdbCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +#endif + +static int memdbGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memdbCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +/* +** Translate a database connection pointer and schema name into a +** MemFile pointer. +*/ +static MemFile *memdbFromDbSchema(sqlite3 *db, const char *zSchema){ + MemFile *p = 0; + int rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return 0; + if( p->base.pMethods!=&memdb_io_methods ) return 0; + return p; +} + +/* +** Return the serialization of a database +*/ +unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which database within the connection */ + sqlite3_int64 *piSize, /* Write size here, if not NULL */ + unsigned int mFlags /* Maybe SQLITE_SERIALIZE_NOCOPY */ +){ + MemFile *p; + int iDb; + Btree *pBt; + sqlite3_int64 sz; + int szPage = 0; + sqlite3_stmt *pStmt = 0; + unsigned char *pOut; + char *zSql; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + p = memdbFromDbSchema(db, zSchema); + iDb = sqlite3FindDbName(db, zSchema); + if( piSize ) *piSize = -1; + if( iDb<0 ) return 0; + if( p ){ + if( piSize ) *piSize = p->sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = p->aData; + }else{ + pOut = sqlite3_malloc64( p->sz ); + if( pOut ) memcpy(pOut, p->aData, p->sz); + } + return pOut; + } + pBt = db->aDb[iDb].pBt; + if( pBt==0 ) return 0; + szPage = sqlite3BtreeGetPageSize(pBt); + zSql = sqlite3_mprintf("PRAGMA \"%w\".page_count", zSchema); + rc = zSql ? sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) : SQLITE_NOMEM; + sqlite3_free(zSql); + if( rc ) return 0; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + pOut = 0; + }else{ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( piSize ) *piSize = sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = 0; + }else{ + pOut = sqlite3_malloc64( sz ); + if( pOut ){ + int nPage = sqlite3_column_int(pStmt, 0); + Pager *pPager = sqlite3BtreePager(pBt); + int pgno; + for(pgno=1; pgno<=nPage; pgno++){ + DbPage *pPage = 0; + unsigned char *pTo = pOut + szPage*(sqlite3_int64)(pgno-1); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pPage, 0); + if( rc==SQLITE_OK ){ + memcpy(pTo, sqlite3PagerGetData(pPage), szPage); + }else{ + memset(pTo, 0, szPage); + } + sqlite3PagerUnref(pPage); + } + } + } + } + sqlite3_finalize(pStmt); + return pOut; +} + +/* Convert zSchema to a MemDB and initialize its content. +*/ +int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +){ + MemFile *p; + char *zSql; + sqlite3_stmt *pStmt = 0; + int rc; + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( szDb<0 ) return SQLITE_MISUSE_BKPT; + if( szBuf<0 ) return SQLITE_MISUSE_BKPT; +#endif + + sqlite3_mutex_enter(db->mutex); + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + iDb = sqlite3FindDbName(db, zSchema); + if( iDb<0 ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + if( rc ) goto end_deserialize; + db->init.iDb = (u8)iDb; + db->init.reopenMemdb = 1; + rc = sqlite3_step(pStmt); + db->init.reopenMemdb = 0; + if( rc!=SQLITE_DONE ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + p = memdbFromDbSchema(db, zSchema); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + p->aData = pData; + pData = 0; + p->sz = szDb; + p->szAlloc = szBuf; + p->szMax = szBuf; + if( p->szMaxszMax = sqlite3GlobalConfig.mxMemdbSize; + } + p->mFlags = mFlags; + rc = SQLITE_OK; + } + +end_deserialize: + sqlite3_finalize(pStmt); + if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){ + sqlite3_free(pData); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3MemdbInit(void){ + sqlite3_vfs *pLower = sqlite3_vfs_find(0); + int sz = pLower->szOsFile; + memdb_vfs.pAppData = pLower; + /* The following conditional can only be true when compiled for + ** Windows x86 and SQLITE_MAX_MMAP_SIZE=0. We always leave + ** it in, to be safe, but it is marked as NO_TEST since there + ** is no way to reach it under most builds. */ + if( szp->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; + } + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += p->nChunkSize; + } + }else{ + pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); + } + + iChunkOffset = (int)(iOfst%p->nChunkSize); + do { + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; + p->readpoint.pChunk = pChunk; + + return SQLITE_OK; +} + +/* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(FileChunk *pFirst){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int memjrnlCreateFile(MemJournal *p){ + int rc; + sqlite3_file *pReal = (sqlite3_file*)p; + MemJournal copy = *p; + + memset(p, 0, sizeof(MemJournal)); + rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = copy.nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ + if( iOff + nChunk > copy.endpoint.iOffset ){ + nChunk = copy.endpoint.iOffset - iOff; + } + rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); + if( rc ) break; + iOff += nChunk; + } + if( rc==SQLITE_OK ){ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(copy.pFirst); + } + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while creating or writing to the file, restore + ** the original before returning. This way, SQLite uses the in-memory + ** journal data to roll back changes made to the internal page-cache + ** before this function was called. */ + sqlite3OsClose(pReal); + *p = copy; + } + return rc; +} + + +/* +** Write data to the file. +*/ +static int memjrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + MemJournal *p = (MemJournal *)pJfd; + int nWrite = iAmt; + u8 *zWrite = (u8 *)zBuf; + + /* If the file should be created now, create it and write the new data + ** into the file on disk. */ + if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ + int rc = memjrnlCreateFile(p); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); + } + return rc; + } + + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); + }else +#else + assert( iOfst>0 || p->pFirst==0 ); +#endif + { + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + } + } + + return SQLITE_OK; +} + +/* +** Truncate the in-memory file. +*/ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + MemJournal *p = (MemJournal *)pJfd; + FileChunk *pIter = 0; + + if( size==0 ){ + memjrnlFreeChunks(p->pFirst); + p->pFirst = 0; + }else{ + i64 iOff = p->nChunkSize; + for(pIter=p->pFirst; ALWAYS(pIter) && iOff<=size; pIter=pIter->pNext){ + iOff += p->nChunkSize; + } + if( ALWAYS(pIter) ){ + memjrnlFreeChunks(pIter->pNext); + pIter->pNext = 0; + } + } + + p->endpoint.pChunk = pIter; + p->endpoint.iOffset = size; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int memjrnlClose(sqlite3_file *pJfd){ + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p->pFirst); + return SQLITE_OK; +} + +/* +** Sync the file. +** +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. +*/ +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + UNUSED_PARAMETER2(pJfd, flags); + return SQLITE_OK; +} + +/* +** Query the size of the file in bytes. +*/ +static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + MemJournal *p = (MemJournal *)pJfd; + *pSize = (sqlite_int64) p->endpoint.iOffset; + return SQLITE_OK; +} + +/* +** Table of methods for MemJournal sqlite3_file object. +*/ +static const struct sqlite3_io_methods MemJournalMethods = { + 1, /* iVersion */ + memjrnlClose, /* xClose */ + memjrnlRead, /* xRead */ + memjrnlWrite, /* xWrite */ + memjrnlTruncate, /* xTruncate */ + memjrnlSync, /* xSync */ + memjrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and +** accessed using the underlying VFS. If nSpill is less than zero, then +** all content is always stored in main-memory. Finally, if nSpill is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nSpill bytes in size, +** or when sqlite3JournalCreate() is called. +*/ +int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nSpill /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nSpill was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nSpill==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nSpill>0 ){ + p->nChunkSize = nSpill; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + pJfd->pMethods = (const sqlite3_io_methods*)&MemJournalMethods; + p->nSpill = nSpill; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** Open an in-memory journal file. +*/ +void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying +** file has not yet been created, create it now. +*/ +int sqlite3JournalCreate(sqlite3_file *pJfd){ + int rc = SQLITE_OK; + MemJournal *p = (MemJournal*)pJfd; + if( pJfd->pMethods==&MemJournalMethods && ( +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + p->nSpill>0 +#else + /* While this appears to not be possible without ATOMIC_WRITE, the + ** paths are complex, so it seems prudent to leave the test in as + ** a NEVER(), in case our analysis is subtly flawed. */ + NEVER(p->nSpill>0) +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (p->flags & SQLITE_OPEN_MAIN_JOURNAL) +#endif + )){ + rc = memjrnlCreateFile(p); + } + return rc; +} +#endif + +/* +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, +** or false otherwise. +*/ +int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods; +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); +} diff --git a/third_party/sqlite3/msvc.inc b/third_party/sqlite3/msvc.inc new file mode 100644 index 000000000..72232846e --- /dev/null +++ b/third_party/sqlite3/msvc.inc @@ -0,0 +1,42 @@ +/* +** 2015 January 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to MSVC. +*/ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H +/* clang-format off */ + +#if defined(_MSC_VER) +#pragma warning(disable : 4054) +#pragma warning(disable : 4055) +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4130) +#pragma warning(disable : 4152) +#pragma warning(disable : 4189) +#pragma warning(disable : 4206) +#pragma warning(disable : 4210) +#pragma warning(disable : 4232) +#pragma warning(disable : 4244) +#pragma warning(disable : 4305) +#pragma warning(disable : 4306) +#pragma warning(disable : 4702) +#pragma warning(disable : 4706) +#endif /* defined(_MSC_VER) */ + +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + +#endif /* SQLITE_MSVC_H */ diff --git a/third_party/sqlite3/mutex.c b/third_party/sqlite3/mutex.c new file mode 100644 index 000000000..256287739 --- /dev/null +++ b/third_party/sqlite3/mutex.c @@ -0,0 +1,362 @@ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This file contains code that is common across all mutex implementations. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) +/* +** For debugging purposes, record when the mutex subsystem is initialized +** and uninitialized so that we can assert() if there is an attempt to +** allocate a mutex while the system is uninitialized. +*/ +static SQLITE_WSD int mutexIsInit = 0; +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ + + +#ifndef SQLITE_MUTEX_OMIT + +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +/* +** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** the implementation of a wrapper around the system default mutex +** implementation (sqlite3DefaultMutex()). +** +** Most calls are passed directly through to the underlying default +** mutex implementation. Except, if a mutex is configured by calling +** sqlite3MutexWarnOnContention() on it, then if contention is ever +** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** +** This type of mutex is used as the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +*/ + +/* +** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** is defined. Variable CheckMutex.mutex is a pointer to the real mutex +** allocated by the system mutex implementation. Variable iType is usually set +** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST +** or one of the static mutex identifiers. Or, if this is a recursive mutex +** that has been configured using sqlite3MutexWarnOnContention(), it is +** set to SQLITE_MUTEX_WARNONCONTENTION. +*/ +typedef struct CheckMutex CheckMutex; +struct CheckMutex { + int iType; + sqlite3_mutex *mutex; +}; + +#define SQLITE_MUTEX_WARNONCONTENTION (-1) + +/* +** Pointer to real mutex methods object used by the CheckMutex +** implementation. Set by checkMutexInit(). +*/ +static SQLITE_WSD const sqlite3_mutex_methods *pGlobalMutexMethods; + +#ifdef SQLITE_DEBUG +static int checkMutexHeld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexHeld(((CheckMutex*)p)->mutex); +} +static int checkMutexNotheld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexNotheld(((CheckMutex*)p)->mutex); +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int checkMutexInit(void){ + pGlobalMutexMethods = sqlite3DefaultMutex(); + return SQLITE_OK; +} +static int checkMutexEnd(void){ + pGlobalMutexMethods = 0; + return SQLITE_OK; +} + +/* +** Allocate a mutex. +*/ +static sqlite3_mutex *checkMutexAlloc(int iType){ + static CheckMutex staticMutexes[] = { + {2, 0}, {3, 0}, {4, 0}, {5, 0}, + {6, 0}, {7, 0}, {8, 0}, {9, 0}, + {10, 0}, {11, 0}, {12, 0}, {13, 0} + }; + CheckMutex *p = 0; + + assert( SQLITE_MUTEX_RECURSIVE==1 && SQLITE_MUTEX_FAST==0 ); + if( iType<2 ){ + p = sqlite3MallocZero(sizeof(CheckMutex)); + if( p==0 ) return 0; + p->iType = iType; + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + } + + if( p->mutex==0 ){ + p->mutex = pGlobalMutexMethods->xMutexAlloc(iType); + if( p->mutex==0 ){ + if( iType<2 ){ + sqlite3_free(p); + } + p = 0; + } + } + + return (sqlite3_mutex*)p; +} + +/* +** Free a mutex. +*/ +static void checkMutexFree(sqlite3_mutex *p){ + assert( SQLITE_MUTEX_RECURSIVE<2 ); + assert( SQLITE_MUTEX_FAST<2 ); + assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); + +#if SQLITE_ENABLE_API_ARMOR + if( ((CheckMutex*)p)->iType<2 ) +#endif + { + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexFree(pCheck->mutex); + sqlite3_free(pCheck); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** Enter the mutex. +*/ +static void checkMutexEnter(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + if( pCheck->iType==SQLITE_MUTEX_WARNONCONTENTION ){ + if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){ + return; + } + sqlite3_log(SQLITE_MISUSE, + "illegal multi-threaded access to database connection" + ); + } + pGlobalMutexMethods->xMutexEnter(pCheck->mutex); +} + +/* +** Enter the mutex (do not block). +*/ +static int checkMutexTry(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + return pGlobalMutexMethods->xMutexTry(pCheck->mutex); +} + +/* +** Leave the mutex. +*/ +static void checkMutexLeave(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexLeave(pCheck->mutex); +} + +sqlite3_mutex_methods const *multiThreadedCheckMutex(void){ + static const sqlite3_mutex_methods sMutex = { + checkMutexInit, + checkMutexEnd, + checkMutexAlloc, + checkMutexFree, + checkMutexEnter, + checkMutexTry, + checkMutexLeave, +#ifdef SQLITE_DEBUG + checkMutexHeld, + checkMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +/* +** Mark the SQLITE_MUTEX_RECURSIVE mutex passed as the only argument as +** one on which there should be no contention. +*/ +void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ + if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){ + CheckMutex *pCheck = (CheckMutex*)p; + assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE ); + pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; + } +} +#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ + +/* +** Initialize the mutex system. +*/ +int sqlite3MutexInit(void){ + int rc = SQLITE_OK; + if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ + /* If the xMutexAlloc method has not been set, then the user did not + ** install a mutex implementation via sqlite3_config() prior to + ** sqlite3_initialize() being called. This block copies pointers to + ** the default implementation into the sqlite3GlobalConfig structure. + */ + sqlite3_mutex_methods const *pFrom; + sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; + + if( sqlite3GlobalConfig.bCoreMutex ){ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + pFrom = multiThreadedCheckMutex(); +#else + pFrom = sqlite3DefaultMutex(); +#endif + }else{ + pFrom = sqlite3NoopMutex(); + } + pTo->xMutexInit = pFrom->xMutexInit; + pTo->xMutexEnd = pFrom->xMutexEnd; + pTo->xMutexFree = pFrom->xMutexFree; + pTo->xMutexEnter = pFrom->xMutexEnter; + pTo->xMutexTry = pFrom->xMutexTry; + pTo->xMutexLeave = pFrom->xMutexLeave; + pTo->xMutexHeld = pFrom->xMutexHeld; + pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); + pTo->xMutexAlloc = pFrom->xMutexAlloc; + } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); + rc = sqlite3GlobalConfig.mutex.xMutexInit(); + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 1; +#endif + + sqlite3MemoryBarrier(); + return rc; +} + +/* +** Shutdown the mutex system. This call frees resources allocated by +** sqlite3MutexInit(). +*/ +int sqlite3MutexEnd(void){ + int rc = SQLITE_OK; + if( sqlite3GlobalConfig.mutex.xMutexEnd ){ + rc = sqlite3GlobalConfig.mutex.xMutexEnd(); + } + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 0; +#endif + + return rc; +} + +/* +** Retrieve a pointer to a static mutex or allocate a new dynamic one. +*/ +sqlite3_mutex *sqlite3_mutex_alloc(int id){ +#ifndef SQLITE_OMIT_AUTOINIT + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; +#endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +sqlite3_mutex *sqlite3MutexAlloc(int id){ + if( !sqlite3GlobalConfig.bCoreMutex ){ + return 0; + } + assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +/* +** Free a dynamic mutex. +*/ +void sqlite3_mutex_free(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); + sqlite3GlobalConfig.mutex.xMutexFree(p); + } +} + +/* +** Obtain the mutex p. If some other thread already has the mutex, block +** until it can be obtained. +*/ +void sqlite3_mutex_enter(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); + sqlite3GlobalConfig.mutex.xMutexEnter(p); + } +} + +/* +** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another +** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. +*/ +int sqlite3_mutex_try(sqlite3_mutex *p){ + int rc = SQLITE_OK; + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); + return sqlite3GlobalConfig.mutex.xMutexTry(p); + } + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was previously +** entered by the same thread. The behavior is undefined if the mutex +** is not currently entered. If a NULL pointer is passed as an argument +** this function is a no-op. +*/ +void sqlite3_mutex_leave(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); + sqlite3GlobalConfig.mutex.xMutexLeave(p); + } +} + +#ifndef NDEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +int sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); + return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); +} +int sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); + return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); +} +#endif + +#endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/third_party/sqlite3/mutex.inc b/third_party/sqlite3/mutex.inc new file mode 100644 index 000000000..22ff0fea7 --- /dev/null +++ b/third_party/sqlite3/mutex.inc @@ -0,0 +1,72 @@ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the common header for all mutex implementations. +** The sqliteInt.h header #includes this file so that it is available +** to all source files. We break it out in an effort to keep the code +** better organized. +** +** NOTE: source files should *not* #include this header file directly. +** Source files should #include the sqliteInt.h file and let that file +** include this one indirectly. +*/ +/* clang-format off */ + + +/* +** Figure out what version of the code to use. The choices are +** +** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The +** mutexes implementation cannot be overridden +** at start-time. +** +** SQLITE_MUTEX_NOOP For single-threaded applications. No +** mutual exclusion is provided. But this +** implementation can be overridden at +** start-time. +** +** SQLITE_MUTEX_PTHREADS For multi-threaded applications on Unix. +** +** SQLITE_MUTEX_W32 For multi-threaded applications on Win32. +*/ +#if !SQLITE_THREADSAFE +# define SQLITE_MUTEX_OMIT +#endif +#if SQLITE_THREADSAFE && !defined(SQLITE_MUTEX_NOOP) +# if SQLITE_OS_UNIX +# define SQLITE_MUTEX_PTHREADS +# elif SQLITE_OS_WIN +# define SQLITE_MUTEX_W32 +# else +# define SQLITE_MUTEX_NOOP +# endif +#endif + +#ifdef SQLITE_MUTEX_OMIT +/* +** If this is a no-op implementation, implement everything as macros. +*/ +#define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) +#define sqlite3_mutex_free(X) +#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_try(X) SQLITE_OK +#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) +#define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) +#define sqlite3MutexInit() SQLITE_OK +#define sqlite3MutexEnd() +#define MUTEX_LOGIC(X) +#else +#define MUTEX_LOGIC(X) X +int sqlite3_mutex_held(sqlite3_mutex*); +#endif /* defined(SQLITE_MUTEX_OMIT) */ diff --git a/third_party/sqlite3/mutex_noop.c b/third_party/sqlite3/mutex_noop.c new file mode 100644 index 000000000..6802b7888 --- /dev/null +++ b/third_party/sqlite3/mutex_noop.c @@ -0,0 +1,216 @@ +/* +** 2008 October 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This implementation in this file does not provide any mutual +** exclusion and is thus suitable for use only in applications +** that use SQLite in a single thread. The routines defined +** here are place-holders. Applications can substitute working +** mutex routines at start-time using the +** +** sqlite3_config(SQLITE_CONFIG_MUTEX,...) +** +** interface. +** +** If compiled with SQLITE_DEBUG, then additional logic is inserted +** that does error checking on mutexes to make sure they are being +** called correctly. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#ifndef SQLITE_MUTEX_OMIT + +#ifndef SQLITE_DEBUG +/* +** Stub routines for all mutex methods. +** +** This routines provide no mutual exclusion or error checking. +*/ +static int noopMutexInit(void){ return SQLITE_OK; } +static int noopMutexEnd(void){ return SQLITE_OK; } +static sqlite3_mutex *noopMutexAlloc(int id){ + UNUSED_PARAMETER(id); + return (sqlite3_mutex*)8; +} +static void noopMutexFree(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static void noopMutexEnter(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static int noopMutexTry(sqlite3_mutex *p){ + UNUSED_PARAMETER(p); + return SQLITE_OK; +} +static void noopMutexLeave(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } + +sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + noopMutexInit, + noopMutexEnd, + noopMutexAlloc, + noopMutexFree, + noopMutexEnter, + noopMutexTry, + noopMutexLeave, + + 0, + 0, + }; + + return &sMutex; +} +#endif /* !SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** In this implementation, error checking is provided for testing +** and debugging purposes. The mutexes still do not provide any +** mutual exclusion. +*/ + +/* +** The mutex object +*/ +typedef struct sqlite3_debug_mutex { + int id; /* The mutex type */ + int cnt; /* Number of entries without a matching leave */ +} sqlite3_debug_mutex; + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int debugMutexHeld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt>0; +} +static int debugMutexNotheld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt==0; +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int debugMutexInit(void){ return SQLITE_OK; } +static int debugMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. +*/ +static sqlite3_mutex *debugMutexAlloc(int id){ + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; + sqlite3_debug_mutex *pNew = 0; + switch( id ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + pNew = sqlite3Malloc(sizeof(*pNew)); + if( pNew ){ + pNew->id = id; + pNew->cnt = 0; + } + break; + } + default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( id-2<0 || id-2>=ArraySize(aStatic) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pNew = &aStatic[id-2]; + pNew->id = id; + break; + } + } + return (sqlite3_mutex*)pNew; +} + +/* +** This routine deallocates a previously allocated mutex. +*/ +static void debugMutexFree(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->cnt==0 ); + if( p->id==SQLITE_MUTEX_RECURSIVE || p->id==SQLITE_MUTEX_FAST ){ + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; +#endif + } +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void debugMutexEnter(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; +} +static int debugMutexTry(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; + return SQLITE_OK; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void debugMutexLeave(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( debugMutexHeld(pX) ); + p->cnt--; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); +} + +sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + debugMutexInit, + debugMutexEnd, + debugMutexAlloc, + debugMutexFree, + debugMutexEnter, + debugMutexTry, + debugMutexLeave, + + debugMutexHeld, + debugMutexNotheld + }; + + return &sMutex; +} +#endif /* SQLITE_DEBUG */ + +/* +** If compiled with SQLITE_MUTEX_NOOP, then the no-op mutex implementation +** is used regardless of the run-time threadsafety setting. +*/ +#ifdef SQLITE_MUTEX_NOOP +sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + return sqlite3NoopMutex(); +} +#endif /* defined(SQLITE_MUTEX_NOOP) */ +#endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/third_party/sqlite3/mutex_unix.c b/third_party/sqlite3/mutex_unix.c new file mode 100644 index 000000000..2228e07f5 --- /dev/null +++ b/third_party/sqlite3/mutex_unix.c @@ -0,0 +1,395 @@ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for pthreads +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** The code in this file is only used if we are compiling threadsafe +** under unix with pthreads. +** +** Note that this implementation requires a version of pthreads that +** supports recursive mutexes. +*/ +#ifdef SQLITE_MUTEX_PTHREADS + +#include + +/* +** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields +** are necessary under two condidtions: (1) Debug builds and (2) using +** home-grown mutexes. Encapsulate these conditions into a single #define. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) +# define SQLITE_MUTEX_NREF 1 +#else +# define SQLITE_MUTEX_NREF 0 +#endif + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + pthread_mutex_t mutex; /* Mutex controlling the lock */ +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + int id; /* Mutex type */ +#endif +#if SQLITE_MUTEX_NREF + volatile int nRef; /* Number of entrances */ + volatile pthread_t owner; /* Thread that is within this mutex */ + int trace; /* True to trace changes */ +#endif +}; +#if SQLITE_MUTEX_NREF +# define SQLITE3_MUTEX_INITIALIZER(id) \ + {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0} +#elif defined(SQLITE_ENABLE_API_ARMOR) +# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id } +#else +#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER } +#endif + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. On some platforms, +** there might be race conditions that can cause these routines to +** deliver incorrect results. In particular, if pthread_equal() is +** not an atomic operation, then these routines might delivery +** incorrect results. On most platforms, pthread_equal() is a +** comparison of two integers and is therefore atomic. But we are +** told that HPUX is not such a platform. If so, then these routines +** will not always work correctly on HPUX. +** +** On those platforms where pthread_equal() is not atomic, SQLite +** should be compiled without -DSQLITE_DEBUG and with -DNDEBUG to +** make sure no assert() statements are evaluated and hence these +** routines are never called. +*/ +#if !defined(NDEBUG) || defined(SQLITE_DEBUG) +static int pthreadMutexHeld(sqlite3_mutex *p){ + return (p->nRef!=0 && pthread_equal(p->owner, pthread_self())); +} +static int pthreadMutexNotheld(sqlite3_mutex *p){ + return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; +} +#endif + +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int pthreadMutexInit(void){ return SQLITE_OK; } +static int pthreadMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. SQLite +** will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MAIN +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +**
+** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *pthreadMutexAlloc(int iType){ + static sqlite3_mutex staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) + }; + sqlite3_mutex *p; + switch( iType ){ + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, we will have to + ** build our own. See below. */ + pthread_mutex_init(&p->mutex, 0); +#else + /* Use a recursive mutex if it is available */ + pthread_mutexattr_t recursiveAttr; + pthread_mutexattr_init(&recursiveAttr); + pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&p->mutex, &recursiveAttr); + pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_RECURSIVE; +#endif + } + break; + } + case SQLITE_MUTEX_FAST: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + pthread_mutex_init(&p->mutex, 0); +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_FAST; +#endif + } + break; + } + default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + break; + } + } +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + assert( p==0 || p->id==iType ); +#endif + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void pthreadMutexFree(sqlite3_mutex *p){ + assert( p->nRef==0 ); +#if SQLITE_ENABLE_API_ARMOR + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) +#endif + { + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void pthreadMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + }else{ + pthread_mutex_lock(&p->mutex); + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + pthread_mutex_lock(&p->mutex); +#if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); + p->owner = pthread_self(); + p->nRef++; +#endif +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} +static int pthreadMutexTry(sqlite3_mutex *p){ + int rc; + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + rc = SQLITE_OK; + }else if( pthread_mutex_trylock(&p->mutex)==0 ){ + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + if( pthread_mutex_trylock(&p->mutex)==0 ){ +#if SQLITE_MUTEX_NREF + p->owner = pthread_self(); + p->nRef++; +#endif + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } +#endif + +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK && p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void pthreadMutexLeave(sqlite3_mutex *p){ + assert( pthreadMutexHeld(p) ); +#if SQLITE_MUTEX_NREF + p->nRef--; + if( p->nRef==0 ) p->owner = 0; +#endif + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + if( p->nRef==0 ){ + pthread_mutex_unlock(&p->mutex); + } +#else + pthread_mutex_unlock(&p->mutex); +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} + +sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + static const sqlite3_mutex_methods sMutex = { + pthreadMutexInit, + pthreadMutexEnd, + pthreadMutexAlloc, + pthreadMutexFree, + pthreadMutexEnter, + pthreadMutexTry, + pthreadMutexLeave, +#ifdef SQLITE_DEBUG + pthreadMutexHeld, + pthreadMutexNotheld +#else + 0, + 0 +#endif + }; + + return &sMutex; +} + +#endif /* SQLITE_MUTEX_PTHREADS */ diff --git a/third_party/sqlite3/notify.c b/third_party/sqlite3/notify.c new file mode 100644 index 000000000..66f7fb7d1 --- /dev/null +++ b/third_party/sqlite3/notify.c @@ -0,0 +1,333 @@ +/* +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. +*/ +#include "third_party/sqlite3/btreeInt.inc" +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + +/* +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() +*/ + +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)) ) + +/* +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MAIN +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: +** +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. +** +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. +** +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. +*/ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; + + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); + + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } +} + +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; +} + +/* +** Obtain the STATIC_MAIN mutex. +*/ +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); + checkListProperties(0); +} + +/* +** Release the STATIC_MAIN mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +int sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); + } + } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. +*/ +void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); +} + +/* +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. +*/ +void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ + + aArg = aStatic; + enterMutex(); /* Enter STATIC_MAIN mutex */ + + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; + } + + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; + } + } + + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MAIN mutex */ +} + +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif diff --git a/third_party/sqlite3/opcodes.c b/third_party/sqlite3/opcodes.c new file mode 100644 index 000000000..a8282b1f0 --- /dev/null +++ b/third_party/sqlite3/opcodes.c @@ -0,0 +1,196 @@ +/* clang-format off */ +/* Automatically generated. Do not edit */ +/* See the tool/mkopcodec.tcl script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) \ + || defined(VDBE_PROFILE) \ + || defined(SQLITE_DEBUG) +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) +# define OpHelp(X) "\0" X +#else +# define OpHelp(X) +#endif +const char *sqlite3OpcodeName(int i){ + static const char *const azName[] = { + /* 0 */ "Savepoint" OpHelp(""), + /* 1 */ "AutoCommit" OpHelp(""), + /* 2 */ "Transaction" OpHelp(""), + /* 3 */ "SorterNext" OpHelp(""), + /* 4 */ "Prev" OpHelp(""), + /* 5 */ "Next" OpHelp(""), + /* 6 */ "Checkpoint" OpHelp(""), + /* 7 */ "JournalMode" OpHelp(""), + /* 8 */ "Vacuum" OpHelp(""), + /* 9 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 10 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 11 */ "Goto" OpHelp(""), + /* 12 */ "Gosub" OpHelp(""), + /* 13 */ "InitCoroutine" OpHelp(""), + /* 14 */ "Yield" OpHelp(""), + /* 15 */ "MustBeInt" OpHelp(""), + /* 16 */ "Jump" OpHelp(""), + /* 17 */ "Once" OpHelp(""), + /* 18 */ "If" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "IfNot" OpHelp(""), + /* 21 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 22 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 26 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 27 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 28 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 29 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 30 */ "Found" OpHelp("key=r[P3@P4]"), + /* 31 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 32 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 33 */ "Last" OpHelp(""), + /* 34 */ "IfSmaller" OpHelp(""), + /* 35 */ "SorterSort" OpHelp(""), + /* 36 */ "Sort" OpHelp(""), + /* 37 */ "Rewind" OpHelp(""), + /* 38 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 39 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 40 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 42 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 45 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 46 */ "Program" OpHelp(""), + /* 47 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 48 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 49 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), + /* 53 */ "Eq" OpHelp("IF r[P3]==r[P1]"), + /* 54 */ "Gt" OpHelp("IF r[P3]>r[P1]"), + /* 55 */ "Le" OpHelp("IF r[P3]<=r[P1]"), + /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), + /* 58 */ "ElseNotEq" OpHelp(""), + /* 59 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 60 */ "IncrVacuum" OpHelp(""), + /* 61 */ "VNext" OpHelp(""), + /* 62 */ "Init" OpHelp("Start at P2"), + /* 63 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 64 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 65 */ "Return" OpHelp(""), + /* 66 */ "EndCoroutine" OpHelp(""), + /* 67 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 68 */ "Halt" OpHelp(""), + /* 69 */ "Integer" OpHelp("r[P2]=P1"), + /* 70 */ "Int64" OpHelp("r[P2]=P4"), + /* 71 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 72 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 73 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 74 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 75 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 76 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 77 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 78 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 79 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 80 */ "ChngCntRow" OpHelp("output=r[P1]"), + /* 81 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 82 */ "CollSeq" OpHelp(""), + /* 83 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 84 */ "RealAffinity" OpHelp(""), + /* 85 */ "Cast" OpHelp("affinity(r[P1])"), + /* 86 */ "Permutation" OpHelp(""), + /* 87 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 88 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 89 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 90 */ "Column" OpHelp("r[P3]=PX"), + /* 91 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 92 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 93 */ "Count" OpHelp("r[P2]=count()"), + /* 94 */ "ReadCookie" OpHelp(""), + /* 95 */ "SetCookie" OpHelp(""), + /* 96 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 97 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 98 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 99 */ "OpenDup" OpHelp(""), + /* 100 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 101 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 102 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 103 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 104 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 106 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 107 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 108 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 109 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 110 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 111 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 112 */ "SorterOpen" OpHelp(""), + /* 113 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 114 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 115 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 116 */ "String8" OpHelp("r[P2]='P4'"), + /* 117 */ "Close" OpHelp(""), + /* 118 */ "ColumnsUsed" OpHelp(""), + /* 119 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 120 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 121 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 122 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 123 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 124 */ "RowCell" OpHelp(""), + /* 125 */ "Delete" OpHelp(""), + /* 126 */ "ResetCount" OpHelp(""), + /* 127 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 128 */ "SorterData" OpHelp("r[P2]=data"), + /* 129 */ "RowData" OpHelp("r[P2]=data"), + /* 130 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 131 */ "NullRow" OpHelp(""), + /* 132 */ "SeekEnd" OpHelp(""), + /* 133 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 134 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 135 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 136 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 137 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 138 */ "FinishSeek" OpHelp(""), + /* 139 */ "Destroy" OpHelp(""), + /* 140 */ "Clear" OpHelp(""), + /* 141 */ "ResetSorter" OpHelp(""), + /* 142 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 143 */ "SqlExec" OpHelp(""), + /* 144 */ "ParseSchema" OpHelp(""), + /* 145 */ "LoadAnalysis" OpHelp(""), + /* 146 */ "DropTable" OpHelp(""), + /* 147 */ "DropIndex" OpHelp(""), + /* 148 */ "DropTrigger" OpHelp(""), + /* 149 */ "IntegrityCk" OpHelp(""), + /* 150 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 151 */ "Param" OpHelp(""), + /* 152 */ "Real" OpHelp("r[P2]=P4"), + /* 153 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 154 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 155 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 156 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 157 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 158 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 159 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 160 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 161 */ "Expire" OpHelp(""), + /* 162 */ "CursorLock" OpHelp(""), + /* 163 */ "CursorUnlock" OpHelp(""), + /* 164 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 165 */ "VBegin" OpHelp(""), + /* 166 */ "VCreate" OpHelp(""), + /* 167 */ "VDestroy" OpHelp(""), + /* 168 */ "VOpen" OpHelp(""), + /* 169 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 170 */ "VRename" OpHelp(""), + /* 171 */ "Pagecount" OpHelp(""), + /* 172 */ "MaxPgcnt" OpHelp(""), + /* 173 */ "Trace" OpHelp(""), + /* 174 */ "CursorHint" OpHelp(""), + /* 175 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 176 */ "Noop" OpHelp(""), + /* 177 */ "Explain" OpHelp(""), + /* 178 */ "Abortable" OpHelp(""), + }; + return azName[i]; +} +#endif diff --git a/third_party/sqlite3/opcodes.inc b/third_party/sqlite3/opcodes.inc new file mode 100644 index 000000000..e808c9787 --- /dev/null +++ b/third_party/sqlite3/opcodes.inc @@ -0,0 +1,232 @@ +/* Automatically generated. Do not edit */ +/* See the tool/mkopcodeh.tcl script for details */ +#define OP_Savepoint 0 +#define OP_AutoCommit 1 +#define OP_Transaction 2 +#define OP_SorterNext 3 /* jump */ +#define OP_Prev 4 /* jump */ +#define OP_Next 5 /* jump */ +#define OP_Checkpoint 6 +#define OP_JournalMode 7 +#define OP_Vacuum 8 +#define OP_VFilter 9 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 10 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 11 /* jump */ +#define OP_Gosub 12 /* jump */ +#define OP_InitCoroutine 13 /* jump */ +#define OP_Yield 14 /* jump */ +#define OP_MustBeInt 15 /* jump */ +#define OP_Jump 16 /* jump */ +#define OP_Once 17 /* jump */ +#define OP_If 18 /* jump */ +#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ +#define OP_IfNot 20 /* jump */ +#define OP_IfNullRow \ + 21 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 22 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 25 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 26 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 30 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_NotExists 32 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 33 /* jump */ +#define OP_IfSmaller 34 /* jump */ +#define OP_SorterSort 35 /* jump */ +#define OP_Sort 36 /* jump */ +#define OP_Rewind 37 /* jump */ +#define OP_IdxLE 38 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 39 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 42 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_RowSetTest 45 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 46 /* jump */ +#define OP_FkIfZero 47 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 48 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 49 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 \ + */ +#define OP_IsNull \ + 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull \ + 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +#define OP_Eq 53 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */ +#define OP_Gt 54 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */ +#define OP_Le 55 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */ +#define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ +#define OP_ElseNotEq 58 /* jump, same as TK_ESCAPE */ +#define OP_DecrJumpZero 59 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 60 /* jump */ +#define OP_VNext 61 /* jump */ +#define OP_Init 62 /* jump, synopsis: Start at P2 */ +#define OP_PureFunc 63 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 64 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 65 +#define OP_EndCoroutine 66 +#define OP_HaltIfNull 67 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 68 +#define OP_Integer 69 /* synopsis: r[P2]=P1 */ +#define OP_Int64 70 /* synopsis: r[P2]=P4 */ +#define OP_String 71 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 72 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 73 /* synopsis: r[P1]=NULL */ +#define OP_Blob 74 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 75 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 76 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 77 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 78 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 79 /* synopsis: r[P2]=r[P1] */ +#define OP_ChngCntRow 80 /* synopsis: output=r[P1] */ +#define OP_ResultRow 81 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 82 +#define OP_AddImm 83 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 84 +#define OP_Cast 85 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 86 +#define OP_Compare 87 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 88 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_Offset 89 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 90 /* synopsis: r[P3]=PX */ +#define OP_Affinity 91 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 92 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 93 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 94 +#define OP_SetCookie 95 +#define OP_ReopenIdx 96 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 97 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 98 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenDup 99 +#define OP_OpenAutoindex 100 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 101 /* synopsis: nColumn=P2 */ +#define OP_BitAnd 102 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 103 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 104 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] \ + */ +#define OP_Add 106 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 107 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 108 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 109 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 110 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 111 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_SorterOpen 112 +#define OP_BitNot 113 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_SequenceTest 114 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 115 /* synopsis: P3 columns in r[P2] */ +#define OP_String8 116 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_Close 117 +#define OP_ColumnsUsed 118 +#define OP_SeekScan 119 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 120 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 121 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 122 /* synopsis: r[P2]=rowid */ +#define OP_Insert 123 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 124 +#define OP_Delete 125 +#define OP_ResetCount 126 +#define OP_SorterCompare 127 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 \ + */ +#define OP_SorterData 128 /* synopsis: r[P2]=data */ +#define OP_RowData 129 /* synopsis: r[P2]=data */ +#define OP_Rowid 130 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 131 +#define OP_SeekEnd 132 +#define OP_IdxInsert 133 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 134 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 135 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 136 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 137 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 138 +#define OP_Destroy 139 +#define OP_Clear 140 +#define OP_ResetSorter 141 +#define OP_CreateBtree 142 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 143 +#define OP_ParseSchema 144 +#define OP_LoadAnalysis 145 +#define OP_DropTable 146 +#define OP_DropIndex 147 +#define OP_DropTrigger 148 +#define OP_IntegrityCk 149 +#define OP_RowSetAdd 150 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 151 +#define OP_Real 152 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_FkCounter 153 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 154 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit \ + 155 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 156 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 157 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 158 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 159 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 160 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 161 +#define OP_CursorLock 162 +#define OP_CursorUnlock 163 +#define OP_TableLock 164 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 165 +#define OP_VCreate 166 +#define OP_VDestroy 167 +#define OP_VOpen 168 +#define OP_VColumn 169 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 170 +#define OP_Pagecount 171 +#define OP_MaxPgcnt 172 +#define OP_Trace 173 +#define OP_CursorHint 174 +#define OP_ReleaseReg 175 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 176 +#define OP_Explain 177 +#define OP_Abortable 178 +/* clang-format off */ + +/* Properties such as "out2" or "jump" that are specified in +** comments following the "case" for each opcode in the vdbe.c +** are encoded into bitvectors as follows: +*/ +#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x02 /* in1: P1 is an input */ +#define OPFLG_IN2 0x04 /* in2: P2 is an input */ +#define OPFLG_IN3 0x08 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_INITIALIZER {\ +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ +/* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ +/* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x01, 0x09, 0x09,\ +/* 24 */ 0x09, 0x09, 0x01, 0x09, 0x09, 0x09, 0x09, 0x09,\ +/* 32 */ 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ +/* 40 */ 0x01, 0x01, 0x23, 0x26, 0x26, 0x0b, 0x01, 0x01,\ +/* 48 */ 0x03, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x01, 0x01, 0x00,\ +/* 64 */ 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10, 0x10,\ +/* 72 */ 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10, 0x10,\ +/* 80 */ 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ +/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00,\ +/* 96 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x26,\ +/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 112 */ 0x00, 0x12, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ +/* 120 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x04, 0x04, 0x00,\ +/* 136 */ 0x00, 0x10, 0x00, 0x10, 0x00, 0x00, 0x10, 0x00,\ +/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10,\ +/* 152 */ 0x10, 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00,\ +/* 176 */ 0x00, 0x00, 0x00,} + +/* The sqlite3P2Values() routine is able to run faster if it knows +** the value of the largest JUMP opcode. The smaller the maximum +** JUMP opcode the better, so the mkopcodeh.tcl script that +** generated this include file strives to group all JUMP opcodes +** together near the beginning of the list. +*/ +#define SQLITE_MX_JUMP_OPCODE 62 /* Maximum JUMP opcode */ diff --git a/third_party/sqlite3/os.c b/third_party/sqlite3/os.c new file mode 100644 index 000000000..dc1fdfb79 --- /dev/null +++ b/third_party/sqlite3/os.c @@ -0,0 +1,439 @@ +/* +** 2005 November 29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains OS interface code that is common to all +** architectures. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#if defined(SQLITE_TEST) +int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +int sqlite3_io_error_benign = 0; /* True if errors are benign */ +int sqlite3_diskfull_pending = 0; +int sqlite3_diskfull = 0; +#endif /* defined(SQLITE_TEST) */ + +/* +** When testing, also keep a count of the number of open files. +*/ +#if defined(SQLITE_TEST) +int sqlite3_open_file_count = 0; +#endif /* defined(SQLITE_TEST) */ + +/* +** The default SQLite sqlite3_vfs implementations do not allocate +** memory (actually, os_unix.c allocates a small amount of memory +** from within OsOpen()), but some third-party implementations may. +** So we test the effects of a malloc() failing and the sqlite3OsXXX() +** function returning SQLITE_IOERR_NOMEM using the DO_OS_MALLOC_TEST macro. +** +** The following functions are instrumented for malloc() failure +** testing: +** +** sqlite3OsRead() +** sqlite3OsWrite() +** sqlite3OsSync() +** sqlite3OsFileSize() +** sqlite3OsLock() +** sqlite3OsCheckReservedLock() +** sqlite3OsFileControl() +** sqlite3OsShmMap() +** sqlite3OsOpen() +** sqlite3OsDelete() +** sqlite3OsAccess() +** sqlite3OsFullPathname() +** +*/ +#if defined(SQLITE_TEST) +int sqlite3_memdebug_vfs_oom_test = 1; + #define DO_OS_MALLOC_TEST(x) \ + if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \ + void *pTstAlloc = sqlite3Malloc(10); \ + if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT; \ + sqlite3_free(pTstAlloc); \ + } +#else + #define DO_OS_MALLOC_TEST(x) +#endif + +/* +** The following routines are convenience wrappers around methods +** of the sqlite3_file object. This is mostly just syntactic sugar. All +** of this would be completely automatic if SQLite were coded using +** C++ instead of plain old C. +*/ +void sqlite3OsClose(sqlite3_file *pId){ + if( pId->pMethods ){ + pId->pMethods->xClose(pId); + pId->pMethods = 0; + } +} +int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xRead(id, pBuf, amt, offset); +} +int sqlite3OsWrite(sqlite3_file *id, const void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xWrite(id, pBuf, amt, offset); +} +int sqlite3OsTruncate(sqlite3_file *id, i64 size){ + return id->pMethods->xTruncate(id, size); +} +int sqlite3OsSync(sqlite3_file *id, int flags){ + DO_OS_MALLOC_TEST(id); + return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; +} +int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFileSize(id, pSize); +} +int sqlite3OsLock(sqlite3_file *id, int lockType){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xLock(id, lockType); +} +int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + return id->pMethods->xUnlock(id, lockType); +} +int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xCheckReservedLock(id, pResOut); +} + +/* +** Use sqlite3OsFileControl() when we are doing something that might fail +** and we need to know about the failures. Use sqlite3OsFileControlHint() +** when simply tossing information over the wall to the VFS and we do not +** really care if the VFS receives and understands the information since it +** is only a hint and can be safely ignored. The sqlite3OsFileControlHint() +** routine has no return value since the return value would be meaningless. +*/ +int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ + if( id->pMethods==0 ) return SQLITE_NOTFOUND; +#ifdef SQLITE_TEST + if( op!=SQLITE_FCNTL_COMMIT_PHASETWO + && op!=SQLITE_FCNTL_LOCK_TIMEOUT + && op!=SQLITE_FCNTL_CKPT_DONE + && op!=SQLITE_FCNTL_CKPT_START + ){ + /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point + ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** but the transaction is committed anyway. + ** + ** The core must call OsFileControl() though, not OsFileControlHint(), + ** as if a custom VFS (e.g. zipvfs) returns an error here, it probably + ** means the commit really has failed and an error should be returned + ** to the user. + ** + ** The CKPT_DONE and CKPT_START file-controls are write-only signals + ** to the cksumvfs. Their return code is meaningless and is ignored + ** by the SQLite core, so there is no point in simulating OOMs for them. + */ + DO_OS_MALLOC_TEST(id); + } +#endif + return id->pMethods->xFileControl(id, op, pArg); +} +void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){ + if( id->pMethods ) (void)id->pMethods->xFileControl(id, op, pArg); +} + +int sqlite3OsSectorSize(sqlite3_file *id){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); +} +int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + return id->pMethods->xDeviceCharacteristics(id); +} +#ifndef SQLITE_OMIT_WAL +int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ + return id->pMethods->xShmLock(id, offset, n, flags); +} +void sqlite3OsShmBarrier(sqlite3_file *id){ + id->pMethods->xShmBarrier(id); +} +int sqlite3OsShmUnmap(sqlite3_file *id, int deleteFlag){ + return id->pMethods->xShmUnmap(id, deleteFlag); +} +int sqlite3OsShmMap( + sqlite3_file *id, /* Database file handle */ + int iPage, + int pgsz, + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Pointer to mapping */ +){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); +} +#endif /* SQLITE_OMIT_WAL */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + +/* +** The next group of routines are convenience wrappers around the +** VFS methods. +*/ +int sqlite3OsOpen( + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, + int *pFlagsOut +){ + int rc; + DO_OS_MALLOC_TEST(0); + /* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed + ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, + ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before + ** reaching the VFS. */ + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); + assert( rc==SQLITE_OK || pFile->pMethods==0 ); + return rc; +} +int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + DO_OS_MALLOC_TEST(0); + assert( dirSync==0 || dirSync==1 ); + return pVfs->xDelete(pVfs, zPath, dirSync); +} +int sqlite3OsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + DO_OS_MALLOC_TEST(0); + return pVfs->xAccess(pVfs, zPath, flags, pResOut); +} +int sqlite3OsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, + char *zPathOut +){ + DO_OS_MALLOC_TEST(0); + zPathOut[0] = 0; + return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); +} +#ifndef SQLITE_OMIT_LOAD_EXTENSION +void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return pVfs->xDlOpen(pVfs, zPath); +} +void sqlite3OsDlError(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + pVfs->xDlError(pVfs, nByte, zBufOut); +} +void (*sqlite3OsDlSym(sqlite3_vfs *pVfs, void *pHdle, const char *zSym))(void){ + return pVfs->xDlSym(pVfs, pHdle, zSym); +} +void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + pVfs->xDlClose(pVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + if( sqlite3Config.iPrngSeed ){ + memset(zBufOut, 0, nByte); + if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); + memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); + return SQLITE_OK; + }else{ + return pVfs->xRandomness(pVfs, nByte, zBufOut); + } + +} +int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ + return pVfs->xSleep(pVfs, nMicro); +} +int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} +int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); + }else{ + double r; + rc = pVfs->xCurrentTime(pVfs, &r); + *pTimeOut = (sqlite3_int64)(r*86400000.0); + } + return rc; +} + +int sqlite3OsOpenMalloc( + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, + int flags, + int *pOutFlags +){ + int rc; + sqlite3_file *pFile; + pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); + if( pFile ){ + rc = sqlite3OsOpen(pVfs, zFile, pFile, flags, pOutFlags); + if( rc!=SQLITE_OK ){ + sqlite3_free(pFile); + }else{ + *ppFile = pFile; + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + return rc; +} +void sqlite3OsCloseFree(sqlite3_file *pFile){ + assert( pFile ); + sqlite3OsClose(pFile); + sqlite3_free(pFile); +} + +/* +** This function is a wrapper around the OS specific implementation of +** sqlite3_os_init(). The purpose of the wrapper is to provide the +** ability to simulate a malloc failure, so that the handling of an +** error in sqlite3_os_init() by the upper layers can be tested. +*/ +int sqlite3OsInit(void){ + void *p = sqlite3_malloc(10); + if( p==0 ) return SQLITE_NOMEM_BKPT; + sqlite3_free(p); + return sqlite3_os_init(); +} + +/* +** The list of all registered VFS implementations. +*/ +static sqlite3_vfs * SQLITE_WSD vfsList = 0; +#define vfsList GLOBAL(sqlite3_vfs *, vfsList) + +/* +** Locate a VFS by name. If no name is given, simply return the +** first VFS on the list. +*/ +sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ + sqlite3_vfs *pVfs = 0; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return 0; +#endif +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + sqlite3_mutex_enter(mutex); + for(pVfs = vfsList; pVfs; pVfs=pVfs->pNext){ + if( zVfs==0 ) break; + if( strcmp(zVfs, pVfs->zName)==0 ) break; + } + sqlite3_mutex_leave(mutex); + return pVfs; +} + +/* +** Unlink a VFS from the linked list +*/ +static void vfsUnlink(sqlite3_vfs *pVfs){ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)) ); + if( pVfs==0 ){ + /* No-op */ + }else if( vfsList==pVfs ){ + vfsList = pVfs->pNext; + }else if( vfsList ){ + sqlite3_vfs *p = vfsList; + while( p->pNext && p->pNext!=pVfs ){ + p = p->pNext; + } + if( p->pNext==pVfs ){ + p->pNext = pVfs->pNext; + } + } +} + +/* +** Register a VFS with the system. It is harmless to register the same +** VFS multiple times. The new VFS becomes the default if makeDflt is +** true. +*/ +int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + if( makeDflt || vfsList==0 ){ + pVfs->pNext = vfsList; + vfsList = pVfs; + }else{ + pVfs->pNext = vfsList->pNext; + vfsList->pNext = pVfs; + } + assert(vfsList); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/* +** Unregister a VFS so that it is no longer accessible. +*/ +int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} diff --git a/third_party/sqlite3/os.inc b/third_party/sqlite3/os.inc new file mode 100644 index 000000000..248074373 --- /dev/null +++ b/third_party/sqlite3/os.inc @@ -0,0 +1,213 @@ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ +/* clang-format off */ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +#include "third_party/sqlite3/os_setup.inc" + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +void sqlite3OsClose(sqlite3_file*); +int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +int sqlite3OsTruncate(sqlite3_file*, i64 size); +int sqlite3OsSync(sqlite3_file*, int); +int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +int sqlite3OsLock(sqlite3_file*, int); +int sqlite3OsUnlock(sqlite3_file*, int); +int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +int sqlite3OsFileControl(sqlite3_file*,int,void*); +void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +int sqlite3OsSectorSize(sqlite3_file *id); +int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +#ifndef SQLITE_OMIT_WAL +int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +void sqlite3OsShmBarrier(sqlite3_file *id); +int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +void sqlite3OsDlError(sqlite3_vfs *, int, char *); +void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +int sqlite3OsSleep(sqlite3_vfs *, int); +int sqlite3OsGetLastError(sqlite3_vfs*); +int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +void sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ diff --git a/third_party/sqlite3/os_common.inc b/third_party/sqlite3/os_common.inc new file mode 100644 index 000000000..fc2b12f28 --- /dev/null +++ b/third_party/sqlite3/os_common.inc @@ -0,0 +1,106 @@ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ +/* clang-format off */ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +#include "third_party/sqlite3/hwtime.inc" + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#if defined(SQLITE_TEST) +extern int sqlite3_io_error_hit; +extern int sqlite3_io_error_hardhit; +extern int sqlite3_io_error_pending; +extern int sqlite3_io_error_persist; +extern int sqlite3_io_error_benign; +extern int sqlite3_diskfull_pending; +extern int sqlite3_diskfull; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif /* defined(SQLITE_TEST) */ + +/* +** When testing, keep a count of the number of open files. +*/ +#if defined(SQLITE_TEST) +extern int sqlite3_open_file_count; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif /* defined(SQLITE_TEST) */ + +#endif /* !defined(_OS_COMMON_H_) */ diff --git a/third_party/sqlite3/os_setup.inc b/third_party/sqlite3/os_setup.inc new file mode 100644 index 000000000..fe0dded97 --- /dev/null +++ b/third_party/sqlite3/os_setup.inc @@ -0,0 +1,58 @@ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H +/* clang-format off */ + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#endif /* SQLITE_OS_SETUP_H */ diff --git a/third_party/sqlite3/os_unix.c b/third_party/sqlite3/os_unix.c new file mode 100644 index 000000000..ec1cc62dd --- /dev/null +++ b/third_party/sqlite3/os_unix.c @@ -0,0 +1,8041 @@ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the VFS implementation for unix-like operating systems +** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. +** +** There are actually several different VFS implementations in this file. +** The differences are in the way that file locking is done. The default +** implementation uses Posix Advisory Locks. Alternative implementations +** use flock(), dot-files, various proprietary locking schemas, or simply +** skip locking all together. +** +** This source file is organized into divisions where the logic for various +** subfunctions is contained within the appropriate division. PLEASE +** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed +** in the correct division and should be clearly labeled. +** +** The layout of divisions is as follows: +** +** * General-purpose declarations and utility functions. +** * Unique file ID logic used by VxWorks. +** * Various locking primitive implementations (all except proxy locking): +** + for Posix Advisory Locks +** + for no-op locks +** + for dot-file locks +** + for flock() locking +** + for named semaphore locks (VxWorks only) +** + for AFP filesystem locks (MacOSX only) +** * sqlite3_file methods not associated with locking. +** * Definitions of sqlite3_io_methods objects for all locking +** methods plus "finder" functions for each locking method. +** * sqlite3_vfs method implementations. +** * Locking primitives for the proxy uber-locking-method. (MacOSX only) +** * Definitions of sqlite3_vfs objects for all locking methods +** plus implementations of sqlite3_os_init() and sqlite3_os_end(). +*/ +#include "third_party/sqlite3/sqliteInt.inc" +#if SQLITE_OS_UNIX /* This file is used on unix only */ + /* clang-format off */ + +/* +** There are various methods for file locking used for concurrency +** control: +** +** 1. POSIX locking (the default), +** 2. No locking, +** 3. Dot-file locking, +** 4. flock() locking, +** 5. AFP locking (OSX only), +** 6. Named POSIX semaphores (VXWorks only), +** 7. proxy locking. (OSX only) +** +** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE +** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic +** selection of the appropriate locking style based on the filesystem +** where the database is located. +*/ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/* Use pread() and pwrite() if they are available */ +#if defined(__APPLE__) +# define HAVE_PREAD 1 +# define HAVE_PWRITE 1 +#endif +#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) +# undef USE_PREAD +# define USE_PREAD64 1 +#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) +# undef USE_PREAD64 +# define USE_PREAD 1 +#endif + +/* +** standard include files. +*/ +#include "libc/calls/calls.h" +#include "libc/calls/struct/flock.h" +#include "libc/calls/weirdtypes.h" +#include "libc/errno.h" +#include "libc/isystem/unistd.h" +#include "libc/runtime/sysconf.h" +#include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/mremap.h" +#include "libc/sysv/consts/o.h" +#include "libc/sysv/consts/ok.h" +#include "libc/sysv/consts/prot.h" +#include "libc/time/struct/tm.h" +#include "libc/time/time.h" +#include "libc/mem/mem.h" + +/* +** Try to determine if gethostuuid() is available based on standard +** macros. This might sometimes compute the wrong value for some +** obscure platforms. For those cases, simply compile with one of +** the following: +** +** -DHAVE_GETHOSTUUID=0 +** -DHAVE_GETHOSTUUID=1 +** +** None if this matters except when building on Apple products with +** -DSQLITE_ENABLE_LOCKING_STYLE. +*/ +#ifndef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 0 +# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0))\ + && (!defined(TARGET_OS_MACCATALYST) || (TARGET_OS_MACCATALYST==0)) +# undef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif +# endif +#endif + + +#if OS_VXWORKS +#include +#include + +#include "libc/limits.h" +#endif /* OS_VXWORKS */ + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE +# include +#endif + +#ifdef HAVE_UTIME +#include "libc/time/time.h" +#endif + +/* +** Allowed values of unixFile.fsFlags +*/ +#define SQLITE_FSFLAGS_IS_MSDOS 0x1 + +/* +** If we are to be thread-safe, include the pthreads header. +*/ +#if SQLITE_THREADSAFE +# include +#endif + +/* +** Default permissions when creating a new file +*/ +#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS +# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 +#endif + +/* +** Default permissions when creating auto proxy dir +*/ +#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 +#endif + +/* +** Maximum supported path-length. +*/ +#define MAX_PATHNAME 512 + +/* +** Maximum supported symbolic links +*/ +#define SQLITE_MAX_SYMLINKS 100 + +/* Always cast the getpid() return type for compatibility with +** kernel modules in VxWorks. */ +#define osGetpid(X) (pid_t)getpid() + +/* +** Only set the lastErrno if the error code is a real error and not +** a normal expected return code of SQLITE_BUSY or SQLITE_OK +*/ +#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) + +/* Forward references */ +typedef struct unixShm unixShm; /* Connection shared memory */ +typedef struct unixShmNode unixShmNode; /* Shared memory instance */ +typedef struct unixInodeInfo unixInodeInfo; /* An i-node */ +typedef struct UnixUnusedFd UnixUnusedFd; /* An unused file descriptor */ + +/* +** Sometimes, after a file handle is closed by SQLite, the file descriptor +** cannot be closed immediately. In these cases, instances of the following +** structure are used to store the file descriptor while waiting for an +** opportunity to either close or reuse it. +*/ +struct UnixUnusedFd { + int fd; /* File descriptor to close */ + int flags; /* Flags this file descriptor was opened with */ + UnixUnusedFd *pNext; /* Next unused file descriptor on same file */ +}; + +/* +** The unixFile structure is subclass of sqlite3_file specific to the unix +** VFS implementations. +*/ +typedef struct unixFile unixFile; +struct unixFile { + sqlite3_io_methods const *pMethod; /* Always the first entry */ + sqlite3_vfs *pVfs; /* The VFS that created this unixFile */ + unixInodeInfo *pInode; /* Info about locks on this inode */ + int h; /* The file descriptor */ + unsigned char eFileLock; /* The type of lock held on this fd */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + int lastErrno; /* The unix errno from last I/O error */ + void *lockingContext; /* Locking style specific state */ + UnixUnusedFd *pPreallocatedUnused; /* Pre-allocated UnixUnusedFd */ + const char *zPath; /* Name of the file */ + unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ +#endif + int sectorSize; /* Device sector size */ + int deviceCharacteristics; /* Precomputed device characteristics */ +#if SQLITE_ENABLE_LOCKING_STYLE + int openFlags; /* The flags specified at open() */ +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + unsigned fsFlags; /* cached details from statfs() */ +#endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + unsigned iBusyTimeout; /* Wait this many millisec on locks */ +#endif +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID */ +#endif +#ifdef SQLITE_DEBUG + /* The next group of variables are used to track whether or not the + ** transaction counter in bytes 24-27 of database files are updated + ** whenever any part of the database changes. An assertion fault will + ** occur if a file is updated without also updating the transaction + ** counter. This test is made to avoid new problems similar to the + ** one described by ticket #3584. + */ + unsigned char transCntrChng; /* True if the transaction counter changed */ + unsigned char dbUpdate; /* True if any part of database file changed */ + unsigned char inNormalWrite; /* True if in a normal write operation */ + +#endif + +#ifdef SQLITE_TEST + /* In test mode, increase the size of this structure a bit so that + ** it is larger than the struct CrashFile defined in test6.c. + */ + char aPadding[32]; +#endif +}; + +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static pid_t randomnessPid = 0; + +/* +** Allowed values for the unixFile.ctrlFlags bitmask: +*/ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#ifndef SQLITE_DISABLE_DIRSYNC +# define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ +#else +# define UNIXFILE_DIRSYNC 0x00 +#endif +#define UNIXFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +#define UNIXFILE_DELETE 0x20 /* Delete on close */ +#define UNIXFILE_URI 0x40 /* Filename might have query parameters */ +#define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ + +/* +** Include code that is common to all os_*.c files +*/ +#include "third_party/sqlite3/os_common.inc" + +/* +** Define various macros that are missing from some systems. +*/ +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifdef SQLITE_DISABLE_LFS +# undef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifndef O_NOFOLLOW +# define O_NOFOLLOW 0 +#endif +#ifndef O_BINARY +# define O_BINARY 0 +#endif + +/* +** The threadid macro resolves to the thread-id or to 0. Used for +** testing and debugging only. +*/ +#if SQLITE_THREADSAFE +#define threadid pthread_self() +#else +#define threadid 0 +#endif + +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + +/* +** Explicitly call the 64-bit version of lseek() on Android. Otherwise, lseek() +** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. +*/ +#ifdef __ANDROID__ +# define lseek lseek64 +#endif + +#ifdef __linux__ +/* +** Linux-specific IOCTL magic numbers used for controlling F2FS +*/ +#define F2FS_IOCTL_MAGIC 0xf5 +#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1) +#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2) +#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3) +#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5) +#define F2FS_IOC_GET_FEATURES _IOR(F2FS_IOCTL_MAGIC, 12, u32) +#define F2FS_FEATURE_ATOMIC_WRITE 0x0004 +#endif /* __linux__ */ + + +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. +** +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. +*/ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} + +/* Forward reference */ +static int openDirectory(const char*, int*); +static int unixGetpagesize(void); + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct unix_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) + + { "close", (sqlite3_syscall_ptr)close, 0 }, +#define osClose ((int(*)(int))aSyscall[1].pCurrent) + + { "access", (sqlite3_syscall_ptr)access, 0 }, +#define osAccess ((int(*)(const char*,int))aSyscall[2].pCurrent) + + { "getcwd", (sqlite3_syscall_ptr)getcwd, 0 }, +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[3].pCurrent) + + { "stat", (sqlite3_syscall_ptr)stat, 0 }, +#define osStat ((int(*)(const char*,struct stat*))aSyscall[4].pCurrent) + +/* +** The DJGPP compiler environment looks mostly like Unix, but it +** lacks the fcntl() system call. So redefine fcntl() to be something +** that always succeeds. This means that locking does not occur under +** DJGPP. But it is DOS - what did you expect? +*/ +#ifdef __DJGPP__ + { "fstat", 0, 0 }, +#define osFstat(a,b,c) 0 +#else + { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, +#define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) +#endif + + { "ftruncate", (sqlite3_syscall_ptr)ftruncate, 0 }, +#define osFtruncate ((int(*)(int,off_t))aSyscall[6].pCurrent) + + { "fcntl", (sqlite3_syscall_ptr)fcntl, 0 }, +#define osFcntl ((int(*)(int,int,...))aSyscall[7].pCurrent) + + { "read", (sqlite3_syscall_ptr)read, 0 }, +#define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pread", (sqlite3_syscall_ptr)pread, 0 }, +#else + { "pread", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].pCurrent) + +#if defined(USE_PREAD64) + { "pread64", (sqlite3_syscall_ptr)pread64, 0 }, +#else + { "pread64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) + + { "write", (sqlite3_syscall_ptr)write, 0 }, +#define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, +#else + { "pwrite", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[12].pCurrent) + +#if defined(USE_PREAD64) + { "pwrite64", (sqlite3_syscall_ptr)pwrite64, 0 }, +#else + { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ + aSyscall[13].pCurrent) + + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, +#else + { "fallocate", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) + + { "unlink", (sqlite3_syscall_ptr)unlink, 0 }, +#define osUnlink ((int(*)(const char*))aSyscall[16].pCurrent) + + { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, +#define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) + + { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, +#define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) + + { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, +#define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + +#if defined(HAVE_FCHOWN) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + +#if defined(HAVE_FCHOWN) + { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#else + { "geteuid", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#else + { "mmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#else + { "munmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMunmap ((int(*)(void*,size_t))aSyscall[23].pCurrent) + +#if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[24].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#else + { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) + +#if defined(HAVE_READLINK) + { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) + +#if defined(HAVE_LSTAT) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#else + { "lstat", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) + +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) +# else + { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) +# endif +#else + { "ioctl", (sqlite3_syscall_ptr)0, 0 }, +#endif + +}; /* End of the overrideable system calls */ + + +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int robustFchown(int fd, uid_t uid, gid_t gid){ +#if defined(HAVE_FCHOWN) + return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; +#endif +} + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int unixSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + osClose(fd); + sqlite3_log(SQLITE_WARNING, + "attempt to open \"%s\" as file descriptor %d", z, fd); + fd = -1; + if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; + } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } +#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + } + return fd; +} + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the unixInodeInfo and +** vxworksFileId objects used by this file, all of which may be +** shared by multiple threads. +** +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** unixEnterMutex() +** assert( unixMutexHeld() ); +** unixEnterLeave() +** +** To prevent deadlock, the global unixBigLock must must be acquired +** before the unixInodeInfo.pLockMutex mutex, if both are held. It is +** OK to get the pLockMutex without holding unixBigLock first, but if +** that happens, the unixBigLock mutex must not be acquired until after +** pLockMutex is released. +** +** OK: enter(unixBigLock), enter(pLockInfo) +** OK: enter(unixBigLock) +** OK: enter(pLockInfo) +** ERROR: enter(pLockInfo), enter(unixBigLock) +*/ +static sqlite3_mutex *unixBigLock = 0; +static void unixEnterMutex(void){ + assert( sqlite3_mutex_notheld(unixBigLock) ); /* Not a recursive mutex */ + sqlite3_mutex_enter(unixBigLock); +} +static void unixLeaveMutex(void){ + assert( sqlite3_mutex_held(unixBigLock) ); + sqlite3_mutex_leave(unixBigLock); +} +#ifdef SQLITE_DEBUG +static int unixMutexHeld(void) { + return sqlite3_mutex_held(unixBigLock); +} +#endif + + +#ifdef SQLITE_HAVE_OS_TRACE +/* +** Helper function for printing out trace information from debugging +** binaries. This returns the string representation of the supplied +** integer lock-type. +*/ +static const char *azFileLock(int eFileLock){ + switch( eFileLock ){ + case NO_LOCK: return "NONE"; + case SHARED_LOCK: return "SHARED"; + case RESERVED_LOCK: return "RESERVED"; + case PENDING_LOCK: return "PENDING"; + case EXCLUSIVE_LOCK: return "EXCLUSIVE"; + } + return "ERROR"; +} +#endif + +#ifdef SQLITE_LOCK_TRACE +/* +** Print out information about all locking operations. +** +** This routine is used for troubleshooting locks on multithreaded +** platforms. Enable by compiling with the -DSQLITE_LOCK_TRACE +** command-line option on the compiler. This code is normally +** turned off. +*/ +static int lockTrace(int fd, int op, struct flock *p){ + char *zOpName, *zType; + int s; + int savedErrno; + if( op==F_GETLK ){ + zOpName = "GETLK"; + }else if( op==F_SETLK ){ + zOpName = "SETLK"; + }else{ + s = osFcntl(fd, op, p); + sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s); + return s; + } + if( p->l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( p->l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( p->l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + assert( p->l_whence==SEEK_SET ); + s = osFcntl(fd, op, p); + savedErrno = errno; + sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n", + threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len, + (int)p->l_pid, s); + if( s==(-1) && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){ + struct flock l2; + l2 = *p; + osFcntl(fd, F_GETLK, &l2); + if( l2.l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( l2.l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( l2.l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n", + zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid); + } + errno = savedErrno; + return s; +} +#undef osFcntl +#define osFcntl lockTrace +#endif /* SQLITE_LOCK_TRACE */ + +/* +** Retry ftruncate() calls that fail due to EINTR +** +** All calls to ftruncate() within this file should be made through +** this wrapper. On the Android platform, bypassing the logic below +** could lead to a corrupt database. +*/ +static int robust_ftruncate(int h, sqlite3_int64 sz){ + int rc; +#ifdef __ANDROID__ + /* On Android, ftruncate() always uses 32-bit offsets, even if + ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to + ** truncate a file to any size larger than 2GiB. Silently ignore any + ** such attempts. */ + if( sz>(sqlite3_int64)0x7FFFFFFF ){ + rc = SQLITE_OK; + }else +#endif + do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); + return rc; +} + +/* +** This routine translates a standard POSIX errno code into something +** useful to the clients of the sqlite3 functions. Specifically, it is +** intended to translate a variety of "try again" errors into SQLITE_BUSY +** and a variety of "please close the file descriptor NOW" errors into +** SQLITE_IOERR +** +** Errors during initialization of locks, or file system support for locks, +** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. +*/ +static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + assert((sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK)); + // changed switch to if-else + if (posixError == EACCES || posixError == EAGAIN || posixError == ETIMEDOUT || + posixError == EBUSY || posixError == EINTR || posixError == ENOLCK) + /* random NFS retry error, unless during file system support + * introspection, in which it actually means what it says */ + return SQLITE_BUSY; + + else if (posixError == EPERM) + return SQLITE_PERM; + + else + return sqliteIOErr; +} + + +/****************************************************************************** +****************** Begin Unique File ID Utility Used By VxWorks *************** +** +** On most versions of unix, we can get a unique ID for a file by concatenating +** the device number and the inode number. But this does not work on VxWorks. +** On VxWorks, a unique file id must be based on the canonical filename. +** +** A pointer to an instance of the following structure can be used as a +** unique file ID in VxWorks. Each instance of this structure contains +** a copy of the canonical filename. There is also a reference count. +** The structure is reclaimed when the number of pointers to it drops to +** zero. +** +** There are never very many files open at one time and lookups are not +** a performance-critical path, so it is sufficient to put these +** structures on a linked list. +*/ +struct vxworksFileId { + struct vxworksFileId *pNext; /* Next in a list of them all */ + int nRef; /* Number of references to this one */ + int nName; /* Length of the zCanonicalName[] string */ + char *zCanonicalName; /* Canonical filename */ +}; + +#if OS_VXWORKS +/* +** All unique filenames are held on a linked list headed by this +** variable: +*/ +static struct vxworksFileId *vxworksFileList = 0; + +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * removing any trailing and duplicate / +** * convert /./ into just / +** * convert /A/../ where A is any simple name into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..n-1]. Return the number of +** characters in the simplified name. +*/ +static int vxworksSimplifyName(char *z, int n){ + int i, j; + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + z[j++] = z[i]; + } + z[j] = 0; + return j; +} + +/* +** Find a unique file ID for the given absolute pathname. Return +** a pointer to the vxworksFileId object. This pointer is the unique +** file ID. +** +** The nRef field of the vxworksFileId object is incremented before +** the object is returned. A new vxworksFileId object is created +** and added to the global list if necessary. +** +** If a memory allocation error occurs, return NULL. +*/ +static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ + struct vxworksFileId *pNew; /* search key and new file ID */ + struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ + int n; /* Length of zAbsoluteName string */ + + assert( zAbsoluteName[0]=='/' ); + n = (int)strlen(zAbsoluteName); + pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); + if( pNew==0 ) return 0; + pNew->zCanonicalName = (char*)&pNew[1]; + memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); + n = vxworksSimplifyName(pNew->zCanonicalName, n); + + /* Search for an existing entry that matching the canonical name. + ** If found, increment the reference count and return a pointer to + ** the existing file ID. + */ + unixEnterMutex(); + for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ + if( pCandidate->nName==n + && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 + ){ + sqlite3_free(pNew); + pCandidate->nRef++; + unixLeaveMutex(); + return pCandidate; + } + } + + /* No match was found. We will make a new file ID */ + pNew->nRef = 1; + pNew->nName = n; + pNew->pNext = vxworksFileList; + vxworksFileList = pNew; + unixLeaveMutex(); + return pNew; +} + +/* +** Decrement the reference count on a vxworksFileId object. Free +** the object when the reference count reaches zero. +*/ +static void vxworksReleaseFileId(struct vxworksFileId *pId){ + unixEnterMutex(); + assert( pId->nRef>0 ); + pId->nRef--; + if( pId->nRef==0 ){ + struct vxworksFileId **pp; + for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} + assert( *pp==pId ); + *pp = pId->pNext; + sqlite3_free(pId); + } + unixLeaveMutex(); +} +#endif /* OS_VXWORKS */ +/*************** End of Unique File ID Utility Used By VxWorks **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Posix Advisory Locking **************************** +** +** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) +** section 6.5.2.2 lines 483 through 490 specify that when a process +** sets or clears a lock, that operation overrides any prior locks set +** by the same process. It does not explicitly say so, but this implies +** that it overrides locks set by the same process using a different +** file descriptor. Consider this test case: +** +** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); +** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); +** +** Suppose ./file1 and ./file2 are really the same file (because +** one is a hard or symbolic link to the other) then if you set +** an exclusive lock on fd1, then try to get an exclusive lock +** on fd2, it works. I would have expected the second lock to +** fail since there was already a lock on the file due to fd1. +** But not so. Since both locks came from the same process, the +** second overrides the first, even though they were on different +** file descriptors opened on different file names. +** +** This means that we cannot use POSIX locks to synchronize file access +** among competing threads of the same process. POSIX locks will work fine +** to synchronize access for threads in separate processes, but not +** threads within the same process. +** +** To work around the problem, SQLite has to manage file locks internally +** on its own. Whenever a new database is opened, we have to find the +** specific inode of the database file (the inode is determined by the +** st_dev and st_ino fields of the stat structure that fstat() fills in) +** and check for locks already existing on that inode. When locks are +** created or removed, we have to look at our own internal record of the +** locks to see if another thread has previously set a lock on that same +** inode. +** +** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. +** For VxWorks, we have to use the alternative unique ID system based on +** canonical filename and implemented in the previous division.) +** +** The sqlite3_file structure for POSIX is no longer just an integer file +** descriptor. It is now a structure that holds the integer file +** descriptor and a pointer to a structure that describes the internal +** locks on the corresponding inode. There is one locking structure +** per inode, so if the same inode is opened twice, both unixFile structures +** point to the same locking structure. The locking structure keeps +** a reference count (so we will know when to delete it) and a "cnt" +** field that tells us its internal lock status. cnt==0 means the +** file is unlocked. cnt==-1 means the file has an exclusive lock. +** cnt>0 means there are cnt shared locks on the file. +** +** Any attempt to lock or unlock a file first checks the locking +** structure. The fcntl() system call is only invoked to set a +** POSIX lock if the internal lock structure transitions between +** a locked and an unlocked state. +** +** But wait: there are yet more problems with POSIX advisory locks. +** +** If you close a file descriptor that points to a file that has locks, +** all locks on that file that are owned by the current process are +** released. To work around this problem, each unixInodeInfo object +** maintains a count of the number of pending locks on tha inode. +** When an attempt is made to close an unixFile, if there are +** other unixFile open on the same inode that are holding locks, the call +** to close() the file descriptor is deferred until all of the locks clear. +** The unixInodeInfo structure keeps a list of file descriptors that need to +** be closed and that list is walked (and cleared) when the last lock +** clears. +** +** Yet another problem: LinuxThreads do not play well with posix locks. +** +** Many older versions of linux use the LinuxThreads library which is +** not posix compliant. Under LinuxThreads, a lock created by thread +** A cannot be modified or overridden by a different thread B. +** Only thread A can modify the lock. Locking behavior is correct +** if the appliation uses the newer Native Posix Thread Library (NPTL) +** on linux - with NPTL a lock created by thread A can override locks +** in thread B. But there is no way to know at compile-time which +** threading library is being used. So there is no way to know at +** compile-time whether or not thread A can override locks on thread B. +** One has to do a run-time check to discover the behavior of the +** current process. +** +** SQLite used to support LinuxThreads. But support for LinuxThreads +** was dropped beginning with version 3.7.0. SQLite will still work with +** LinuxThreads provided that (1) there is no more than one connection +** per database file in the same process and (2) database connections +** do not move across threads. +*/ + +/* +** An instance of the following structure serves as the key used +** to locate a particular unixInodeInfo object. +*/ +struct unixFileId { + dev_t dev; /* Device number */ +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID for vxworks. */ +#else + /* We are told that some versions of Android contain a bug that + ** sizes ino_t at only 32-bits instead of 64-bits. (See + ** https://android-review.googlesource.com/#/c/115351/3/dist/sqlite3.c) + ** To work around this, always allocate 64-bits for the inode number. + ** On small machines that only have 32-bit inodes, this wastes 4 bytes, + ** but that should not be a big deal. */ + /* WAS: ino_t ino; */ + u64 ino; /* Inode number */ +#endif +}; + +/* +** An instance of the following structure is allocated for each open +** inode. +** +** A single inode can have multiple file descriptors, so each unixFile +** structure contains a pointer to an instance of this object and this +** object keeps a count of the number of unixFile pointing to it. +** +** Mutex rules: +** +** (1) Only the pLockMutex mutex must be held in order to read or write +** any of the locking fields: +** nShared, nLock, eFileLock, bProcessLock, pUnused +** +** (2) When nRef>0, then the following fields are unchanging and can +** be read (but not written) without holding any mutex: +** fileId, pLockMutex +** +** (3) With the exceptions above, all the fields may only be read +** or written while holding the global unixBigLock mutex. +** +** Deadlock prevention: The global unixBigLock mutex may not +** be acquired while holding the pLockMutex mutex. If both unixBigLock +** and pLockMutex are needed, then unixBigLock must be acquired first. +*/ +struct unixInodeInfo { + struct unixFileId fileId; /* The lookup key */ + sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ + int nShared; /* Number of SHARED locks held */ + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ + unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ + unixInodeInfo *pPrev; /* .... doubly linked */ +#if SQLITE_ENABLE_LOCKING_STYLE + unsigned long long sharedByte; /* for AFP simulated shared lock */ +#endif +#if OS_VXWORKS + sem_t *pSem; /* Named POSIX semaphore */ + char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ +#endif +}; + +/* +** A lists of all unixInodeInfo objects. +** +** Must hold unixBigLock in order to read or write this variable. +*/ +static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ + +#ifdef SQLITE_DEBUG +/* +** True if the inode mutex (on the unixFile.pFileMutex field) is held, or not. +** This routine is used only within assert() to help verify correct mutex +** usage. +*/ +int unixFileMutexHeld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_held(pFile->pInode->pLockMutex); +} +int unixFileMutexNotheld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_notheld(pFile->pInode->pLockMutex); +} +#endif + +/* +** +** This function - unixLogErrorAtLine(), is only ever called via the macro +** unixLogError(). +** +** It is invoked after an error occurs in an OS function and errno has been +** set. It logs a message using sqlite3_log() containing the current value of +** errno and, if possible, the human-readable equivalent from strerror() or +** strerror_r(). +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed (e.g. "unlink", "open") and the associated file-system path, +** if any. +*/ +#define unixLogError(a,b,c) unixLogErrorAtLine(a,b,c,__LINE__) +static int unixLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char *zErr; /* Message from strerror() or equivalent */ + int iErrno = errno; /* Saved syscall error number */ + + /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use + ** the strerror() function to obtain the human-readable error message + ** equivalent to errno. Otherwise, use strerror_r(). + */ +#if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) + char aErr[80]; + memset(aErr, 0, sizeof(aErr)); + zErr = aErr; + + /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, + ** assume that the system provides the GNU version of strerror_r() that + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of + ** strerror_r(), which always writes an error message into aErr[]. + ** + ** If the code incorrectly assumes that it is the POSIX version that is + ** available, the error message will often be an empty string. Not a + ** huge problem. Incorrectly concluding that the GNU version is available + ** could lead to a segfault though. + */ +#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) + zErr = +# endif + strerror_r(iErrno, aErr, sizeof(aErr)-1); + +#elif SQLITE_THREADSAFE + /* This is a threadsafe build, but strerror_r() is not available. */ + zErr = ""; +#else + /* Non-threadsafe build, use strerror(). */ + zErr = strerror(iErrno); +#endif + + if( zPath==0 ) zPath = ""; + sqlite3_log(errcode, + "os_unix.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zErr + ); + + return errcode; +} + +/* +** Close a file descriptor. +** +** We assume that close() almost always works, since it is only in a +** very sick application or on a very sick platform that it might fail. +** If it does fail, simply leak the file descriptor, but do log the +** error. +** +** Note that it is not safe to retry close() after EINTR since the +** file descriptor might have already been reused by another thread. +** So we don't even try to recover from an EINTR. Just log the error +** and move on. +*/ +static void robust_close(unixFile *pFile, int h, int lineno){ + if( osClose(h) ){ + unixLogErrorAtLine(SQLITE_IOERR_CLOSE, "close", + pFile ? pFile->zPath : 0, lineno); + } +} + +/* +** Set the pFile->lastErrno. Do this in a subroutine as that provides +** a convenient place to set a breakpoint. +*/ +static void storeLastErrno(unixFile *pFile, int error){ + pFile->lastErrno = error; +} + +/* +** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +*/ +static void closePendingFds(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p; + UnixUnusedFd *pNext; + assert( unixFileMutexHeld(pFile) ); + for(p=pInode->pUnused; p; p=pNext){ + pNext = p->pNext; + robust_close(pFile, p->fd, __LINE__); + sqlite3_free(p); + } + pInode->pUnused = 0; +} + +/* +** Release a unixInodeInfo structure previously allocated by findInodeInfo(). +** +** The global mutex must be held when this routine is called, but the mutex +** on the inode being deleted must NOT be held. +*/ +static void releaseInodeInfo(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + assert( unixFileMutexNotheld(pFile) ); + if( ALWAYS(pInode) ){ + pInode->nRef--; + if( pInode->nRef==0 ){ + assert( pInode->pShmNode==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); + closePendingFds(pFile); + sqlite3_mutex_leave(pInode->pLockMutex); + if( pInode->pPrev ){ + assert( pInode->pPrev->pNext==pInode ); + pInode->pPrev->pNext = pInode->pNext; + }else{ + assert( inodeList==pInode ); + inodeList = pInode->pNext; + } + if( pInode->pNext ){ + assert( pInode->pNext->pPrev==pInode ); + pInode->pNext->pPrev = pInode->pPrev; + } + sqlite3_mutex_free(pInode->pLockMutex); + sqlite3_free(pInode); + } + } +} + +/* +** Given a file descriptor, locate the unixInodeInfo object that +** describes that file descriptor. Create a new one if necessary. The +** return value might be uninitialized if an error occurs. +** +** The global mutex must held when calling this routine. +** +** Return an appropriate error code. +*/ +static int findInodeInfo( + unixFile *pFile, /* Unix file with file desc used in the key */ + unixInodeInfo **ppInode /* Return the unixInodeInfo object here */ +){ + int rc; /* System call return code */ + int fd; /* The file descriptor for pFile */ + struct unixFileId fileId; /* Lookup key for the unixInodeInfo */ + struct stat statbuf; /* Low-level file information */ + unixInodeInfo *pInode = 0; /* Candidate unixInodeInfo object */ + + assert( unixMutexHeld() ); + + /* Get low-level information about the file that we can used to + ** create a unique name for the file. + */ + fd = pFile->h; + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + storeLastErrno(pFile, errno); +#if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; +#endif + return SQLITE_IOERR; + } + +#ifdef __APPLE__ + /* On OS X on an msdos filesystem, the inode number is reported + ** incorrectly for zero-size files. See ticket #3260. To work + ** around this problem (we consider it a bug in OS X, not SQLite) + ** we always increase the file size to 1 by writing a single byte + ** prior to accessing the inode number. The one byte written is + ** an ASCII 'S' character which also happens to be the first byte + ** in the header of every SQLite database. In this way, if there + ** is a race condition such that another thread has already populated + ** the first page of the database, no damage is done. + */ + if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ + do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + if( rc!=1 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR; + } + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR; + } + } +#endif + + memset(&fileId, 0, sizeof(fileId)); + fileId.dev = statbuf.st_dev; +#if OS_VXWORKS + fileId.pId = pFile->pId; +#else + fileId.ino = (u64)statbuf.st_ino; +#endif + assert( unixMutexHeld() ); + pInode = inodeList; + while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ + pInode = pInode->pNext; + } + if( pInode==0 ){ + pInode = sqlite3_malloc64( sizeof(*pInode) ); + if( pInode==0 ){ + return SQLITE_NOMEM_BKPT; + } + memset(pInode, 0, sizeof(*pInode)); + memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pInode->pLockMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pInode->pLockMutex==0 ){ + sqlite3_free(pInode); + return SQLITE_NOMEM_BKPT; + } + } + pInode->nRef = 1; + assert( unixMutexHeld() ); + pInode->pNext = inodeList; + pInode->pPrev = 0; + if( inodeList ) inodeList->pPrev = pInode; + inodeList = pInode; + }else{ + pInode->nRef++; + } + *ppInode = pInode; + return SQLITE_OK; +} + +/* +** Return TRUE if pFile has been renamed or unlinked since it was first opened. +*/ +static int fileHasMoved(unixFile *pFile){ +#if OS_VXWORKS + return pFile->pInode!=0 && pFile->pId!=pFile->pInode->fileId.pId; +#else + struct stat buf; + return pFile->pInode!=0 && + (osStat(pFile->zPath, &buf)!=0 + || (u64)buf.st_ino!=pFile->pInode->fileId.ino); +#endif +} + + +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + return; + } + if( buf.st_nlink==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + return; + } + if( fileHasMoved(pFile) ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + return; + } +} + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); + sqlite3_mutex_enter(pFile->pInode->pLockMutex); + + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ +#ifndef __DJGPP__ + if( !reserved && !pFile->pInode->bProcessLock ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = RESERVED_BYTE; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pFile->h, F_GETLK, &lock) ){ + rc = SQLITE_IOERR_CHECKRESERVEDLOCK; + storeLastErrno(pFile, errno); + } else if( lock.l_type!=F_UNLCK ){ + reserved = 1; + } + } +#endif + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); + OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* Forward declaration*/ +static int unixSleep(sqlite3_vfs*,int); + +/* +** Set a posix-advisory-lock. +** +** There are two versions of this routine. If compiled with +** SQLITE_ENABLE_SETLK_TIMEOUT then the routine has an extra parameter +** which is a pointer to a unixFile. If the unixFile->iBusyTimeout +** value is set, then it is the number of milliseconds to wait before +** failing the lock. The iBusyTimeout value is always reset back to +** zero on each call. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is not defined, then do a non-blocking +** attempt to set the lock. +*/ +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT +# define osSetPosixAdvisoryLock(h,x,t) osFcntl(h,F_SETLK,x) +#else +static int osSetPosixAdvisoryLock( + int h, /* The file descriptor on which to take the lock */ + struct flock *pLock, /* The description of the lock */ + unixFile *pFile /* Structure holding timeout value */ +){ + int tm = pFile->iBusyTimeout; + int rc = osFcntl(h,F_SETLK,pLock); + while( rc<0 && tm>0 ){ + /* On systems that support some kind of blocking file lock with a timeout, + ** make appropriate changes here to invoke that blocking file lock. On + ** generic posix, however, there is no such API. So we simply try the + ** lock once every millisecond until either the timeout expires, or until + ** the lock is obtained. */ + unixSleep(0,1000); + rc = osFcntl(h,F_SETLK,pLock); + tm--; + } + return rc; +} +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + + +/* +** Attempt to set a system-lock on the file pFile. The lock is +** described by pLock. +** +** If the pFile was opened read/write from unix-excl, then the only lock +** ever obtained is an exclusive lock, and it is obtained exactly once +** the first time any lock is attempted. All subsequent system locking +** operations become no-ops. Locking operations still happen internally, +** in order to coordinate access between separate database connections +** within this process, but all of that is handled in memory and the +** operating system does not participate. +** +** This function is a pass-through to fcntl(F_SETLK) if pFile is using +** any VFS other than "unix-excl" or if pFile is opened on "unix-excl" +** and is read-only. +** +** Zero is returned if the call completes successfully, or -1 if a call +** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). +*/ +static int unixFileLock(unixFile *pFile, struct flock *pLock){ + int rc; + unixInodeInfo *pInode = pFile->pInode; + assert( pInode!=0 ); + assert( sqlite3_mutex_held(pInode->pLockMutex) ); + if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ + if( pInode->bProcessLock==0 ){ + struct flock lock; + assert( pInode->nLock==0 ); + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + lock.l_type = F_WRLCK; + rc = osSetPosixAdvisoryLock(pFile->h, &lock, pFile); + if( rc<0 ) return rc; + pInode->bProcessLock = 1; + pInode->nLock++; + }else{ + rc = 0; + } + }else{ + rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int unixLock(sqlite3_file *id, int eFileLock){ + /* The following describes the implementation of the various locks and + ** lock transitions in terms of the POSIX advisory shared and exclusive + ** lock primitives (called read-locks and write-locks below, to avoid + ** confusion with SQLite lock names). The algorithms are complicated + ** slightly in order to be compatible with Windows95 systems simultaneously + ** accessing the same database file, in case that is ever required. + ** + ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** byte', each single bytes at well known offsets, and the 'shared byte + ** range', a range of 510 bytes at a well known offset. + ** + ** To obtain a SHARED lock, a read-lock is obtained on the 'pending + ** byte'. If this is successful, 'shared byte range' is read-locked + ** and the lock on the 'pending byte' released. (Legacy note: When + ** SQLite was first developed, Windows95 systems were still very common, + ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** single randomly selected by from the 'shared byte range' is locked. + ** Windows95 is now pretty much extinct, but this work-around for the + ** lack of shared-locks on Windows95 lives on, for backwards + ** compatibility.) + ** + ** A process may only obtain a RESERVED lock after it has a SHARED lock. + ** A RESERVED lock is implemented by grabbing a write-lock on the + ** 'reserved byte'. + ** + ** A process may only obtain a PENDING lock after it has obtained a + ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock + ** on the 'pending byte'. This ensures that no new SHARED locks can be + ** obtained, but existing SHARED locks are allowed to persist. A process + ** does not have to obtain a RESERVED lock on the way to a PENDING lock. + ** This property is used by the algorithm for rolling back a journal file + ** after a crash. + ** + ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is + ** implemented by obtaining a write-lock on the entire 'shared byte + ** range'. Since all other locks require a read-lock on one of the bytes + ** within this range, this ensures that no other locks are held on the + ** database. + */ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int tErrno = 0; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared, + osGetpid(0))); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct. + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto end_lock; + } + + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + lock.l_len = 1L; + lock.l_whence = SEEK_SET; + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocknShared==0 ); + assert( pInode->eFileLock==0 ); + assert( rc==SQLITE_OK ); + + /* Now get the read-lock */ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + } + + /* Drop the temporary PENDING lock */ + lock.l_start = PENDING_BYTE; + lock.l_len = 1L; + lock.l_type = F_UNLCK; + if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + /* This could happen with a network mount */ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + } + + if( rc ){ + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + goto end_lock; + }else{ + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + assert( 0!=pFile->eFileLock ); + lock.l_type = F_WRLCK; + + assert( eFileLock==RESERVED_LOCK || eFileLock==EXCLUSIVE_LOCK ); + if( eFileLock==RESERVED_LOCK ){ + lock.l_start = RESERVED_BYTE; + lock.l_len = 1L; + }else{ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + } + + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + } + } + + +#ifdef SQLITE_DEBUG + /* Set up the transaction-counter change checking flags when + ** transitioning from a SHARED to a RESERVED lock. The change + ** from SHARED to RESERVED marks the beginning of a normal + ** write operation (not a hot journal rollback). + */ + if( rc==SQLITE_OK + && pFile->eFileLock<=SHARED_LOCK + && eFileLock==RESERVED_LOCK + ){ + pFile->transCntrChng = 0; + pFile->dbUpdate = 0; + pFile->inNormalWrite = 1; + } +#endif + + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +end_lock: + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Add the file descriptor used by file handle pFile to the corresponding +** pUnused list. +*/ +static void setPendingFd(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p = pFile->pPreallocatedUnused; + assert( unixFileMutexHeld(pFile) ); + p->pNext = pInode->pUnused; + pInode->pUnused = p; + pFile->h = -1; + pFile->pPreallocatedUnused = 0; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED +** the byte range is divided into 2 parts and the first part is unlocked then +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** remove the write lock on a region when a read lock is set. +*/ +static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int rc = SQLITE_OK; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + osGetpid(0))); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + pFile->inNormalWrite = 0; +#endif + + /* downgrading to a shared lock on NFS involves clearing the write lock + ** before establishing the readlock - to avoid a race condition we downgrade + ** the lock in 2 blocks, so that part of the range will be covered by a + ** write lock until the rest is covered by a read lock: + ** 1: [WWWWW] + ** 2: [....W] + ** 3: [RRRRW] + ** 4: [RRRR.] + */ + if( eFileLock==SHARED_LOCK ){ +#if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE + (void)handleNFSUnlock; + assert( handleNFSUnlock==0 ); +#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + if( handleNFSUnlock ){ + int tErrno; /* Error code from system call errors */ + off_t divSize = SHARED_SIZE - 1; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + goto end_unlock; + } + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + goto end_unlock; + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST+divSize; + lock.l_len = SHARED_SIZE-divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + goto end_unlock; + } + }else +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + { + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + /* In theory, the call to unixFileLock() cannot fail because another + ** process is holding an incompatible lock. If it does, this + ** indicates that the other process is not following the locking + ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ + rc = SQLITE_IOERR_RDLOCK; + storeLastErrno(pFile, errno); + goto end_unlock; + } + } + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = PENDING_BYTE; + lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = SHARED_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, errno); + goto end_unlock; + } + } + if( eFileLock==NO_LOCK ){ + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + pInode->nShared--; + if( pInode->nShared==0 ){ + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = NO_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, errno); + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + + /* Decrement the count of locks against this same file. When the + ** count reaches zero, close any other file descriptors whose close + ** was deferred because of outstanding locks. + */ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ) closePendingFds(pFile); + } + +end_unlock: + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int unixUnlock(sqlite3_file *id, int eFileLock){ +#if SQLITE_MAX_MMAP_SIZE>0 + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); +#endif + return posixUnlock(id, eFileLock, 0); +} + +#if SQLITE_MAX_MMAP_SIZE>0 +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); +#endif + +/* +** This function performs the parts of the "close file" operation +** common to all locking schemes. It closes the directory and file +** handles, if they are valid, and sets all fields of the unixFile +** structure to 0. +** +** It is *not* necessary to hold the mutex when this routine is called, +** even on VxWorks. A mutex will be acquired on VxWorks by the +** vxworksReleaseFileId() routine. +*/ +static int closeUnixFile(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; +#if SQLITE_MAX_MMAP_SIZE>0 + unixUnmapfile(pFile); +#endif + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + pFile->h = -1; + } +#if OS_VXWORKS + if( pFile->pId ){ + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->pId->zCanonicalName); + } + vxworksReleaseFileId(pFile->pId); + pFile->pId = 0; + } +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->zPath); + sqlite3_free(*(char**)&pFile->zPath); + pFile->zPath = 0; + } +#endif + OSTRACE(("CLOSE %-3d\n", pFile->h)); + OpenCounter(-1); + sqlite3_free(pFile->pPreallocatedUnused); + memset(pFile, 0, sizeof(unixFile)); + return SQLITE_OK; +} + +/* +** Close a file. +*/ +static int unixClose(sqlite3_file *id){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile *)id; + unixInodeInfo *pInode = pFile->pInode; + + assert( pInode!=0 ); + verifyDbFile(pFile); + unixUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + + /* unixFile.pInode is always valid here. Otherwise, a different close + ** routine (e.g. nolockClose()) would be called instead. + */ + assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->pUnused list. It will be automatically closed + ** when the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); + releaseInodeInfo(pFile); + assert( pFile->pShm==0 ); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} + +/************** End of the posix advisory lock implementation ***************** +******************************************************************************/ + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ + UNUSED_PARAMETER(NotUsed); + *pResOut = 0; + return SQLITE_OK; +} +static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} +static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int nolockClose(sqlite3_file *id) { + return closeUnixFile(id); +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************* Begin dot-file Locking ****************************** +** +** The dotfile locking implementation uses the existence of separate lock +** files (really a directory) to control access to the database. This works +** on just about every filesystem imaginable. But there are serious downsides: +** +** (1) There is zero concurrency. A single reader blocks all other +** connections from reading or writing the database. +** +** (2) An application crash or power loss can leave stale lock files +** sitting around that need to be cleared manually. +** +** Nevertheless, a dotlock is an appropriate locking mode for use if no +** other locking strategy is available. +** +** Dotfile locking works by creating a subdirectory in the same directory as +** the database and with the same name but with a ".lock" extension added. +** The existence of a lock directory implies an EXCLUSIVE lock. All other +** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +*/ + +/* +** The file suffix added to the data base filename in order to create the +** lock directory. +*/ +#define DOTLOCK_SUFFIX ".lock" + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** +** In dotfile locking, either a lock exists or it does not. So in this +** variation of CheckReservedLock(), *pResOut is set to true if any lock +** is held on the file and false if the file is unlocked. +*/ +static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + reserved = osAccess((const char*)pFile->lockingContext, 0)==0; + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +** +** With dotfile locking, we really only support state (4): EXCLUSIVE. +** But we track the other locking levels internally. +*/ +static int dotlockLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc = SQLITE_OK; + + + /* If we have any lock, then the lock file already exists. All we have + ** to do is adjust our internal record of the lock level. + */ + if( pFile->eFileLock > NO_LOCK ){ + pFile->eFileLock = eFileLock; + /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else + utimes(zLockFile, NULL); +#endif + return SQLITE_OK; + } + + /* grab an exclusive lock */ + rc = osMkdir(zLockFile, 0777); + if( rc<0 ){ + /* failed to open/create the lock directory */ + int tErrno = errno; + if( EEXIST == tErrno ){ + rc = SQLITE_BUSY; + } else { + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + } + return rc; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** When the locking level reaches NO_LOCK, delete the lock file. +*/ +static int dotlockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* To downgrade to shared, simply update our internal notion of the + ** lock state. No need to mess with the file on disk. + */ + if( eFileLock==SHARED_LOCK ){ + pFile->eFileLock = SHARED_LOCK; + return SQLITE_OK; + } + + /* To fully unlock the database, delete the lock file */ + assert( eFileLock==NO_LOCK ); + rc = osRmdir(zLockFile); + if( rc<0 ){ + int tErrno = errno; + if( tErrno==ENOENT ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* +** Close a file. Make sure the lock has been released before closing. +*/ +static int dotlockClose(sqlite3_file *id) { + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + return closeUnixFile(id); +} +/****************** End of the dot-file lock implementation ******************* +******************************************************************************/ + +/****************************************************************************** +************************** Begin flock Locking ******************************** +** +** Use the flock() system call to do file locking. +** +** flock() locking is like dot-file locking in that the various +** fine-grain locking levels supported by SQLite are collapsed into +** a single exclusive lock. In other words, SHARED, RESERVED, and +** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite +** still works when you do this, but concurrency is reduced since +** only a single process can be reading the database at a time. +** +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off +*/ +#if SQLITE_ENABLE_LOCKING_STYLE + +/* +** Retry flock() calls that fail with EINTR +*/ +#ifdef EINTR +static int robust_flock(int fd, int op){ + int rc; + do{ rc = flock(fd,op); }while( rc<0 && errno==EINTR ); + return rc; +} +#else +# define robust_flock(a,b) flock(a,b) +#endif + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + /* attempt to get the lock */ + int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); + if( !lrc ){ + /* got the lock, unlock it */ + lrc = robust_flock(pFile->h, LOCK_UN); + if ( lrc ) { + int tErrno = errno; + /* unlock failed with an error */ + lrc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + rc = lrc; + } + } else { + int tErrno = errno; + reserved = 1; + /* someone else might have it reserved */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(lrc) ){ + storeLastErrno(pFile, tErrno); + rc = lrc; + } + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); + +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & 0xff) == SQLITE_IOERR ){ + rc = SQLITE_OK; + reserved=1; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** flock() only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int flockLock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* grab an exclusive lock */ + + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { + int tErrno = errno; + /* didn't get, must be busy */ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + } else { + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + } + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & 0xff) == SQLITE_IOERR ){ + rc = SQLITE_BUSY; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int flockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really, unlock. */ + if( robust_flock(pFile->h, LOCK_UN) ){ +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + return SQLITE_OK; +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return SQLITE_IOERR_UNLOCK; + }else{ + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; + } +} + +/* +** Close a file. +*/ +static int flockClose(sqlite3_file *id) { + assert( id!=0 ); + flockUnlock(id, NO_LOCK); + return closeUnixFile(id); +} + +#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ + +/******************* End of the flock lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************ Begin Named Semaphore Locking ************************ +** +** Named semaphore locking is only supported on VxWorks. +** +** Semaphore locking is like dot-lock and flock in that it really only +** supports EXCLUSIVE locking. Only a single process can read or write +** the database file at a time. This reduces potential concurrency, but +** makes the lock implementation much easier. +*/ +#if OS_VXWORKS + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + sem_t *pSem = pFile->pInode->pSem; + + if( sem_trywait(pSem)==-1 ){ + int tErrno = errno; + if( EAGAIN != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + storeLastErrno(pFile, tErrno); + } else { + /* someone else has the lock when we are in NO_LOCK */ + reserved = (pFile->eFileLock < SHARED_LOCK); + } + }else{ + /* we could have it if we want it */ + sem_post(pSem); + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** Semaphore locks only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int semXLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + int rc = SQLITE_OK; + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + rc = SQLITE_OK; + goto sem_end_lock; + } + + /* lock semaphore now but bail out when already locked. */ + if( sem_trywait(pSem)==-1 ){ + rc = SQLITE_BUSY; + goto sem_end_lock; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + + sem_end_lock: + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int semXUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + + assert( pFile ); + assert( pSem ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really unlock. */ + if ( sem_post(pSem)==-1 ) { + int rc, tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* + ** Close a file. + */ +static int semXClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + semXUnlock(id, NO_LOCK); + assert( pFile ); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + releaseInodeInfo(pFile); + unixLeaveMutex(); + closeUnixFile(id); + } + return SQLITE_OK; +} + +#endif /* OS_VXWORKS */ +/* +** Named semaphore locking is only available on VxWorks. +** +*************** End of the named semaphore lock implementation **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Begin AFP Locking ********************************* +** +** AFP is the Apple Filing Protocol. AFP is a network filesystem found +** on Apple Macintosh computers - both OS9 and OSX. +** +** Third-party implementations of AFP are available. But this code here +** only works on OSX. +*/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** The afpLockingContext structure contains all afp lock specific state +*/ +typedef struct afpLockingContext afpLockingContext; +struct afpLockingContext { + int reserved; + const char *dbPath; /* Name of the open file */ +}; + +struct ByteRangeLockPB2 +{ + unsigned long long offset; /* offset to first byte to lock */ + unsigned long long length; /* nbr of bytes to lock */ + unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ + unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ + unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ + int fd; /* file desc to assoc this lock with */ +}; + +#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) + +/* +** This is a utility for setting or clearing a bit-range lock on an +** AFP filesystem. +** +** Return SQLITE_OK on success, SQLITE_BUSY on failure. +*/ +static int afpSetLock( + const char *path, /* Name of the file to be locked or unlocked */ + unixFile *pFile, /* Open file descriptor on path */ + unsigned long long offset, /* First byte to be locked */ + unsigned long long length, /* Number of bytes to lock */ + int setLockFlag /* True to set lock. False to clear lock */ +){ + struct ByteRangeLockPB2 pb; + int err; + + pb.unLockFlag = setLockFlag ? 0 : 1; + pb.startEndFlag = 0; + pb.offset = offset; + pb.length = length; + pb.fd = pFile->h; + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), + offset, length)); + err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); + if ( err==-1 ) { + int rc; + int tErrno = errno; + OSTRACE(("AFPSETLOCK failed to fsctl() '%s' %d %s\n", + path, tErrno, strerror(tErrno))); +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + rc = SQLITE_BUSY; +#else + rc = sqliteErrorFromPosixError(tErrno, + setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); +#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + return rc; + } else { + return SQLITE_OK; + } +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + afpLockingContext *context; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + context = (afpLockingContext *) pFile->lockingContext; + if( context->reserved ){ + *pResOut = 1; + return SQLITE_OK; + } + sqlite3_mutex_enter(pFile->pInode->pLockMutex); + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ + if( !reserved ){ + /* lock the RESERVED byte */ + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( SQLITE_OK==lrc ){ + /* if we succeeded in taking the reserved lock, unlock it to restore + ** the original state */ + lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + } else { + /* if we failed to get the lock then someone else must have it */ + reserved = 1; + } + if( IS_LOCK_ERROR(lrc) ){ + rc=lrc; + } + } + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); + OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int afpLock(sqlite3_file *id, int eFileLock){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode = pFile->pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pInode->eFileLock), pInode->nShared , osGetpid(0))); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (afp)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto afp_end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto afp_end_lock; + } + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockdbPath, pFile, PENDING_BYTE, 1, 1); + if (failed) { + rc = failed; + goto afp_end_lock; + } + } + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + int lrc1, lrc2, lrc1Errno = 0; + long lk, mask; + + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; + /* Now get the read-lock SHARED_LOCK */ + /* note that the quality of the randomness doesn't matter that much */ + lk = random(); + pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); + lrc1 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST+pInode->sharedByte, 1, 1); + if( IS_LOCK_ERROR(lrc1) ){ + lrc1Errno = pFile->lastErrno; + } + /* Drop the temporary PENDING lock */ + lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + + if( IS_LOCK_ERROR(lrc1) ) { + storeLastErrno(pFile, lrc1Errno); + rc = lrc1; + goto afp_end_lock; + } else if( IS_LOCK_ERROR(lrc2) ){ + rc = lrc2; + goto afp_end_lock; + } else if( lrc1 != SQLITE_OK ) { + rc = lrc1; + } else { + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + int failed = 0; + assert( 0!=pFile->eFileLock ); + if (eFileLock >= RESERVED_LOCK && pFile->eFileLock < RESERVED_LOCK) { + /* Acquire a RESERVED lock */ + failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( !failed ){ + context->reserved = 1; + } + } + if (!failed && eFileLock == EXCLUSIVE_LOCK) { + /* Acquire an EXCLUSIVE lock */ + + /* Remove the shared lock before trying the range. we'll need to + ** reestablish the shared lock if we can't get the afpUnlock + */ + if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + + pInode->sharedByte, 1, 0)) ){ + int failed2 = SQLITE_OK; + /* now attemmpt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + SHARED_SIZE, 1); + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ + /* Can't reestablish the shared lock. Sqlite can't deal, this is + ** a critical I/O error + */ + rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : + SQLITE_IOERR_LOCK; + goto afp_end_lock; + } + }else{ + rc = failed; + } + } + if( failed ){ + rc = failed; + } + } + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +afp_end_lock: + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int afpUnlock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int skipShared = 0; +#ifdef SQLITE_TEST + int h = pFile->h; +#endif + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + osGetpid(0))); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif + + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ + /* only re-establish the shared lock if necessary */ + int sharedLockByte = SHARED_FIRST+pInode->sharedByte; + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); + } else { + skipShared = 1; + } + } + if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + } + if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ + rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( !rc ){ + context->reserved = 0; + } + } + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ + pInode->eFileLock = SHARED_LOCK; + } + } + if( rc==SQLITE_OK && eFileLock==NO_LOCK ){ + + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; + pInode->nShared--; + if( pInode->nShared==0 ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( !skipShared ){ + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); + } + if( !rc ){ + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + if( rc==SQLITE_OK ){ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ) closePendingFds(pFile); + } + } + + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } + return rc; +} + +/* +** Close a file & cleanup AFP specific locking context +*/ +static int afpClose(sqlite3_file *id) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + afpUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + if( pFile->pInode ){ + unixInodeInfo *pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); + } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the AFP lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. If you don't compile for a mac, then the "unix-afp" +** VFS is not available. +** +********************* End of the AFP lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +*************************** Begin NFS Locking ********************************/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* + ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock + ** must be either NO_LOCK or SHARED_LOCK. + ** + ** If the locking level of the file descriptor is already at or below + ** the requested locking level, this routine is a no-op. + */ +static int nfsUnlock(sqlite3_file *id, int eFileLock){ + return posixUnlock(id, eFileLock, 1); +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the NFS lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. +** +********************* End of the NFS lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +**************** Non-locking sqlite3_file methods ***************************** +** +** The next division contains implementations for all methods of the +** sqlite3_file object other than the locking methods. The locking +** methods were defined in divisions above (one locking method per +** division). Those methods that are common to all locking modes +** are gather together into this division. +*/ + +/* +** Seek to the offset passed as the second argument, then read cnt +** bytes into pBuf. Return the number of bytes actually read. +** +** NB: If you define USE_PREAD or USE_PREAD64, then it might also +** be necessary to define _XOPEN_SOURCE to be 500. This varies from +** one system to another. Since SQLite does not define USE_PREAD +** in any form by default, we will not attempt to define _XOPEN_SOURCE. +** See tickets #2741 and #2681. +** +** To avoid stomping the errno value on a failed read the lastErrno value +** is set before returning. +*/ +static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ + int got; + int prior = 0; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; +#endif + TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + assert( id->h>2 ); + do{ +#if defined(USE_PREAD) + got = osPread(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#elif defined(USE_PREAD64) + got = osPread64(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset = -1 ); + if( newOffset<0 ){ + storeLastErrno((unixFile*)id, errno); + return -1; + } + got = osRead(id->h, pBuf, cnt); +#endif + if( got==cnt ) break; + if( got<0 ){ + if( errno==EINTR ){ got = 1; continue; } + prior = 0; + storeLastErrno((unixFile*)id, errno); + break; + }else if( got>0 ){ + cnt -= got; + offset += got; + prior += got; + pBuf = (void*)(got + (char*)pBuf); + } + }while( got>0 ); + TIMER_END; + OSTRACE(("READ %-3d %5d %7lld %llu\n", + id->h, got+prior, offset-prior, TIMER_ELAPSED)); + return got+prior; +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int unixRead( + sqlite3_file *id, + void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile *)id; + int got; + assert( id ); + assert( offset>=0 ); + assert( amt>0 ); + + /* If this is a database file (not a journal, super-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pPreallocatedUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + got = seekAndRead(pFile, offset, pBuf, amt); + if( got==amt ){ + return SQLITE_OK; + }else if( got<0 ){ + /* pFile->lastErrno has been set by seekAndRead(). + ** Usually we return SQLITE_IOERR_READ here, though for some + ** kinds of errors we return SQLITE_IOERR_CORRUPTFS. The + ** SQLITE_IOERR_CORRUPTFS will be converted into SQLITE_CORRUPT + ** prior to returning to the application by the sqlite3ApiExit() + ** routine. + */ + // changed switch to if-else + if (pFile->lastErrno == ERANGE || pFile->lastErrno == EIO +#ifdef ENXIO + || pFile->lastErrno == ENXIO +#endif +#ifdef EDEVERR + || pFile->lastErrno == EDEVERR +#endif + ) + return SQLITE_IOERR_CORRUPTFS; + + return SQLITE_IOERR_READ; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } +} + +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + assert( fd>2 ); + assert( piErrno!=0 ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = (int)osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = (int)osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek = -1 ); + if( iSeek<0 ){ + rc = -1; + break; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 ) *piErrno = errno; + return rc; +} + + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); +} + + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int unixWrite( + sqlite3_file *id, + const void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile*)id; + int wrote = 0; + assert( id ); + assert( amt>0 ); + + /* If this is a database file (not a journal, super-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pPreallocatedUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) then record the fact that the database + ** has changed. If the transaction counter is modified, record that + ** fact too. + */ + if( pFile->inNormalWrite ){ + pFile->dbUpdate = 1; /* The database has been modified */ + if( offset<=24 && offset+amt>=27 ){ + int rc; + char oldCntr[4]; + SimulateIOErrorBenign(1); + rc = seekAndRead(pFile, 24, oldCntr, 4); + SimulateIOErrorBenign(0); + if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ + pFile->transCntrChng = 1; /* The transaction counter has changed */ + } + } + } +#endif + +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ + amt -= wrote; + offset += wrote; + pBuf = &((char*)pBuf)[wrote]; + } + SimulateIOError(( wrote=(-1), amt=1 )); + SimulateDiskfullError(( wrote=0, amt=1 )); + + if( amt>wrote ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ + /* lastErrno set by seekAndWrite */ + return SQLITE_IOERR_WRITE; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ + return SQLITE_FULL; + } + } + + return SQLITE_OK; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occurring at the right times. +*/ +int sqlite3_sync_count = 0; +int sqlite3_fullsync_count = 0; +#endif + +/* +** We do not trust systems to provide a working fdatasync(). Some do. +** Others do no. To be safe, we will stick with the (slightly slower) +** fsync(). If you know that your system does support fdatasync() correctly, +** then simply compile with -Dfdatasync=fdatasync or -DHAVE_FDATASYNC +*/ +#if !defined(fdatasync) && !HAVE_FDATASYNC +# define fdatasync fsync +#endif + +/* +** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not +** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently +** only available on Mac OS X. But that could change. +*/ +#ifdef F_FULLFSYNC +# define HAVE_FULLFSYNC 1 +#else +# define HAVE_FULLFSYNC 0 +#endif + + +/* +** The fsync() system call does not work as advertised on many +** unix systems. The following procedure is an attempt to make +** it work better. +** +** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful +** for testing when we want to run through the test suite quickly. +** You are strongly advised *not* to deploy with SQLITE_NO_SYNC +** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash +** or power failure will likely corrupt the database file. +** +** SQLite sets the dataOnly flag if the size of the file is unchanged. +** The idea behind dataOnly is that it should only write the file content +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, +** Ted Ts'o tells us that fdatasync() will also write the inode if the +** file size has changed. The only real difference between fdatasync() +** and fsync(), Ted tells us, is that fdatasync() will not flush the +** inode if the mtime or owner or other inode attributes have changed. +** We only care about the file size, not the other file attributes, so +** as far as SQLite is concerned, an fdatasync() is always adequate. +** So, we always use fdatasync() if it is available, regardless of +** the value of the dataOnly flag. +*/ +static int full_fsync(int fd, int fullSync, int dataOnly){ + int rc; + + /* The following "ifdef/elif/else/" block has the same structure as + ** the one below. It is replicated here solely to avoid cluttering + ** up the real code with the UNUSED_PARAMETER() macros. + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#elif HAVE_FULLFSYNC + UNUSED_PARAMETER(dataOnly); +#else + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#endif + + /* Record the number of times that we do a normal fsync() and + ** FULLSYNC. This is used during testing to verify that this procedure + ** gets called with the correct arguments. + */ +#ifdef SQLITE_TEST + if( fullSync ) sqlite3_fullsync_count++; + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op. But go ahead and call fstat() to validate the file + ** descriptor as we need a method to provoke a failure during + ** coverate testing. + */ +#ifdef SQLITE_NO_SYNC + { + struct stat buf; + rc = osFstat(fd, &buf); + } +#elif HAVE_FULLFSYNC + if( fullSync ){ + rc = osFcntl(fd, F_FULLFSYNC, 0); + }else{ + rc = 1; + } + /* If the FULLFSYNC failed, fall back to attempting an fsync(). + ** It shouldn't be possible for fullfsync to fail on the local + ** file system (on OSX), so failure indicates that FULLFSYNC + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid + ** the fcntl call every time sync is called. + */ + if( rc ) rc = fsync(fd); + +#elif defined(__APPLE__) + /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly + ** so currently we default to the macro that redefines fdatasync to fsync + */ + rc = fsync(fd); +#else + rc = fdatasync(fd); +#if OS_VXWORKS + if( rc==-1 && errno==ENOTSUP ){ + rc = fsync(fd); + } +#endif /* OS_VXWORKS */ +#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ + + if( OS_VXWORKS && rc!= -1 ){ + rc = 0; + } + return rc; +} + +/* +** Open a file descriptor to the directory containing file zFilename. +** If successful, *pFd is set to the opened file descriptor and +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined +** value. +** +** The directory file descriptor is used for only one thing - to +** fsync() a directory to make sure file creation and deletion events +** are flushed to disk. Such fsyncs are not needed on newer +** journaling filesystems, but are required on older filesystems. +** +** This routine can be overridden using the xSetSysCall interface. +** The ability to override this routine was added in support of the +** chromium sandbox. Opening a directory is a security risk (we are +** told) so making it overrideable allows the chromium sandbox to +** replace this routine with a harmless no-op. To make this routine +** a no-op, replace it with a stub that returns SQLITE_OK but leaves +** *pFd set to a negative number. +** +** If SQLITE_OK is returned, the caller is responsible for closing +** the file descriptor *pFd using close(). +*/ +static int openDirectory(const char *zFilename, int *pFd){ + int ii; + int fd = -1; + char zDirname[MAX_PATHNAME+1]; + + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); + for(ii=(int)strlen(zDirname); ii>0 && zDirname[ii]!='/'; ii--); + if( ii>0 ){ + zDirname[ii] = '\0'; + }else{ + if( zDirname[0]!='/' ) zDirname[0] = '.'; + zDirname[1] = 0; + } + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + } + *pFd = fd; + if( fd>=0 ) return SQLITE_OK; + return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname); +} + +/* +** Make sure all writes to a particular file are committed to disk. +** +** If dataOnly==0 then both the file itself and its metadata (file +** size, access time, etc) are synced. If dataOnly!=0 then only the +** file data is synced. +** +** Under Unix, also make sure that the directory entry for the file +** has been created by fsync-ing the directory that contains the file. +** If we do not do this and we encounter a power failure, the directory +** entry for the journal might not exist after we reboot. The next +** SQLite to access the file will not know that the journal exists (because +** the directory entry for the journal was never created) and the transaction +** will not roll back - possibly leading to database corruption. +*/ +static int unixSync(sqlite3_file *id, int flags){ + int rc; + unixFile *pFile = (unixFile*)id; + + int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); + int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; + + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + assert( pFile ); + OSTRACE(("SYNC %-3d\n", pFile->h)); + rc = full_fsync(pFile->h, isFullsync, isDataOnly); + SimulateIOError( rc=1 ); + if( rc ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); + } + + /* Also fsync the directory containing the file if the DIRSYNC flag + ** is set. This is a one-time occurrence. Many systems (examples: AIX) + ** are unable to fsync a directory, so ignore errors on the fsync. + */ + if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ + int dirfd; + OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, + HAVE_FULLFSYNC, isFullsync)); + rc = osOpenDirectory(pFile->zPath, &dirfd); + if( rc==SQLITE_OK ){ + full_fsync(dirfd, 0, 0); + robust_close(pFile, dirfd, __LINE__); + }else{ + assert( rc==SQLITE_CANTOPEN ); + rc = SQLITE_OK; + } + pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; + } + return rc; +} + +/* +** Truncate an open file to a specified size +*/ +static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; + int rc; + assert( pFile ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = robust_ftruncate(pFile->h, nByte); + if( rc ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + }else{ +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) and we truncate the file to zero length, + ** that effectively updates the change counter. This might happen + ** when restoring a database using the backup API from a zero-length + ** source. + */ + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; + } +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + return SQLITE_OK; + } +} + +/* +** Determine the current size of a file in bytes +*/ +static int unixFileSize(sqlite3_file *id, i64 *pSize){ + int rc; + struct stat buf; + assert( id ); + rc = osFstat(((unixFile*)id)->h, &buf); + SimulateIOError( rc=1 ); + if( rc!=0 ){ + storeLastErrno((unixFile*)id, errno); + return SQLITE_IOERR_FSTAT; + } + *pSize = buf.st_size; + + /* When opening a zero-size database, the findInodeInfo() procedure + ** writes a single byte into that file in order to work around a bug + ** in the OS-X msdos filesystem. In order to avoid problems with upper + ** layers, we need to report this file size as zero even though it is + ** really 1. Ticket #3260. + */ + if( *pSize==1 ) *pSize = 0; + + + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Handler for proxy-locking file-control verbs. Defined below in the +** proxying locking division. +*/ +static int proxyFileControl(sqlite3_file*,int,void*); +#endif + +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. Enlarge the database to nBytes in size +** (rounded up to the next chunk-size). If the database is already +** nBytes or larger, this routine is a no-op. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk>0 ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( osFstat(pFile->h, &buf) ){ + return SQLITE_IOERR_FSTAT; + } + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, + ** or an error number on failure". See the manpage for details. */ + int err; + do{ + err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); + }while( err==EINTR ); + if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; +#else + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + int nWrite = 0; /* Number of bytes written by seekAndWrite */ + i64 iWrite; /* Next offset to write to */ + + iWrite = (buf.st_size/nBlk)*nBlk + nBlk - 1; + assert( iWrite>=buf.st_size ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; + nWrite = seekAndWrite(pFile, iWrite, "", 1); + if( nWrite!=1 ) return SQLITE_IOERR_WRITE; + } +#endif + } + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } +#endif + + return SQLITE_OK; +} + +/* +** If *pArg is initially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward declaration */ +static int unixGetTempname(int nBuf, char *zBuf); + +/* +** Information and control of an open file handle. +*/ +static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; + switch( op ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); + return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->eFileLock; + return SQLITE_OK; + } + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = pFile->lastErrno; + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + int rc; + SimulateIOErrorBenign(1); + rc = fcntlSizeHint(pFile, *(i64 *)pArg); + SimulateIOErrorBenign(0); + return rc; + } + case SQLITE_FCNTL_PERSIST_WAL: { + unixModeBit(pFile, UNIXFILE_PERSIST_WAL, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + unixModeBit(pFile, UNIXFILE_PSOW, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + return SQLITE_OK; + } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3_malloc64( pFile->pVfs->mxPathname ); + if( zTFile ){ + unixGetTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_HAS_MOVED: { + *(int*)pArg = fileHasMoved(pFile); + return SQLITE_OK; + } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; + pFile->iBusyTimeout = *(int*)pArg; + *(int*)pArg = iOld; + return SQLITE_OK; + } +#endif +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + + /* The value of newLimit may be eventually cast to (size_t) and passed + ** to mmap(). Restrict its value to 2GB if (size_t) is not at least a + ** 64-bit type. */ + if( newLimit>0 && sizeof(size_t)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + unixUnmapfile(pFile); + rc = unixMapfile(pFile, -1); + } + } + return rc; + } +#endif +#ifdef SQLITE_DEBUG + /* The pager calls this method to signal that it has done + ** a rollback and that the database is therefore unchanged and + ** it hence it is OK for the transaction change counter to be + ** unchanged. + */ + case SQLITE_FCNTL_DB_UNCHANGED: { + ((unixFile*)id)->dbUpdate = 0; + return SQLITE_OK; + } +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { + return proxyFileControl(id,op,pArg); + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + } + return SQLITE_NOTFOUND; +} + +/* +** If pFd->sectorSize is non-zero when this function is called, it is a +** no-op. Otherwise, the values of pFd->sectorSize and +** pFd->deviceCharacteristics are set according to the file-system +** characteristics. +** +** There are two versions of this function. One for QNX and one for all +** other systems. +*/ +#ifndef __QNXNTO__ +static void setDeviceCharacteristics(unixFile *pFd){ + assert( pFd->deviceCharacteristics==0 || pFd->sectorSize!=0 ); + if( pFd->sectorSize==0 ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int res; + u32 f = 0; + + /* Check for support for F2FS atomic batch writes. */ + res = osIoctl(pFd->h, F2FS_IOC_GET_FEATURES, &f); + if( res==0 && (f & F2FS_FEATURE_ATOMIC_WRITE) ){ + pFd->deviceCharacteristics = SQLITE_IOCAP_BATCH_ATOMIC; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + /* Set the POWERSAFE_OVERWRITE flag if requested. */ + if( pFd->ctrlFlags & UNIXFILE_PSOW ){ + pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; + } + + pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#else +#include +#include +static void setDeviceCharacteristics(unixFile *pFile){ + if( pFile->sectorSize == 0 ){ + struct statvfs fsInfo; + + /* Set defaults for non-supported filesystems */ + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + pFile->deviceCharacteristics = 0; + if( fstatvfs(pFile->h, &fsInfo) == -1 ) { + return; + } + + if( !strcmp(fsInfo.f_basetype, "tmp") ) { + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC4K | /* All ram filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "etfs") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* etfs cluster size writes are atomic */ + (pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) | + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx6") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC | /* All filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx4") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "dos") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else{ + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC512 | /* blocks are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + 0; + } + } + /* Last chance verification. If the sector size isn't a multiple of 512 + ** then it isn't valid.*/ + if( pFile->sectorSize % 512 != 0 ){ + pFile->deviceCharacteristics = 0; + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#endif + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->sectorSize; +} + +/* +** Return the device characteristics for the file. +** +** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. +** However, that choice is controversial since technically the underlying +** file system does not always provide powersafe overwrites. (In other +** words, after a power-loss event, parts of the file that were never +** written might end up being altered.) However, non-PSOW behavior is very, +** very rare. And asserting PSOW makes a large reduction in the amount +** of required I/O for journaling, since a lot of padding is eliminated. +** Hence, while POWERSAFE_OVERWRITE is on by default, there is a file-control +** available to turn it off and URI query parameter available to turn it off. +*/ +static int unixDeviceCharacteristics(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->deviceCharacteristics; +} + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + +/* +** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). +*/ +static int unixGetpagesize(void){ +#if OS_VXWORKS + return 1024; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +#endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ + +#ifndef SQLITE_OMIT_WAL + +/* +** Object used to represent an shared memory buffer. +** +** When multiple threads all reference the same wal-index, each thread +** has its own unixShm object, but they all point to a single instance +** of this unixShmNode object. In other words, each wal-index is opened +** only once per process. +** +** Each unixShmNode object is connected to a single unixInodeInfo object. +** We could coalesce this object into unixInodeInfo, but that would mean +** every open file that does not use shared memory (in other words, most +** open files) would have to carry around this extra information. So +** the unixInodeInfo object contains a pointer to this unixShmNode object +** and the unixShmNode object is created only when needed. +** +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** +** The following fields are read-only after the object is created: +** +** hShm +** zFilename +** +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. +*/ +struct unixShmNode { + unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ + char *zFilename; /* Name of the mmapped file */ + int hShm; /* Open file descriptor */ + int szRegion; /* Size of shared-memory regions */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + char **apRegion; /* Array of mapped shared-memory regions */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ + int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif +}; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** unixShm.pShmNode +** unixShm.id +** +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. +*/ +struct unixShm { + unixShmNode *pShmNode; /* The underlying unixShmNode object */ + unixShm *pNext; /* Next unixShm with the same unixShmNode */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ + u8 id; /* Id of this connection within its unixShmNode */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +}; + +/* +** Constants used for locking +*/ +#define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Apply posix advisory locks for all bytes from ofst through ofst+n-1. +** +** Locks block if the mask is exactly UNIX_SHM_C and are non-blocking +** otherwise. +*/ +static int unixShmSystemLock( + unixFile *pFile, /* Open connection to the WAL file */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + int ofst, /* First byte of the locking range */ + int n /* Number of bytes to lock */ +){ + unixShmNode *pShmNode; /* Apply locks to this open shared-memory segment */ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ + + /* Access to the unixShmNode object is serialized by the caller */ + pShmNode = pFile->pInode->pShmNode; + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 || unixMutexHeld() ); + + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); + + /* Locks are within range */ + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + + if( pShmNode->hShm>=0 ){ + int res; + /* Initialize the locking parameters */ + f.l_type = lockType; + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = n; + res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); + if( res==-1 ){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); +#else + rc = SQLITE_BUSY; +#endif + } + } + + /* Update the global lock state and do debug tracing */ +#ifdef SQLITE_DEBUG + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); + } +#endif + + return rc; +} + +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = osGetpagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; + assert( unixMutexHeld() ); + if( p && ALWAYS(p->nRef==0) ){ + int nShmPerMap = unixShmRegionPerMap(); + int i; + assert( p->pInode==pFd->pInode ); + sqlite3_mutex_free(p->pShmMutex); + for(i=0; inRegion; i+=nShmPerMap){ + if( p->hShm>=0 ){ + osMunmap(p->apRegion[i], p->szRegion); + }else{ + sqlite3_free(p->apRegion[i]); + } + } + sqlite3_free(p->apRegion); + if( p->hShm>=0 ){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; + } + p->pInode->pShmNode = 0; + sqlite3_free(p); + } +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->hShm, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to + ** help detect if a -shm file truncation is legitimate or is the work + ** or a rogue process. */ + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} + +/* +** Open a shared-memory area associated with open database file pDbFd. +** This particular implementation uses mmapped files. +** +** The file used to implement shared-memory is in the same directory +** as the open database file and has the same name as the open database +** file with the "-shm" suffix added. For example, if the database file +** is "/home/user1/config.db" then the file that is created and mmapped +** for shared memory will be called "/home/user1/config.db-shm". +** +** Another approach to is to use files in /dev/shm or /dev/tmp or an +** some other tmpfs mount. But if a file in a different directory +** from the database file is used, then differing access permissions +** or a chroot() might cause two different processes on the same +** database to end up using different files for shared memory - +** meaning that their memory would not really be shared - resulting +** in database corruption. Nevertheless, this tmpfs file usage +** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" +** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time +** option results in an incompatible build of SQLite; builds of SQLite +** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the +** same database file at the same time, database corruption will likely +** result. The SQLITE_SHM_DIRECTORY compile-time option is considered +** "unsupported" and may go away in a future SQLite release. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +** +** If the original database file (pDbFd) is using the "unix-excl" VFS +** that means that an exclusive lock is held on the database file and +** that no other processes are able to read or write the database. In +** that case, we do not really need shared memory. No shared memory +** file is created. The shared memory will be simulated with heap memory. +*/ +static int unixOpenSharedMemory(unixFile *pDbFd){ + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + unixInodeInfo *pInode; /* The inode of fd */ + char *zShm; /* Name of the file used for SHM */ + int nShmFilename; /* Size of the SHM filename in bytes */ + + /* Allocate space for the new unixShm object. */ + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM_BKPT; + memset(p, 0, sizeof(*p)); + assert( pDbFd->pShm==0 ); + + /* Check to see if a unixShmNode object already exists. Reuse an existing + ** one if present. Create a new one if necessary. + */ + assert( unixFileMutexNotheld(pDbFd) ); + unixEnterMutex(); + pInode = pDbFd->pInode; + pShmNode = pInode->pShmNode; + if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ +#ifndef SQLITE_SHM_DIRECTORY + const char *zBasePath = pDbFd->zPath; +#endif + + /* Call fstat() to figure out the permissions on the database file. If + ** a new *-shm file is created, an attempt will be made to create it + ** with the same permissions. + */ + if( osFstat(pDbFd->h, &sStat) ){ + rc = SQLITE_IOERR_FSTAT; + goto shm_open_err; + } + +#ifdef SQLITE_SHM_DIRECTORY + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; +#else + nShmFilename = 6 + (int)strlen(zBasePath); +#endif + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); + if( pShmNode==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; +#ifdef SQLITE_SHM_DIRECTORY + sqlite3_snprintf(nShmFilename, zShm, + SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", + (u32)sStat.st_ino, (u32)sStat.st_dev); +#else + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); +#endif + pShmNode->hShm = -1; + pDbFd->pInode->pShmNode = pShmNode; + pShmNode->pInode = pDbFd->pInode; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->pShmMutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + } + + if( pInode->bProcessLock==0 ){ + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, + (sStat.st_mode&0777)); + } + if( pShmNode->hShm<0 ){ + pShmNode->hShm = robust_open(zShm, O_RDONLY|O_NOFOLLOW, + (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; + } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. + */ + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); + + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + } + } + + /* Make the new connection a child of the unixShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the unixEnterMutex() mutex and the pointer from the + ** new (struct unixShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the + ** pShmNode->pShmMutex. + */ + sqlite3_mutex_enter(pShmNode->pShmMutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->pShmMutex); + return rc; + + /* Jump here on any error */ +shm_open_err: + unixShmPurge(pDbFd); /* This call frees pShmNode if required */ + sqlite3_free(p); + unixLeaveMutex(); + return rc; +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + unixFile *pDbFd = (unixFile*)fd; + unixShm *p; + unixShmNode *pShmNode; + int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; + + /* If the shared-memory file has not yet been opened, open it now. */ + if( pDbFd->pShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + } + + p = pDbFd->pShm; + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; + + if( pShmNode->hShm>=0 ){ + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + if( osFstat(pShmNode->hShm, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + + if( sStat.st_sizehShm, iPg*pgsz + pgsz-1,"",1,&x)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } + } + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, nReqRegion*sizeof(char *) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shmpage_out; + } + pShmNode->apRegion = apNew; + while( pShmNode->nRegionhShm>=0 ){ + pMem = osMmap(0, nMap, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion + ); + if( pMem==MAP_FAILED ){ + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); + goto shmpage_out; + } + }else{ + pMem = sqlite3_malloc64(nMap); + if( pMem==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shmpage_out; + } + memset(pMem, 0, nMap); + } + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->pShmMutex); + return rc; +} + +/* +** Check that the pShmNode->aLock[] array comports with the locking bitmasks +** held by each client. Return true if it does, or false otherwise. This +** is to be used in an assert(). e.g. +** +** assert( assertLockingArrayOk(pShmNode) ); +*/ +#ifdef SQLITE_DEBUG +static int assertLockingArrayOk(unixShmNode *pShmNode){ + unixShm *pX; + int aLock[SQLITE_SHM_NLOCK]; + assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); + + memset(aLock, 0, sizeof(aLock)); + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + int i; + for(i=0; iexclMask & (1<sharedMask & (1<=0 ); + aLock[i]++; + } + } + } + + assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) ); + return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); +} +#endif + +/* +** Change the lock state for a shared-memory segment. +** +** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** different here than in posix. In xShmLock(), one can go from unlocked +** to shared and back or from unlocked to exclusive and back. But one may +** not go from shared to exclusive or from exclusive to shared. +*/ +static int unixShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ + unixShm *p = pDbFd->pShm; /* The shared memory being locked */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + int *aLock = pShmNode->aLock; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Checkpointer lock (ofst==1). + ** 2. Write lock (ofst==0). + ** 3. Read locks (ofst>=3 && ofstiBusyTimeout==0 || ( + (ofst!=2) /* not RECOVER */ + && (ofst!=1 || (p->exclMask|p->sharedMask)==0) + && (ofst!=0 || (p->exclMask|p->sharedMask)<3) + && (ofst<3 || (p->exclMask|p->sharedMask)<(1<1 || mask==(1<pShmMutex); + assert( assertLockingArrayOk(pShmNode) ); + if( flags & SQLITE_SHM_UNLOCK ){ + if( (p->exclMask|p->sharedMask) & mask ){ + int ii; + int bUnlock = 1; + + for(ii=ofst; ii((p->sharedMask & (1<sharedMask & (1<1 ); + aLock[ofst]--; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + } + }else if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( (p->exclMask & (1<sharedMask & mask)==0 ){ + if( aLock[ofst]<0 ){ + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + aLock[ofst]++; + } + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ + int ii; + for(ii=ofst; iisharedMask & mask)==0 ); + if( ALWAYS((p->exclMask & (1<sharedMask & mask)==0 ); + p->exclMask |= mask; + for(ii=ofst; iipShmMutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", + p->id, osGetpid(0), p->sharedMask, p->exclMask)); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void unixShmBarrier( + sqlite3_file *fd /* Database file holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) + ); + unixEnterMutex(); /* Also mutex, for redundancy */ + unixLeaveMutex(); +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +** +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. +*/ +static int unixShmUnmap( + sqlite3_file *fd, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ +){ + unixShm *p; /* The connection to be closed */ + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ + + pDbFd = (unixFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->pShmMutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->pShmMutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + assert( unixFileMutexNotheld(pDbFd) ); + unixEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + if( deleteFlag && pShmNode->hShm>=0 ){ + osUnlink(pShmNode->zFilename); + } + unixShmPurge(pDbFd); + } + unixLeaveMutex(); + + return SQLITE_OK; +} + + +#else +# define unixShmMap 0 +# define unixShmLock 0 +# define unixShmBarrier 0 +# define unixShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + +#ifdef SQLITE_MMAP_READWRITE + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; +#endif + + if( pOrig ){ +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else + const int szSyspage = osGetpagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nMap){ + assert( nMap>=0 || pFd->nFetchOut==0 ); + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + if( osFstat(pFd->h, &statbuf) ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); + if( nMap!=pFd->mmapSize ){ + unixRemapfile(pFd, nMap); + } + + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#else + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(iOff); +#endif + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This division contains definitions of sqlite3_io_methods objects that +** implement various file locking strategies. It also contains definitions +** of "finder" functions. A finder-function is used to locate the appropriate +** sqlite3_io_methods object for a particular database file. The pAppData +** field of the sqlite3_vfs VFS objects are initialized to be pointers to +** the correct finder-function for that VFS. +** +** Most finder functions return a pointer to a fixed sqlite3_io_methods +** object. The only interesting finder-function is autolockIoFinder, which +** looks at the filesystem type and tries to guess the best locking +** strategy from that. +** +** For finder-function F, two objects are created: +** +** (1) The real finder-function named "FImpt()". +** +** (2) A constant pointer to this function named just "F". +** +** +** A pointer to the F pointer is used as the pAppData value for VFS +** objects. We have to do this instead of letting pAppData point +** directly at the finder-function since C90 rules prevent a void* +** from be cast into a function pointer. +** +** +** Each instance of this macro generates two objects: +** +** * A constant sqlite3_io_methods object call METHOD that has locking +** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** +** * An I/O method finder function called FINDER that returns a pointer +** to the METHOD object in the previous bullet. +*/ +#define IOMETHODS(FINDER,METHOD,VERSION,CLOSE,LOCK,UNLOCK,CKLOCK,SHMMAP) \ +static const sqlite3_io_methods METHOD = { \ + VERSION, /* iVersion */ \ + CLOSE, /* xClose */ \ + unixRead, /* xRead */ \ + unixWrite, /* xWrite */ \ + unixTruncate, /* xTruncate */ \ + unixSync, /* xSync */ \ + unixFileSize, /* xFileSize */ \ + LOCK, /* xLock */ \ + UNLOCK, /* xUnlock */ \ + CKLOCK, /* xCheckReservedLock */ \ + unixFileControl, /* xFileControl */ \ + unixSectorSize, /* xSectorSize */ \ + unixDeviceCharacteristics, /* xDeviceCapabilities */ \ + SHMMAP, /* xShmMap */ \ + unixShmLock, /* xShmLock */ \ + unixShmBarrier, /* xShmBarrier */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ +}; \ +static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ + UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ + return &METHOD; \ +} \ +static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ + = FINDER##Impl; + +/* +** Here are all of the sqlite3_io_methods objects for each of the +** locking strategies. Functions that return pointers to these methods +** are also created. +*/ +IOMETHODS( + posixIoFinder, /* Finder function name */ + posixIoMethods, /* sqlite3_io_methods object name */ + 3, /* shared memory and mmap are enabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + unixUnlock, /* xUnlock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ +) +IOMETHODS( + nolockIoFinder, /* Finder function name */ + nolockIoMethods, /* sqlite3_io_methods object name */ + 3, /* shared memory and mmap are enabled */ + nolockClose, /* xClose method */ + nolockLock, /* xLock method */ + nolockUnlock, /* xUnlock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +IOMETHODS( + dotlockIoFinder, /* Finder function name */ + dotlockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + dotlockClose, /* xClose method */ + dotlockLock, /* xLock method */ + dotlockUnlock, /* xUnlock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) + +#if SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + flockIoFinder, /* Finder function name */ + flockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + flockClose, /* xClose method */ + flockLock, /* xLock method */ + flockUnlock, /* xUnlock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if OS_VXWORKS +IOMETHODS( + semIoFinder, /* Finder function name */ + semIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + semXClose, /* xClose method */ + semXLock, /* xLock method */ + semXUnlock, /* xUnlock method */ + semXCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + afpIoFinder, /* Finder function name */ + afpIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + afpClose, /* xClose method */ + afpLock, /* xLock method */ + afpUnlock, /* xUnlock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +/* +** The proxy locking method is a "super-method" in the sense that it +** opens secondary file descriptors for the conch and lock files and +** it uses proxy, dot-file, AFP, and flock() locking methods on those +** secondary files. For this reason, the division that implements +** proxy locking is located much further down in the file. But we need +** to go ahead and define the sqlite3_io_methods and finder function +** for proxy locking here. So we forward declare the I/O methods. +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int proxyClose(sqlite3_file*); +static int proxyLock(sqlite3_file*, int); +static int proxyUnlock(sqlite3_file*, int); +static int proxyCheckReservedLock(sqlite3_file*, int*); +IOMETHODS( + proxyIoFinder, /* Finder function name */ + proxyIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + proxyClose, /* xClose method */ + proxyLock, /* xLock method */ + proxyUnlock, /* xUnlock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + nfsIoFinder, /* Finder function name */ + nfsIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + nfsUnlock, /* xUnlock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for MacOSX only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* open file object for the database file */ +){ + static const struct Mapping { + const char *zFilesystem; /* Filesystem type name */ + const sqlite3_io_methods *pMethods; /* Appropriate locking method */ + } aMap[] = { + { "hfs", &posixIoMethods }, + { "ufs", &posixIoMethods }, + { "afpfs", &afpIoMethods }, + { "smbfs", &afpIoMethods }, + { "webdav", &nolockIoMethods }, + { 0, 0 } + }; + int i; + struct statfs fsInfo; + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + if( statfs(filePath, &fsInfo) != -1 ){ + if( fsInfo.f_flags & MNT_RDONLY ){ + return &nolockIoMethods; + } + for(i=0; aMap[i].zFilesystem; i++){ + if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ + return aMap[i].pMethods; + } + } + } + + /* Default case. Handles, amongst others, "nfs". + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ + return &nfsIoMethods; + } else { + return &posixIoMethods; + } + }else{ + return &dotlockIoMethods; + } +} +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + +#if OS_VXWORKS +/* +** This "finder" function for VxWorks checks to see if posix advisory +** locking works. If it does, then that is what is used. If it does not +** work, then fallback to named semaphore locking. +*/ +static const sqlite3_io_methods *vxworksIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* the open file object */ +){ + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + + /* Test if fcntl() is supported and use POSIX style locks. + ** Otherwise fall back to the named semaphore method. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + return &posixIoMethods; + }else{ + return &semIoMethods; + } +} +static const sqlite3_io_methods + *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; + +#endif /* OS_VXWORKS */ + +/* +** An abstract type for a pointer to an IO method finder function: +*/ +typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); + + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +/* +** Initialize the contents of the unixFile structure pointed to by pId. +*/ +static int fillInUnixFile( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + int h, /* Open file descriptor of file being opened */ + sqlite3_file *pId, /* Write to the unixFile structure here */ + const char *zFilename, /* Name of the file being opened */ + int ctrlFlags /* Zero or more UNIXFILE_* values */ +){ + const sqlite3_io_methods *pLockingStyle; + unixFile *pNew = (unixFile *)pId; + int rc = SQLITE_OK; + + assert( pNew->pInode==NULL ); + + /* No locking occurs in temporary files */ + assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 ); + + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); + pNew->h = h; + pNew->pVfs = pVfs; + pNew->zPath = zFilename; + pNew->ctrlFlags = (u8)ctrlFlags; +#if SQLITE_MAX_MMAP_SIZE>0 + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), + "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pNew->ctrlFlags |= UNIXFILE_PSOW; + } + if( strcmp(pVfs->zName,"unix-excl")==0 ){ + pNew->ctrlFlags |= UNIXFILE_EXCL; + } + +#if OS_VXWORKS + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + ctrlFlags |= UNIXFILE_NOLOCK; + rc = SQLITE_NOMEM_BKPT; + } +#endif + + if( ctrlFlags & UNIXFILE_NOLOCK ){ + pLockingStyle = &nolockIoMethods; + }else{ + pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); +#if SQLITE_ENABLE_LOCKING_STYLE + /* Cache zFilename in the locking context (AFP and dotlock override) for + ** proxyLock activation is possible (remote proxy is based on db name) + ** zFilename remains valid until file is closed, to support */ + pNew->lockingContext = (void*)zFilename; +#endif + } + + if( pLockingStyle == &posixIoMethods +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + || pLockingStyle == &nfsIoMethods +#endif + ){ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + /* If an error occurred in findInodeInfo(), close the file descriptor + ** immediately, before releasing the mutex. findInodeInfo() may fail + ** in two scenarios: + ** + ** (a) A call to fstat() failed. + ** (b) A malloc failed. + ** + ** Scenario (b) may only occur if the process is holding no other + ** file descriptors open on the same file. If there were other file + ** descriptors on this file, then no malloc would be required by + ** findInodeInfo(). If this is the case, it is quite safe to close + ** handle h - as it is guaranteed that no posix locks will be released + ** by doing so. + ** + ** If scenario (a) caused the error then things are not so safe. The + ** implicit assumption here is that if fstat() fails, things are in + ** such bad shape that dropping a lock or two doesn't matter much. + */ + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + else if( pLockingStyle == &afpIoMethods ){ + /* AFP locking uses the file path so it needs to be included in + ** the afpLockingContext. + */ + afpLockingContext *pCtx; + pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); + if( pCtx==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + /* NB: zFilename exists and remains valid until the file is closed + ** according to requirement F11141. So we do not need to make a + ** copy of the filename. */ + pCtx->dbPath = zFilename; + pCtx->reserved = 0; + srandomdev(); + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew->lockingContext); + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + } +#endif + + else if( pLockingStyle == &dotlockIoMethods ){ + /* Dotfile locking uses the file path so it needs to be included in + ** the dotlockLockingContext + */ + char *zLockFile; + int nFilename; + assert( zFilename!=0 ); + nFilename = (int)strlen(zFilename) + 6; + zLockFile = (char *)sqlite3_malloc64(nFilename); + if( zLockFile==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + } + pNew->lockingContext = zLockFile; + } + +#if OS_VXWORKS + else if( pLockingStyle == &semIoMethods ){ + /* Named semaphore locking uses the file path so it needs to be + ** included in the semLockingContext + */ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( (rc==SQLITE_OK) && (pNew->pInode->pSem==NULL) ){ + char *zSemName = pNew->pInode->aSemName; + int n; + sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", + pNew->pId->zCanonicalName); + for( n=1; zSemName[n]; n++ ) + if( zSemName[n]=='/' ) zSemName[n] = '_'; + pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); + if( pNew->pInode->pSem == SEM_FAILED ){ + rc = SQLITE_NOMEM_BKPT; + pNew->pInode->aSemName[0] = '\0'; + } + } + unixLeaveMutex(); + } +#endif + + storeLastErrno(pNew, 0); +#if OS_VXWORKS + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + h = -1; + osUnlink(zFilename); + pNew->ctrlFlags |= UNIXFILE_DELETE; + } +#endif + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + }else{ + pId->pMethods = pLockingStyle; + OpenCounter(+1); + verifyDbFile(pNew); + } + return rc; +} + +/* +** Return the name of a directory in which to put temporary files. +** If no suitable temporary file directory can be found, return NULL. +*/ +static const char *unixTempFileDir(void){ + static const char *azDirs[] = { + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + "." + }; + unsigned int i = 0; + struct stat buf; + const char *zDir = sqlite3_temp_directory; + + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + while(1){ + if( zDir!=0 + && osStat(zDir, &buf)==0 + && S_ISDIR(buf.st_mode) + && osAccess(zDir, 03)==0 + ){ + return zDir; + } + if( i>=sizeof(azDirs)/sizeof(azDirs[0]) ) break; + zDir = azDirs[i++]; + } + return 0; +} + +/* +** Create a temporary file name in zBuf. zBuf must be allocated +** by the calling process and must be big enough to hold at least +** pVfs->mxPathname bytes. +*/ +static int unixGetTempname(int nBuf, char *zBuf){ + const char *zDir; + int iLimit = 0; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + zBuf[0] = 0; + SimulateIOError( return SQLITE_IOERR ); + + zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; + }while( osAccess(zBuf,0)==0 ); + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Routine to transform a unixFile into a proxy-locking unixFile. +** Implementation in the proxy-lock division, but used by unixOpen() +** if SQLITE_PREFER_PROXY_LOCKING is defined. +*/ +static int proxyTransformUnixFile(unixFile*, const char*); +#endif + +/* +** Search for an unused file descriptor that was opened on the database +** file (not a journal or super-journal file) identified by pathname +** zPath with SQLITE_OPEN_XXX flags matching those passed as the second +** argument to this function. +** +** Such a file descriptor may exist if a database connection was closed +** but the associated file descriptor could not be closed because some +** other file descriptor open on the same file is holding a file-lock. +** Refer to comments in the unixClose() function and the lengthy comment +** describing "Posix Advisory Locking" at the start of this file for +** further details. Also, ticket #4018. +** +** If a suitable file descriptor is found, then it is returned. If no +** such file descriptor is located, -1 is returned. +*/ +static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ + UnixUnusedFd *pUnused = 0; + + /* Do not search for an unused file descriptor on vxworks. Not because + ** vxworks would not benefit from the change (it might, we're not sure), + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure + ** feature. */ +#if !OS_VXWORKS + struct stat sStat; /* Results of stat() call */ + + unixEnterMutex(); + + /* A stat() call may fail for various reasons. If this happens, it is + ** almost certain that an open() call on the same path will also fail. + ** For this reason, if an error occurs in the stat() call here, it is + ** ignored and -1 is returned. The caller will try to open a new file + ** descriptor on the same path, fail, and return an error to SQLite. + ** + ** Even if a subsequent open() call does succeed, the consequences of + ** not searching for a reusable file descriptor are not dire. */ + if( inodeList!=0 && 0==osStat(zPath, &sStat) ){ + unixInodeInfo *pInode; + + pInode = inodeList; + while( pInode && (pInode->fileId.dev!=sStat.st_dev + || pInode->fileId.ino!=(u64)sStat.st_ino) ){ + pInode = pInode->pNext; + } + if( pInode ){ + UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); + flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + pUnused = *pp; + if( pUnused ){ + *pp = pUnused->pNext; + } + sqlite3_mutex_leave(pInode->pLockMutex); + } + } + unixLeaveMutex(); +#endif /* if !OS_VXWORKS */ + return pUnused; +} + +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + +/* +** This function is called by unixOpen() to determine the unix permissions +** to create new files with. If no error occurs, then SQLITE_OK is returned +** and a value suitable for passing as the third argument to open(2) is +** written to *pMode. If an IO error occurs, an SQLite error code is +** returned and the value of *pMode is not modified. +** +** In most cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions +** as the associated database file. +** +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. +*/ +static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ + int flags, /* Flags passed as 4th argument to xOpen() */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ +){ + int rc = SQLITE_OK; /* Return Code */ + *pMode = 0; + *pUid = 0; + *pGid = 0; + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + char zDb[MAX_PATHNAME+1]; /* Database file path */ + int nDb; /* Number of valid bytes in zDb */ + + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journalNN" + ** "-walNN" + ** + ** where NN is a decimal number. The NN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; + while( zPath[nDb]!='-' ){ + /* In normal operation, the journal file name will always contain + ** a '-' character. However in 8+3 filename mode, or if a corrupt + ** rollback journal specifies a super-journal with a goofy name, then + ** the '-' might be missing. */ + if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; + nDb--; + } + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + + rc = getFileMode(zDb, pMode, pUid, pGid); + }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } + } + return rc; +} + +/* +** Open the file zPath. +** +** Previously, the SQLite OS layer used three functions in place of this +** one: +** +** sqlite3OsOpenReadWrite(); +** sqlite3OsOpenReadOnly(); +** sqlite3OsOpenExclusive(); +** +** These calls correspond to the following combinations of flags: +** +** ReadWrite() -> (READWRITE | CREATE) +** ReadOnly() -> (READONLY) +** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) +** +** The old OpenExclusive() accepted a boolean argument - "delFlag". If +** true, the file was configured to be automatically deleted when the +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for +** OpenExclusive(). +*/ +static int unixOpen( + sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ + const char *zPath, /* Pathname of file to be opened */ + sqlite3_file *pFile, /* The file descriptor to be filled in */ + int flags, /* Input flags to control the opening */ + int *pOutFlags /* Output flags returned to SQLite core */ +){ + unixFile *p = (unixFile *)pFile; + int fd = -1; /* File descriptor returned by open() */ + int openFlags = 0; /* Flags to pass to open() */ + int eType = flags&0x0FFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; /* Function Return Code */ + int ctrlFlags = 0; /* UNIXFILE_* flags */ + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#if SQLITE_ENABLE_LOCKING_STYLE + int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); +#endif +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; +#endif + + /* If creating a super- or main-file journal, this function will open + ** a file-descriptor on the directory too. The first time unixSync() + ** is called the directory file descriptor will be fsync()ed and close()d. + */ + int isNewJrnl = (isCreate && ( + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATHNAME+2]; + const char *zName = zPath; + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and super-journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=osGetpid(0) ){ + randomnessPid = osGetpid(0); + sqlite3_randomness(0,0); + } + memset(p, 0, sizeof(unixFile)); + + if( eType==SQLITE_OPEN_MAIN_DB ){ + UnixUnusedFd *pUnused; + pUnused = findReusableFd(zName, flags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc64(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM_BKPT; + } + } + p->pPreallocatedUnused = pUnused; + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). */ + assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); + + }else if( !zName ){ + /* If zName is NULL, the upper layer is requesting a temp file. */ + assert(isDelete && !isNewJrnl); + rc = unixGetTempname(pVfs->mxPathname, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; + + /* Generated temporary filenames are always double-zero terminated + ** for use by sqlite3_uri_parameter(). */ + assert( zName[strlen(zName)+1]==0 ); + } + + /* Determine the value of the flags parameter passed to POSIX function + ** open(). These must be calculated even if open() is not called, as + ** they may be stored as part of the file handle and used by the + ** 'conch file' locking functions later on. */ + if( isReadonly ) openFlags |= O_RDONLY; + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); + openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); + + if( fd<0 ){ + mode_t openMode; /* Permissions to create file with */ + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); + if( rc!=SQLITE_OK ){ + assert( !p->pPreallocatedUnused ); + assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); + return rc; + } + fd = robust_open(zName, openFlags, openMode); + OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); + assert( !isExclusive || (openFlags & O_CREAT)!=0 ); + if( fd<0 ){ + if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){ + /* If unable to create a journal because the directory is not + ** writable, change the error code to indicate that. */ + rc = SQLITE_READONLY_DIRECTORY; + }else if( errno!=EISDIR && isReadWrite ){ + /* Failed to open the file for read/write access. Try read-only. */ + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags &= ~(O_RDWR|O_CREAT); + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; + fd = robust_open(zName, openFlags, openMode); + } + } + if( fd<0 ){ + int rc2 = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + if( rc==SQLITE_OK ) rc = rc2; + goto open_finished; + } + + /* The owner of the rollback journal or WAL file should always be the + ** same as the owner of the database file. Try to ensure that this is + ** the case. The chown() system call will be a no-op if the current + ** process lacks root privileges, be we should at least try. Without + ** this step, if a root process opens a database file, it can leave + ** behinds a journal/WAL that is owned by root and hence make the + ** database inaccessible to unprivileged processes. + ** + ** If openMode==0, then that means uid and gid are not set correctly + ** (probably because SQLite is configured to use 8+3 filename mode) and + ** in that case we do not want to attempt the chown(). + */ + if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ + robustFchown(fd, uid, gid); + } + } + assert( fd>=0 ); + if( pOutFlags ){ + *pOutFlags = flags; + } + + if( p->pPreallocatedUnused ){ + p->pPreallocatedUnused->fd = fd; + p->pPreallocatedUnused->flags = + flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + } + + if( isDelete ){ +#if OS_VXWORKS + zPath = zName; +#elif defined(SQLITE_UNLINK_AFTER_CLOSE) + zPath = sqlite3_mprintf("%s", zName); + if( zPath==0 ){ + robust_close(p, fd, __LINE__); + return SQLITE_NOMEM_BKPT; + } +#else + osUnlink(zName); +#endif + } +#if SQLITE_ENABLE_LOCKING_STYLE + else{ + p->openFlags = openFlags; + } +#endif + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + if( fstatfs(fd, &fsInfo) == -1 ){ + storeLastErrno(p, errno); + robust_close(p, fd, __LINE__); + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } + if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } +#endif + + /* Set up appropriate ctrlFlags */ + if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; + if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; + if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; + if( isNewJrnl ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; + +#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_PREFER_PROXY_LOCKING + isAutoProxy = 1; +#endif + if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ + char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); + int useProxy = 0; + + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + ** never use proxy, NULL means use proxy for non-local files only. */ + if( envforce!=NULL ){ + useProxy = atoi(envforce)>0; + }else{ + useProxy = !(fsInfo.f_flags&MNT_LOCAL); + } + if( useProxy ){ + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + if( rc==SQLITE_OK ){ + rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + if( rc!=SQLITE_OK ){ + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + */ + unixClose(pFile); + return rc; + } + } + goto open_finished; + } + } +#endif + + assert( zPath==0 || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + ); + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + +open_finished: + if( rc!=SQLITE_OK ){ + sqlite3_free(p->pPreallocatedUnused); + } + return rc; +} + + +/* +** Delete the file at zPath. If the dirSync argument is true, fsync() +** the directory after deleting the file. +*/ +static int unixDelete( + sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ + const char *zPath, /* Name of file to be deleted */ + int dirSync /* If true, fsync() directory after deleting file */ +){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(NotUsed); + SimulateIOError(return SQLITE_IOERR_DELETE); + if( osUnlink(zPath)==(-1) ){ + if( errno==ENOENT +#if OS_VXWORKS + || osAccess(zPath,0)!=0 +#endif + ){ + rc = SQLITE_IOERR_DELETE_NOENT; + }else{ + rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + } + return rc; + } +#ifndef SQLITE_DISABLE_DIRSYNC + if( (dirSync & 1)!=0 ){ + int fd; + rc = osOpenDirectory(zPath, &fd); + if( rc==SQLITE_OK ){ + if( full_fsync(fd,0,0) ){ + rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); + } + robust_close(0, fd, __LINE__); + }else{ + assert( rc==SQLITE_CANTOPEN ); + rc = SQLITE_OK; + } + } +#endif + return rc; +} + +/* +** Test the existence of or access permissions of file zPath. The +** test performed depends on the value of flags: +** +** SQLITE_ACCESS_EXISTS: Return 1 if the file exists +** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. +** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. +** +** Otherwise return 0. +*/ +static int unixAccess( + sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ + const char *zPath, /* Path of the file to examine */ + int flags, /* What do we want to learn about the zPath file? */ + int *pResOut /* Write result boolean here */ +){ + UNUSED_PARAMETER(NotUsed); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + assert( pResOut!=0 ); + + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + + if( flags==SQLITE_ACCESS_EXISTS ){ + struct stat buf; + *pResOut = 0==osStat(zPath, &buf) && + (!S_ISREG(buf.st_mode) || buf.st_size>0); + }else{ + *pResOut = osAccess(zPath, W_OK|R_OK)==0; + } + return SQLITE_OK; +} + +/* +** If the last component of the pathname in z[0]..z[j-1] is something +** other than ".." then back it out and return true. If the last +** component is empty or if it is ".." then return false. +*/ +static int unixBackupDir(const char *z, int *pJ){ + int j = *pJ; + int i; + if( j<=0 ) return 0; + for(i=j-1; i>0 && z[i-1]!='/'; i--){} + if( i==0 ) return 0; + if( z[i]=='.' && i==j-2 && z[i+1]=='.' ) return 0; + *pJ = i-1; + return 1; +} + +/* +** Convert a relative pathname into a full pathname. Also +** simplify the pathname as follows: +** +** Remove all instances of /./ +** Remove all isntances of /X/../ for any X +*/ +static int mkFullPathname( + const char *zPath, /* Input path */ + char *zOut, /* Output buffer */ + int nOut /* Allocated size of buffer zOut */ +){ + int nPath = sqlite3Strlen30(zPath); + int iOff = 0; + int i, j; + if( zPath[0]!='/' ){ + if( osGetcwd(zOut, nOut-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + iOff = sqlite3Strlen30(zOut); + zOut[iOff++] = '/'; + } + if( (iOff+nPath+1)>nOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + + /* Remove duplicate '/' characters. Except, two // at the beginning + ** of a pathname is allowed since this is important on windows. */ + for(i=j=1; zOut[i]; i++){ + zOut[j++] = zOut[i]; + while( zOut[i]=='/' && zOut[i+1]=='/' ) i++; + } + zOut[j] = 0; + + assert( zOut[0]=='/' ); + for(i=j=0; zOut[i]; i++){ + if( zOut[i]=='/' ){ + /* Skip over internal "/." directory components */ + if( zOut[i+1]=='.' && zOut[i+2]=='/' ){ + i += 1; + continue; + } + + /* If this is a "/.." directory component then back out the + ** previous term of the directory if it is something other than "..". + */ + if( zOut[i+1]=='.' + && zOut[i+2]=='.' + && zOut[i+3]=='/' + && unixBackupDir(zOut, &j) + ){ + i += 2; + continue; + } + } + if( ALWAYS(j>=0) ) zOut[j] = zOut[i]; + j++; + } + if( NEVER(j==0) ) zOut[j++] = '/'; + zOut[j] = 0; + return SQLITE_OK; +} + +/* +** Turn a relative pathname into a full pathname. The relative path +** is stored as a nul-terminated string in the buffer pointed to by +** zPath. +** +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** (in this case, MAX_PATHNAME bytes). The full-path is written to +** this buffer before returning. +*/ +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ +#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) + return mkFullPathname(zPath, zOut, nOut); +#else + int rc = SQLITE_OK; + int nByte; + int nLink = 0; /* Number of symbolic links followed so far */ + const char *zIn = zPath; /* Input path for each iteration of loop */ + char *zDel = 0; + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + + do { + + /* Call stat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + struct stat buf; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else{ + bLink = S_ISLNK(buf.st_mode); + } + + if( bLink ){ + nLink++; + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; + }else if( nLink>=SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nOut-1); + if( nByte<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + + assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zOut ){ + rc = mkFullPathname(zIn, zOut, nOut); + } + if( bLink==0 ) break; + zIn = zOut; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + if( rc==SQLITE_OK && nLink ) rc = SQLITE_OK_SYMLINK; + return rc; +#endif /* HAVE_READLINK && HAVE_LSTAT */ +} + + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +#include +static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ + UNUSED_PARAMETER(NotUsed); + return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); +} + +/* +** SQLite calls this function immediately after a call to unixDlSym() or +** unixDlOpen() fails (returns a null pointer). If a more detailed error +** message is available, it is written to zBufOut. If no error message +** is available, zBufOut is left unmodified and SQLite uses a default +** error message. +*/ +static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ + const char *zErr; + UNUSED_PARAMETER(NotUsed); + unixEnterMutex(); + zErr = dlerror(); + if( zErr ){ + sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); + } + unixLeaveMutex(); +} +static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ + /* + ** GCC with -pedantic-errors says that C90 does not allow a void* to be + ** cast into a pointer to a function. And yet the library dlsym() routine + ** returns a void* which is really a pointer to a function. So how do we + ** use dlsym() with -pedantic-errors? + ** + ** Variable x below is defined to be a pointer to a function taking + ** parameters void* and const char* and returning a pointer to a function. + ** We initialize x by assigning it a pointer to the dlsym() function. + ** (That assignment requires a cast.) Then we call the function that + ** x points to. + ** + ** This work-around is unlikely to work correctly on any system where + ** you really cannot cast a function pointer into void*. But then, on the + ** other hand, dlsym() will not work on such a system either, so we have + ** not really lost anything. + */ + void (*(*x)(void*,const char*))(void); + UNUSED_PARAMETER(NotUsed); + x = (void(*(*)(void*,const char*))(void))dlsym; + return (*x)(p, zSym); +} +static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ + UNUSED_PARAMETER(NotUsed); + dlclose(pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define unixDlOpen 0 + #define unixDlError 0 + #define unixDlSym 0 + #define unixDlClose 0 +#endif + +/* +** Write nBuf bytes of random data to the supplied buffer zBuf. +*/ +static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + UNUSED_PARAMETER(NotUsed); + assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + + /* We have to initialize zBuf to prevent valgrind from reporting + ** errors. The reports issued by valgrind are incorrect - we would + ** prefer that the randomness be increased by making use of the + ** uninitialized space in zBuf - but valgrind errors tend to worry + ** some users. Rather than argue, it seems easier just to initialize + ** the whole array and silence valgrind, even if that means less randomness + ** in the random seed. + ** + ** When testing, initializing zBuf[] to zero is all we do. That means + ** that we always use the same random number sequence. This makes the + ** tests repeatable. + */ + memset(zBuf, 0, nBuf); + randomnessPid = osGetpid(0); +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) + { + int fd, got; + fd = robust_open("/dev/urandom", O_RDONLY, 0); + if( fd<0 ){ + time_t t; + time(&t); + memcpy(zBuf, &t, sizeof(t)); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); + }else{ + do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); + robust_close(0, fd, __LINE__); + } + } +#endif + return nBuf; +} + + +/* +** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. +*/ +static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ +#if OS_VXWORKS + struct timespec sp; + + sp.tv_sec = microseconds / 1000000; + sp.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#elif defined(HAVE_USLEEP) && HAVE_USLEEP + if( microseconds>=1000000 ) sleep(microseconds/1000000); + if( microseconds%1000000 ) usleep(microseconds%1000000); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#else + int seconds = (microseconds+999999)/1000000; + sleep(seconds); + UNUSED_PARAMETER(NotUsed); + return seconds*1000000; +#endif +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + int rc = SQLITE_OK; +#if defined(NO_GETTOD) + time_t t; + time(&t); + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; +#elif OS_VXWORKS + struct timespec sNow; + clock_gettime(CLOCK_REALTIME, &sNow); + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; +#endif + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(NotUsed); + return rc; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ + sqlite3_int64 i = 0; + int rc; + UNUSED_PARAMETER(NotUsed); + rc = unixCurrentTimeInt64(0, &i); + *prNow = i/86400000.0; + return rc; +} +#else +# define unixCurrentTime 0 +#endif + +/* +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. +*/ +static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ + UNUSED_PARAMETER(NotUsed); + UNUSED_PARAMETER(NotUsed2); + UNUSED_PARAMETER(NotUsed3); + return errno; +} + + +/* +************************ End of sqlite3_vfs methods *************************** +******************************************************************************/ + +/****************************************************************************** +************************** Begin Proxy Locking ******************************** +** +** Proxy locking is a "uber-locking-method" in this sense: It uses the +** other locking methods on secondary lock files. Proxy locking is a +** meta-layer over top of the primitive locking implemented above. For +** this reason, the division that implements of proxy locking is deferred +** until late in the file (here) after all of the other I/O methods have +** been defined - so that the primitive locking methods are available +** as services to help with the implementation of proxy locking. +** +**** +** +** The default locking schemes in SQLite use byte-range locks on the +** database file to coordinate safe, concurrent access by multiple readers +** and writers [http://sqlite.org/lockingv3.html]. The five file locking +** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented +** as POSIX read & write locks over fixed set of locations (via fsctl), +** on AFP and SMB only exclusive byte-range locks are available via fsctl +** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. +** To simulate a F_RDLCK on the shared range, on AFP a randomly selected +** address in the shared range is taken for a SHARED lock, the entire +** shared range is taken for an EXCLUSIVE lock): +** +** PENDING_BYTE 0x40000000 +** RESERVED_BYTE 0x40000001 +** SHARED_RANGE 0x40000002 -> 0x40000200 +** +** This works well on the local file system, but shows a nearly 100x +** slowdown in read performance on AFP because the AFP client disables +** the read cache when byte-range locks are present. Enabling the read +** cache exposes a cache coherency problem that is present on all OS X +** supported network file systems. NFS and AFP both observe the +** close-to-open semantics for ensuring cache coherency +** [http://nfs.sourceforge.net/#faq_a8], which does not effectively +** address the requirements for concurrent database access by multiple +** readers and writers +** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. +** +** To address the performance and cache coherency issues, proxy file locking +** changes the way database access is controlled by limiting access to a +** single host at a time and moving file locks off of the database file +** and onto a proxy file on the local file system. +** +** +** Using proxy locks +** ----------------- +** +** C APIs +** +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_SET_LOCKPROXYFILE, +** | ":auto:"); +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_GET_LOCKPROXYFILE, +** &); +** +** +** SQL pragmas +** +** PRAGMA [database.]lock_proxy_file= | :auto: +** PRAGMA [database.]lock_proxy_file +** +** Specifying ":auto:" means that if there is a conch file with a matching +** host ID in it, the proxy path in the conch file will be used, otherwise +** a proxy path based on the user's temp dir +** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the +** actual proxy file name is generated from the name and path of the +** database file. For example: +** +** For database path "/Users/me/foo.db" +** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") +** +** Once a lock proxy is configured for a database connection, it can not +** be removed, however it may be switched to a different proxy path via +** the above APIs (assuming the conch file is not being held by another +** connection or process). +** +** +** How proxy locking works +** ----------------------- +** +** Proxy file locking relies primarily on two new supporting files: +** +** * conch file to limit access to the database file to a single host +** at a time +** +** * proxy file to act as a proxy for the advisory locks normally +** taken on the database +** +** The conch file - to use a proxy file, sqlite must first "hold the conch" +** by taking an sqlite-style shared lock on the conch file, reading the +** contents and comparing the host's unique host ID (see below) and lock +** proxy path against the values stored in the conch. The conch file is +** stored in the same directory as the database file and the file name +** is patterned after the database file name as ".-conch". +** If the conch file does not exist, or its contents do not match the +** host ID and/or proxy path, then the lock is escalated to an exclusive +** lock and the conch file contents is updated with the host ID and proxy +** path and the lock is downgraded to a shared lock again. If the conch +** is held by another process (with a shared lock), the exclusive lock +** will fail and SQLITE_BUSY is returned. +** +** The proxy file - a single-byte file used for all advisory file locks +** normally taken on the database file. This allows for safe sharing +** of the database file for multiple readers and writers on the same +** host (the conch ensures that they all use the same local lock file). +** +** Requesting the lock proxy does not immediately take the conch, it is +** only taken when the first request to lock database file is made. +** This matches the semantics of the traditional locking behavior, where +** opening a connection to a database file does not take a lock on it. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. +** +** The proxy file and the lock file are never deleted so they only need +** to be created the first time they are used. +** +** Configuration options +** --------------------- +** +** SQLITE_PREFER_PROXY_LOCKING +** +** Database files accessed on non-local file systems are +** automatically configured for proxy locking, lock files are +** named automatically using the same logic as +** PRAGMA lock_proxy_file=":auto:" +** +** SQLITE_PROXY_DEBUG +** +** Enables the logging of error messages during host id file +** retrieval and creation +** +** LOCKPROXYDIR +** +** Overrides the default directory used for lock proxy files that +** are named automatically via the ":auto:" setting +** +** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +** +** Permissions to use when creating a directory for storing the +** lock proxy files, only used when LOCKPROXYDIR is not set. +** +** +** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, +** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will +** force proxy locking to be used for every database file opened, and 0 +** will force automatic proxy locking to be disabled for all database +** files (explicitly calling the SQLITE_FCNTL_SET_LOCKPROXYFILE pragma or +** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). +*/ + +/* +** Proxy locking is only available on MacOSX +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + +/* +** The proxyLockingContext has the path and file structures for the remote +** and local proxy files in it +*/ +typedef struct proxyLockingContext proxyLockingContext; +struct proxyLockingContext { + unixFile *conchFile; /* Open conch file */ + char *conchFilePath; /* Name of the conch file */ + unixFile *lockProxy; /* Open proxy lock file */ + char *lockProxyPath; /* Name of the proxy lock file */ + char *dbPath; /* Name of the open file */ + int conchHeld; /* 1 if the conch is held, -1 if lockless */ + int nFails; /* Number of conch taking failures */ + void *oldLockingContext; /* Original lockingcontext to restore on close */ + sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ +}; + +/* +** The proxy lock file path for the database at dbPath is written into lPath, +** which must point to valid, writable memory large enough for a maxLen length +** file path. +*/ +static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ + int len; + int dbLen; + int i; + +#ifdef LOCKPROXYDIR + len = strlcpy(lPath, LOCKPROXYDIR, maxLen); +#else +# ifdef _CS_DARWIN_USER_TEMP_DIR + { + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", + lPath, errno, osGetpid(0))); + return SQLITE_IOERR_LOCK; + } + len = strlcat(lPath, "sqliteplocks", maxLen); + } +# else + len = strlcpy(lPath, "/tmp/", maxLen); +# endif +#endif + + if( lPath[len-1]!='/' ){ + len = strlcat(lPath, "/", maxLen); + } + + /* transform the db path to a unique cache name */ + dbLen = (int)strlen(dbPath); + for( i=0; i 0) ){ + /* only mkdir if leaf dir != "." or "/" or ".." */ + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ + buf[i]='\0'; + if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + int err=errno; + if( err!=EEXIST ) { + OSTRACE(("CREATELOCKPATH FAILED creating %s, " + "'%s' proxy lock path=%s pid=%d\n", + buf, strerror(err), lockPath, osGetpid(0))); + return err; + } + } + } + start=i+1; + } + buf[i] = lockPath[i]; + } + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); + return 0; +} + +/* +** Create a new VFS file descriptor (stored in memory obtained from +** sqlite3_malloc) and open the file named "path" in the file descriptor. +** +** The caller is responsible not only for closing the file descriptor +** but also for freeing the memory associated with the file descriptor. +*/ +static int proxyCreateUnixFile( + const char *path, /* path for the new unixFile */ + unixFile **ppFile, /* unixFile created and returned by ref */ + int islockfile /* if non zero missing dirs will be created */ +) { + int fd = -1; + unixFile *pNew; + int rc = SQLITE_OK; + int openFlags = O_RDWR | O_CREAT | O_NOFOLLOW; + sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; + + /* 1. first try to open/create the file + ** 2. if that fails, and this is a lock file (not-conch), try creating + ** the parent directories and then try again. + ** 3. if that fails, try to open the file read-only + ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file + */ + pUnused = findReusableFd(path, openFlags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc64(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM_BKPT; + } + } + if( fd<0 ){ + fd = robust_open(path, openFlags, 0); + terrno = errno; + if( fd<0 && errno==ENOENT && islockfile ){ + if( proxyCreateLockPath(path) == SQLITE_OK ){ + fd = robust_open(path, openFlags, 0); + } + } + } + if( fd<0 ){ + openFlags = O_RDONLY | O_NOFOLLOW; + fd = robust_open(path, openFlags, 0); + terrno = errno; + } + if( fd<0 ){ + if( islockfile ){ + return SQLITE_BUSY; + } + switch (terrno) { + case EACCES: + return SQLITE_PERM; + case EIO: + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; + } + } + + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); + if( pNew==NULL ){ + rc = SQLITE_NOMEM_BKPT; + goto end_create_proxy; + } + memset(pNew, 0, sizeof(unixFile)); + pNew->openFlags = openFlags; + memset(&dummyVfs, 0, sizeof(dummyVfs)); + dummyVfs.pAppData = (void*)&autolockIoFinder; + dummyVfs.zName = "dummy"; + pUnused->fd = fd; + pUnused->flags = openFlags; + pNew->pPreallocatedUnused = pUnused; + + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); + if( rc==SQLITE_OK ){ + *ppFile = pNew; + return SQLITE_OK; + } +end_create_proxy: + robust_close(pNew, fd, __LINE__); + sqlite3_free(pNew); + sqlite3_free(pUnused); + return rc; +} + +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +int sqlite3_hostid_num = 0; +#endif + +#define PROXY_HOSTIDLEN 16 /* conch file host id length */ + +#if HAVE_GETHOSTUUID +/* Not always defined in the headers as it ought to be */ +extern int gethostuuid(uuid_t id, const struct timespec *wait); +#endif + +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +** bytes of writable memory. +*/ +static int proxyGetHostID(unsigned char *pHostID, int *pError){ + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); +#if HAVE_GETHOSTUUID + { + struct timespec timeout = {1, 0}; /* 1 sec timeout */ + if( gethostuuid(pHostID, &timeout) ){ + int err = errno; + if( pError ){ + *pError = err; + } + return SQLITE_IOERR; + } + } +#else + UNUSED_PARAMETER(pError); +#endif +#ifdef SQLITE_TEST + /* simulate multiple hosts by creating unique hostid file paths */ + if( sqlite3_hostid_num != 0){ + pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + } +#endif + + return SQLITE_OK; +} + +/* The conch file contains the header, host id and lock file path + */ +#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ +#define PROXY_HEADERLEN 1 /* conch file header length */ +#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) +#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) + +/* +** Takes an open conch file, copies the contents to a new path and then moves +** it back. The newly created file's file descriptor is assigned to the +** conch file structure and finally the original conch file descriptor is +** closed. Returns zero if successful. +*/ +static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + char tPath[MAXPATHLEN]; + char buf[PROXY_MAXCONCHLEN]; + char *cPath = pCtx->conchFilePath; + size_t readLen = 0; + size_t pathLen = 0; + char errmsg[64] = ""; + int fd = -1; + int rc = -1; + UNUSED_PARAMETER(myHostID); + + /* create a new path by replace the trailing '-conch' with '-break' */ + pathLen = strlcpy(tPath, cPath, MAXPATHLEN); + if( pathLen>MAXPATHLEN || pathLen<6 || + (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); + goto end_breaklock; + } + /* read the conch content */ + readLen = osPread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); + if( readLenh, __LINE__); + conchFile->h = fd; + conchFile->openFlags = O_RDWR | O_CREAT; + +end_breaklock: + if( rc ){ + if( fd>=0 ){ + osUnlink(tPath); + robust_close(pFile, fd, __LINE__); + } + fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); + } + return rc; +} + +/* Take the requested lock on the conch file and break a stale lock if the +** host id matches. +*/ +static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + int nTries = 0; + struct timespec conchModTime; + + memset(&conchModTime, 0, sizeof(conchModTime)); + do { + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + nTries ++; + if( rc==SQLITE_BUSY ){ + /* If the lock failed (busy): + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait + * 10 sec and try again + * 3rd try: break the lock unless the mod time has changed. + */ + struct stat buf; + if( osFstat(conchFile->h, &buf) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_LOCK; + } + + if( nTries==1 ){ + conchModTime = buf.st_mtimespec; + unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/ + continue; + } + + assert( nTries>1 ); + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ + return SQLITE_BUSY; + } + + if( nTries==2 ){ + char tBuf[PROXY_MAXCONCHLEN]; + int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ + /* don't break the lock if the host id doesn't match */ + if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + return SQLITE_BUSY; + } + }else{ + /* don't break the lock on short read or a version mismatch */ + return SQLITE_BUSY; + } + unixSleep(0,10000000); /* wait 10 sec and try the lock again */ + continue; + } + + assert( nTries==3 ); + if( 0==proxyBreakConchLock(pFile, myHostID) ){ + rc = SQLITE_OK; + if( lockType==EXCLUSIVE_LOCK ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + } + if( !rc ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + } + } + } + } while( rc==SQLITE_BUSY && nTries<3 ); + + return rc; +} + +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically +** and written to the conch file. +*/ +static int proxyTakeConch(unixFile *pFile){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + + if( pCtx->conchHeld!=0 ){ + return SQLITE_OK; + }else{ + unixFile *conchFile = pCtx->conchFile; + uuid_t myHostID; + int pError = 0; + char readBuf[PROXY_MAXCONCHLEN]; + char lockPath[MAXPATHLEN]; + char *tempLockPath = NULL; + int rc = SQLITE_OK; + int createConch = 0; + int hostIdMatch = 0; + int readLen = 0; + int tryOldLockPath = 0; + int forceNewLockPath = 0; + + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); + + rc = proxyGetHostID(myHostID, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + storeLastErrno(pFile, pError); + goto end_takeconch; + } + rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + goto end_takeconch; + } + /* read the existing conch file */ + readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + storeLastErrno(pFile, conchFile->lastErrno); + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + readBuf[0]!=(char)PROXY_CONCHVERSION ){ + /* a short read or version format mismatch means we need to create a new + ** conch file. + */ + createConch = 1; + } + /* if the host id matches and the lock path already exists in the conch + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path + */ + do { /* in case we need to try again for an :auto: named lock file */ + + if( !createConch && !forceNewLockPath ){ + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + PROXY_HOSTIDLEN); + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there + */ + if( hostIdMatch ){ + size_t pathLen = (readLen - PROXY_PATHINDEX); + + if( pathLen>=MAXPATHLEN ){ + pathLen=MAXPATHLEN-1; + } + memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); + lockPath[pathLen] = 0; + tempLockPath = lockPath; + tryOldLockPath = 1; + /* create a copy of the lock path if the conch is taken */ + goto end_takeconch; + } + }else if( hostIdMatch + && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], + readLen-PROXY_PATHINDEX) + ){ + /* conch host and lock path match */ + goto end_takeconch; + } + } + + /* if the conch isn't writable and doesn't match, we can't take it */ + if( (conchFile->openFlags&O_RDWR) == 0 ){ + rc = SQLITE_BUSY; + goto end_takeconch; + } + + /* either the conch didn't match or we need to create a new one */ + if( !pCtx->lockProxyPath ){ + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tempLockPath = lockPath; + /* create a copy of the lock path _only_ if the conch is taken */ + } + + /* update conch with host and path (this will fail if other process + ** has a shared lock already), if the host id matches, use the big + ** stick. + */ + futimes(conchFile->h, NULL); + if( hostIdMatch && !createConch ){ + if( conchFile->pInode && conchFile->pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + } else { + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + }else{ + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + if( rc==SQLITE_OK ){ + char writeBuffer[PROXY_MAXCONCHLEN]; + int writeSize = 0; + + writeBuffer[0] = (char)PROXY_CONCHVERSION; + memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); + if( pCtx->lockProxyPath!=NULL ){ + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, + MAXPATHLEN); + }else{ + strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); + } + writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + robust_ftruncate(conchFile->h, writeSize); + rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); + full_fsync(conchFile->h,0,0); + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database + */ + if( rc==SQLITE_OK && createConch ){ + struct stat buf; + int err = osFstat(pFile->h, &buf); + if( err==0 ){ + mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | + S_IROTH|S_IWOTH); + /* try to match the database file R/W permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + osFchmod(conchFile->h, cmode); +#else + do{ + rc = osFchmod(conchFile->h, cmode); + }while( rc==(-1) && errno==EINTR ); + if( rc!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + cmode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); +#endif + } + } + } + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + + end_takeconch: + OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); + if( rc==SQLITE_OK && pFile->openFlags ){ + int fd; + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + } + pFile->h = -1; + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); + OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; + rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ + /* we couldn't create the proxy lock file with the old lock file path + ** so try again via auto-naming + */ + forceNewLockPath = 1; + tryOldLockPath = 0; + continue; /* go back to the do {} while start point, try again */ + } + } + if( rc==SQLITE_OK ){ + /* Need to make a copy of path if we extracted the value + ** from the conch file or the path was allocated on the stack + */ + if( tempLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); + if( !pCtx->lockProxyPath ){ + rc = SQLITE_NOMEM_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + afpLockingContext *afpCtx; + afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; + afpCtx->dbPath = pCtx->lockProxyPath; + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE(("TAKECONCH %d %s\n", conchFile->h, + rc==SQLITE_OK?"ok":"failed")); + return rc; + } while (1); /* in case we need to retry the :auto: lock file - + ** we should never get here except via the 'continue' call. */ + } +} + +/* +** If pFile holds a lock on a conch file, then release that lock. +*/ +static int proxyReleaseConch(unixFile *pFile){ + int rc = SQLITE_OK; /* Subroutine return code */ + proxyLockingContext *pCtx; /* The locking context for the proxy lock */ + unixFile *conchFile; /* Name of the conch file */ + + pCtx = (proxyLockingContext *)pFile->lockingContext; + conchFile = pCtx->conchFile; + OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); + if( pCtx->conchHeld>0 ){ + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + pCtx->conchHeld = 0; + OSTRACE(("RELEASECONCH %d %s\n", conchFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + +/* +** Given the name of a database file, compute the name of its conch file. +** Store the conch filename in memory obtained from sqlite3_malloc64(). +** Make *pConchPath point to the new name. Return SQLITE_OK on success +** or SQLITE_NOMEM if unable to obtain memory. +** +** The caller is responsible for ensuring that the allocated memory +** space is eventually freed. +** +** *pConchPath is set to NULL if a memory allocation error occurs. +*/ +static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ + int i; /* Loop counter */ + int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ + char *conchPath; /* buffer in which to construct conch name */ + + /* Allocate space for the conch filename and initialize the name to + ** the name of the original database file. */ + *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); + if( conchPath==0 ){ + return SQLITE_NOMEM_BKPT; + } + memcpy(conchPath, dbPath, len+1); + + /* now insert a "." before the last / character */ + for( i=(len-1); i>=0; i-- ){ + if( conchPath[i]=='/' ){ + i++; + break; + } + } + conchPath[i]='.'; + while ( ilockingContext; + char *oldPath = pCtx->lockProxyPath; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + + /* nothing to do if the path is NULL, :auto: or matches the existing path */ + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || + (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ + return SQLITE_OK; + }else{ + unixFile *lockProxy = pCtx->lockProxy; + pCtx->lockProxy=NULL; + pCtx->conchHeld = 0; + if( lockProxy!=NULL ){ + rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + } + sqlite3_free(oldPath); + pCtx->lockProxyPath = sqlite3DbStrDup(0, path); + } + + return rc; +} + +/* +** pFile is a file that has been opened by a prior xOpen call. dbPath +** is a string buffer at least MAXPATHLEN+1 characters in size. +** +** This routine find the filename associated with pFile and writes it +** int dbPath. +*/ +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ +#if defined(__APPLE__) + if( pFile->pMethod == &afpIoMethods ){ + /* afp style keeps a reference to the db path in the filePath field + ** of the struct */ + assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, + MAXPATHLEN); + } else +#endif + if( pFile->pMethod == &dotlockIoMethods ){ + /* dot lock style uses the locking context to store the dot lock + ** file path */ + int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + memcpy(dbPath, (char *)pFile->lockingContext, len + 1); + }else{ + /* all other styles use the locking context to store the db file path */ + assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); + } + return SQLITE_OK; +} + +/* +** Takes an already filled in unix file and alters it so all file locking +** will be performed on the local proxy lock file. The following fields +** are preserved in the locking context so that they can be restored and +** the unix structure properly cleaned up at close time: +** ->lockingContext +** ->pMethod +*/ +static int proxyTransformUnixFile(unixFile *pFile, const char *path) { + proxyLockingContext *pCtx; + char dbPath[MAXPATHLEN+1]; /* Name of the database file */ + char *lockPath=NULL; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + proxyGetDbPathForUnixFile(pFile, dbPath); + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ + lockPath=NULL; + }else{ + lockPath=(char *)path; + } + + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, + (lockPath ? lockPath : ":auto:"), osGetpid(0))); + + pCtx = sqlite3_malloc64( sizeof(*pCtx) ); + if( pCtx==0 ){ + return SQLITE_NOMEM_BKPT; + } + memset(pCtx, 0, sizeof(*pCtx)); + + rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); + if( rc==SQLITE_OK ){ + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); + if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ + /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and + ** (c) the file system is read-only, then enable no-locking access. + ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts + ** that openFlags will have only one of O_RDONLY or O_RDWR. + */ + struct statfs fsInfo; + struct stat conchInfo; + int goLockless = 0; + + if( osStat(pCtx->conchFilePath, &conchInfo) == -1 ) { + int err = errno; + if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ + goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; + } + } + if( goLockless ){ + pCtx->conchHeld = -1; /* read only FS/ lockless */ + rc = SQLITE_OK; + } + } + } + if( rc==SQLITE_OK && lockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); + } + + if( rc==SQLITE_OK ){ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + if( pCtx->dbPath==NULL ){ + rc = SQLITE_NOMEM_BKPT; + } + } + if( rc==SQLITE_OK ){ + /* all memory is allocated, proxys are created and assigned, + ** switch the locking context and pMethod then return. + */ + pCtx->oldLockingContext = pFile->lockingContext; + pFile->lockingContext = pCtx; + pCtx->pOldMethod = pFile->pMethod; + pFile->pMethod = &proxyIoMethods; + }else{ + if( pCtx->conchFile ){ + pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); + sqlite3_free(pCtx->conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx); + } + OSTRACE(("TRANSPROXY %d %s\n", pFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + + +/* +** This routine handles sqlite3_file_control() calls that are specific +** to proxy locking. +*/ +static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + if( pFile->pMethod == &proxyIoMethods ){ + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; + proxyTakeConch(pFile); + if( pCtx->lockProxyPath ){ + *(const char **)pArg = pCtx->lockProxyPath; + }else{ + *(const char **)pArg = ":auto: (not held)"; + } + } else { + *(const char **)pArg = NULL; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_SET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + if( pArg==NULL || (const char *)pArg==0 ){ + if( isProxyStyle ){ + /* turn off proxy locking - not supported. If support is added for + ** switching proxy locking mode off then it will need to fail if + ** the journal mode is WAL mode. + */ + rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; + }else{ + /* turn off proxy locking - already off - NOOP */ + rc = SQLITE_OK; + } + }else{ + const char *proxyPath = (const char *)pArg; + if( isProxyStyle ){ + proxyLockingContext *pCtx = + (proxyLockingContext*)pFile->lockingContext; + if( !strcmp(pArg, ":auto:") + || (pCtx->lockProxyPath && + !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) + ){ + rc = SQLITE_OK; + }else{ + rc = switchLockProxyPath(pFile, proxyPath); + } + }else{ + /* turn on proxy file locking */ + rc = proxyTransformUnixFile(pFile, proxyPath); + } + } + return rc; + } + default: { + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } + } + /*NOTREACHED*/ assert(0); + return SQLITE_ERROR; +} + +/* +** Within this division (the proxying locking implementation) the procedures +** above this point are all utilities. The lock-related methods of the +** proxy-locking sqlite3_io_method object follow. +*/ + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + }else{ /* conchHeld < 0 is lockless */ + pResOut=0; + } + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int proxyLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int proxyUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + +/* +** Close a file that uses proxy locks. +*/ +static int proxyClose(sqlite3_file *id) { + if( ALWAYS(id) ){ + unixFile *pFile = (unixFile*)id; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *lockProxy = pCtx->lockProxy; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + + if( lockProxy ){ + rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); + if( rc ) return rc; + rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + pCtx->lockProxy = 0; + } + if( conchFile ){ + if( pCtx->conchHeld ){ + rc = proxyReleaseConch(pFile); + if( rc ) return rc; + } + rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); + if( rc ) return rc; + sqlite3_free(conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3DbFree(0, pCtx->dbPath); + /* restore the original locking context and pMethod then close it */ + pFile->lockingContext = pCtx->oldLockingContext; + pFile->pMethod = pCtx->pOldMethod; + sqlite3_free(pCtx); + return pFile->pMethod->xClose(id); + } + return SQLITE_OK; +} + + + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The proxy locking style is intended for use with AFP filesystems. +** And since AFP is only supported on MacOSX, the proxy locking is also +** restricted to MacOSX. +** +** +******************* End of the proxy lock implementation ********************** +******************************************************************************/ + +/* +** Initialize the operating system interface. +** +** This routine registers all VFS implementations for unix-like operating +** systems. This routine, and the sqlite3_os_end() routine that follows, +** should be the only routines in this file that are visible from other +** files. +** +** This routine is called once during SQLite initialization and by a +** single thread. The memory allocation and mutex subsystems have not +** necessarily been initialized when this routine is called, and so they +** should not be used. +*/ +int sqlite3_os_init(void){ + /* + ** The following macro defines an initializer for an sqlite3_vfs object. + ** The name of the VFS is NAME. The pAppData is a pointer to a pointer + ** to the "finder" function. (pAppData is a pointer to a pointer because + ** silly C90 rules prohibit a void* from being cast to a function pointer + ** and so we have to go through the intermediate pointer to avoid problems + ** when compiling with -pedantic-errors on GCC.) + ** + ** The FINDER parameter to this macro is the name of the pointer to the + ** finder-function. The finder-function returns a pointer to the + ** sqlite_io_methods object that implements the desired locking + ** behaviors. See the division above that contains the IOMETHODS + ** macro for addition information on finder-functions. + ** + ** Most finders simply return a pointer to a fixed sqlite3_io_methods + ** object. But the "autolockIoFinder" available on MacOSX does a little + ** more than that; it looks at the filesystem type that hosts the + ** database file and tries to choose an locking method appropriate for + ** that filesystem time. + */ + #define UNIXVFS(VFSNAME, FINDER) { \ + 3, /* iVersion */ \ + sizeof(unixFile), /* szOsFile */ \ + MAX_PATHNAME, /* mxPathname */ \ + 0, /* pNext */ \ + VFSNAME, /* zName */ \ + (void*)&FINDER, /* pAppData */ \ + unixOpen, /* xOpen */ \ + unixDelete, /* xDelete */ \ + unixAccess, /* xAccess */ \ + unixFullPathname, /* xFullPathname */ \ + unixDlOpen, /* xDlOpen */ \ + unixDlError, /* xDlError */ \ + unixDlSym, /* xDlSym */ \ + unixDlClose, /* xDlClose */ \ + unixRandomness, /* xRandomness */ \ + unixSleep, /* xSleep */ \ + unixCurrentTime, /* xCurrentTime */ \ + unixGetLastError, /* xGetLastError */ \ + unixCurrentTimeInt64, /* xCurrentTimeInt64 */ \ + unixSetSystemCall, /* xSetSystemCall */ \ + unixGetSystemCall, /* xGetSystemCall */ \ + unixNextSystemCall, /* xNextSystemCall */ \ + } + + /* + ** All default VFSes for unix are contained in the following array. + ** + ** Note that the sqlite3_vfs.pNext field of the VFS object is modified + ** by the SQLite core when the VFS is registered. So the following + ** array cannot be const. + */ + static sqlite3_vfs aVfs[] = { +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix", autolockIoFinder ), +#elif OS_VXWORKS + UNIXVFS("unix", vxworksIoFinder ), +#else + UNIXVFS("unix", posixIoFinder ), +#endif + UNIXVFS("unix-none", nolockIoFinder ), + UNIXVFS("unix-dotfile", dotlockIoFinder ), + UNIXVFS("unix-excl", posixIoFinder ), +#if OS_VXWORKS + UNIXVFS("unix-namedsem", semIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS + UNIXVFS("unix-posix", posixIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-flock", flockIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-nfs", nfsIoFinder ), + UNIXVFS("unix-proxy", proxyIoFinder ), +#endif + }; + unsigned int i; /* Loop counter */ + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==29 ); + + /* Register all VFSes defined in the aVfs[] array */ + for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ + sqlite3_vfs_register(&aVfs[i], i==0); + } + unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); + return SQLITE_OK; +} + +/* +** Shutdown the operating system interface. +** +** Some operating systems might need to do some cleanup in this routine, +** to release dynamically allocated objects. But not on unix. +** This routine is a no-op for unix. +*/ +int sqlite3_os_end(void){ + unixBigLock = 0; + return SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX */ diff --git a/third_party/sqlite3/os_win.c b/third_party/sqlite3/os_win.c new file mode 100644 index 000000000..26eaa9b89 --- /dev/null +++ b/third_party/sqlite3/os_win.c @@ -0,0 +1,6149 @@ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +#if SQLITE_OS_WIN /* This file is used for Windows only */ +/* clang-format off */ + +/* +** Include code that is common to all os_*.c files +*/ +#include "third_party/sqlite3/os_common.inc" + +/* +** Compiling and using WAL mode requires several APIs that are only +** available in Windows platforms based on the NT kernel. +*/ +#if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) +# error "WAL mode requires support from the Windows NT kernel, compile\ + with SQLITE_OMIT_WAL." +#endif + +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ + !defined(SQLITE_WIN32_NO_WIDE) +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Make sure at least one set of Win32 APIs is available. +*/ +#if !defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_WIN32_HAS_WIDE) +# error "At least one of SQLITE_WIN32_HAS_ANSI and SQLITE_WIN32_HAS_WIDE\ + must be defined." +#endif + +/* +** Define the required Windows SDK version constants if they are not +** already available. +*/ +#ifndef NTDDI_WIN8 +# define NTDDI_WIN8 0x06020000 +#endif + +#ifndef NTDDI_WINBLUE +# define NTDDI_WINBLUE 0x06030000 +#endif + +#ifndef NTDDI_WINTHRESHOLD +# define NTDDI_WINTHRESHOLD 0x06040000 +#endif + +/* +** Check to see if the GetVersionEx[AW] functions are deprecated on the +** target system. GetVersionEx was first deprecated in Win8.1. +*/ +#ifndef SQLITE_WIN32_GETVERSIONEX +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE +# define SQLITE_WIN32_GETVERSIONEX 0 /* GetVersionEx() is deprecated */ +# else +# define SQLITE_WIN32_GETVERSIONEX 1 /* GetVersionEx() is current */ +# endif +#endif + +/* +** Check to see if the CreateFileMappingA function is supported on the +** target system. It is unavailable when using "mincore.lib" on Win10. +** When compiling for Windows 10, always assume "mincore.lib" is in use. +*/ +#ifndef SQLITE_WIN32_CREATEFILEMAPPINGA +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINTHRESHOLD +# define SQLITE_WIN32_CREATEFILEMAPPINGA 0 +# else +# define SQLITE_WIN32_CREATEFILEMAPPINGA 1 +# endif +#endif + +/* +** This constant should already be defined (in the "WinDef.h" SDK file). +*/ +#ifndef MAX_PATH +# define MAX_PATH (260) +#endif + +/* +** Maximum pathname length (in chars) for Win32. This should normally be +** MAX_PATH. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_CHARS +# define SQLITE_WIN32_MAX_PATH_CHARS (MAX_PATH) +#endif + +/* +** This constant should already be defined (in the "WinNT.h" SDK file). +*/ +#ifndef UNICODE_STRING_MAX_CHARS +# define UNICODE_STRING_MAX_CHARS (32767) +#endif + +/* +** Maximum pathname length (in chars) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS. +*/ +#ifndef SQLITE_WINNT_MAX_PATH_CHARS +# define SQLITE_WINNT_MAX_PATH_CHARS (UNICODE_STRING_MAX_CHARS) +#endif + +/* +** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in +** characters, so we allocate 4 bytes per character assuming worst-case of +** 4-bytes-per-character for UTF8. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_BYTES +# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*4) +#endif + +/* +** Maximum pathname length (in bytes) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS * sizeof(WCHAR). +*/ +#ifndef SQLITE_WINNT_MAX_PATH_BYTES +# define SQLITE_WINNT_MAX_PATH_BYTES \ + (sizeof(WCHAR) * SQLITE_WINNT_MAX_PATH_CHARS) +#endif + +/* +** Maximum error message length (in chars) for WinRT. +*/ +#ifndef SQLITE_WIN32_MAX_ERRMSG_CHARS +# define SQLITE_WIN32_MAX_ERRMSG_CHARS (1024) +#endif + +/* +** Returns non-zero if the character should be treated as a directory +** separator. +*/ +#ifndef winIsDirSep +# define winIsDirSep(a) (((a) == '/') || ((a) == '\\')) +#endif + +/* +** This macro is used when a local variable is set to a value that is +** [sometimes] not used by the code (e.g. via conditional compilation). +*/ +#ifndef UNUSED_VARIABLE_VALUE +# define UNUSED_VARIABLE_VALUE(x) (void)(x) +#endif + +/* +** Returns the character that should be used as the directory separator. +*/ +#ifndef winGetDirSep +# define winGetDirSep() '\\' +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); + +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** These file mapping APIs are common to both Win32 and WinRT. +*/ + +WINBASEAPI BOOL WINAPI FlushViewOfFile(LPCVOID, SIZE_T); +WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API */ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_FILE_ATTRIBUTES +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +#endif + +#ifndef FILE_FLAG_MASK +# define FILE_FLAG_MASK (0xFF3C0000) +#endif + +#ifndef FILE_ATTRIBUTE_MASK +# define FILE_ATTRIBUTE_MASK (0x0003FFF7) +#endif + +#ifndef SQLITE_OMIT_WAL +/* Forward references to structures used for WAL */ +typedef struct winShm winShm; /* A connection to shared-memory */ +typedef struct winShmNode winShmNode; /* A region of shared-memory */ +#endif + +/* +** WinCE lacks native support for file locking so we have to fake it +** with some code of our own. +*/ +#if SQLITE_OS_WINCE +typedef struct winceLock { + int nReaders; /* Number of reader locks obtained */ + BOOL bPending; /* Indicates a pending lock has been obtained */ + BOOL bReserved; /* Indicates a reserved lock has been obtained */ + BOOL bExclusive; /* Indicates an exclusive lock has been obtained */ +} winceLock; +#endif + +/* +** The winFile structure is a subclass of sqlite3_file* specific to the win32 +** portability layer. +*/ +typedef struct winFile winFile; +struct winFile { + const sqlite3_io_methods *pMethod; /*** Must be first ***/ + sqlite3_vfs *pVfs; /* The VFS used to open this file */ + HANDLE h; /* Handle for accessing the file */ + u8 locktype; /* Type of lock currently held on this file */ + short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 ctrlFlags; /* Flags. See WINFILE_* below */ + DWORD lastErrno; /* The Windows errno from the last I/O error */ +#ifndef SQLITE_OMIT_WAL + winShm *pShm; /* Instance of shared memory on this file */ +#endif + const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_OS_WINCE + LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hShared; /* Shared memory segment used for locking */ + winceLock local; /* Locks obtained by this instance of winFile */ + winceLock *shared; /* Global shared lock memory for the file */ +#endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif +}; + +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + +/* +** Allowed values for winFile.ctrlFlags +*/ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ +#define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ + +/* + * The size of the buffer used by sqlite3_win32_write_debug(). + */ +#ifndef SQLITE_WIN32_DBG_BUF_SIZE +# define SQLITE_WIN32_DBG_BUF_SIZE ((int)(4096-sizeof(DWORD))) +#endif + +/* + * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the + * various Win32 API heap functions instead of our own. + */ +#ifdef SQLITE_WIN32_MALLOC + +/* + * If this is non-zero, an isolated heap will be created by the native Win32 + * allocator subsystem; otherwise, the default process heap will be used. This + * setting has no effect when compiling for WinRT. By default, this is enabled + * and an isolated heap will be created to store all allocated data. + * + ****************************************************************************** + * WARNING: It is important to note that when this setting is non-zero and the + * winMemShutdown function is called (e.g. by the sqlite3_shutdown + * function), all data that was allocated using the isolated heap will + * be freed immediately and any attempt to access any of that freed + * data will almost certainly result in an immediate access violation. + ****************************************************************************** + */ +#ifndef SQLITE_WIN32_HEAP_CREATE +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) +#endif + +/* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) +#endif + +/* + * The initial size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_SIZE +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) +#endif + +/* + * The maximum size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_SIZE +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +#endif + +/* + * The extra flags to use in calls to the Win32 heap APIs. This value may be + * zero for the default behavior. + */ +#ifndef SQLITE_WIN32_HEAP_FLAGS +# define SQLITE_WIN32_HEAP_FLAGS (0) +#endif + + +/* +** The winMemData structure stores information required by the Win32-specific +** sqlite3_mem_methods implementation. +*/ +typedef struct winMemData winMemData; +struct winMemData { +#ifndef NDEBUG + u32 magic1; /* Magic number to detect structure corruption. */ +#endif + HANDLE hHeap; /* The handle to our heap. */ + BOOL bOwned; /* Do we own the heap (i.e. destroy it on shutdown)? */ +#ifndef NDEBUG + u32 magic2; /* Magic number to detect structure corruption. */ +#endif +}; + +#ifndef NDEBUG +#define WINMEM_MAGIC1 0x42b2830b +#define WINMEM_MAGIC2 0xbd4d7cf4 +#endif + +static struct winMemData win_mem_data = { +#ifndef NDEBUG + WINMEM_MAGIC1, +#endif + NULL, FALSE +#ifndef NDEBUG + ,WINMEM_MAGIC2 +#endif +}; + +#ifndef NDEBUG +#define winMemAssertMagic1() assert( win_mem_data.magic1==WINMEM_MAGIC1 ) +#define winMemAssertMagic2() assert( win_mem_data.magic2==WINMEM_MAGIC2 ) +#define winMemAssertMagic() winMemAssertMagic1(); winMemAssertMagic2(); +#else +#define winMemAssertMagic() +#endif + +#define winMemGetDataPtr() &win_mem_data +#define winMemGetHeap() win_mem_data.hHeap +#define winMemGetOwned() win_mem_data.bOwned + +static void *winMemMalloc(int nBytes); +static void winMemFree(void *pPrior); +static void *winMemRealloc(void *pPrior, int nBytes); +static int winMemSize(void *p); +static int winMemRoundup(int n); +static int winMemInit(void *pAppData); +static void winMemShutdown(void *pAppData); + +const sqlite3_mem_methods *sqlite3MemGetWin32(void); +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** The following variable is (normally) set once and never changes +** thereafter. It records whether the operating system is Win9x +** or WinNT. +** +** 0: Operating system unknown. +** 1: Operating system is Win9x. +** 2: Operating system is WinNT. +** +** In order to facilitate testing on a WinNT system, the test fixture +** can manually set this value to 1 to emulate Win98 behavior. +*/ +#ifdef SQLITE_TEST +LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; +#else +static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; +#endif + +#ifndef SYSCALL +# define SYSCALL sqlite3_syscall_ptr +#endif + +/* +** This function is not available on Windows CE or WinRT. + */ + +#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +# define osAreFileApisANSI() 1 +#endif + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct win_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, +#else + { "AreFileApisANSI", (SYSCALL)0, 0 }, +#endif + +#ifndef osAreFileApisANSI +#define osAreFileApisANSI ((BOOL(WINAPI*)(VOID))aSyscall[0].pCurrent) +#endif + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharLowerW", (SYSCALL)CharLowerW, 0 }, +#else + { "CharLowerW", (SYSCALL)0, 0 }, +#endif + +#define osCharLowerW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[1].pCurrent) + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharUpperW", (SYSCALL)CharUpperW, 0 }, +#else + { "CharUpperW", (SYSCALL)0, 0 }, +#endif + +#define osCharUpperW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[2].pCurrent) + + { "CloseHandle", (SYSCALL)CloseHandle, 0 }, + +#define osCloseHandle ((BOOL(WINAPI*)(HANDLE))aSyscall[3].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "CreateFileA", (SYSCALL)CreateFileA, 0 }, +#else + { "CreateFileA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateFileW", (SYSCALL)CreateFileW, 0 }, +#else + { "CreateFileW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ + SQLITE_WIN32_CREATEFILEMAPPINGA + { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, +#else + { "CreateFileMappingA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) + { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, +#else + { "CreateFileMappingW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, +#else + { "CreateMutexW", (SYSCALL)0, 0 }, +#endif + +#define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ + LPCWSTR))aSyscall[8].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, +#else + { "DeleteFileA", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, +#else + { "DeleteFileW", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, +#else + { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPFILETIME))aSyscall[11].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, +#else + { "FileTimeToSystemTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPSYSTEMTIME))aSyscall[12].pCurrent) + + { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, + +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, +#else + { "FormatMessageA", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ + DWORD,va_list*))aSyscall[14].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, +#else + { "FormatMessageW", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ + DWORD,va_list*))aSyscall[15].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif + +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) + + { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, + +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, +#else + { "GetDiskFreeSpaceA", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[18].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, +#else + { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[19].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, +#else + { "GetFileAttributesA", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, +#else + { "GetFileAttributesW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, +#else + { "GetFileAttributesExW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ + LPVOID))aSyscall[22].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetFileSize", (SYSCALL)GetFileSize, 0 }, +#else + { "GetFileSize", (SYSCALL)0, 0 }, +#endif + +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, +#else + { "GetFullPathNameA", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ + LPSTR*))aSyscall[24].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, +#else + { "GetFullPathNameW", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ + LPWSTR*))aSyscall[25].pCurrent) + + { "GetLastError", (SYSCALL)GetLastError, 0 }, + +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) +#if SQLITE_OS_WINCE + /* The GetProcAddressA() routine is only available on Windows CE. */ + { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, +#else + /* All other Windows platforms expect GetProcAddress() to take + ** an ANSI string regardless of the _UNICODE setting */ + { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, +#endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif + +#define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ + LPCSTR))aSyscall[27].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, +#else + { "GetSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) + + { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, + +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) + +#if !SQLITE_OS_WINCE + { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, +#else + { "GetSystemTimeAsFileTime", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ + LPFILETIME))aSyscall[30].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, +#else + { "GetTempPathA", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, +#else + { "GetTempPathW", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetTickCount", (SYSCALL)GetTickCount, 0 }, +#else + { "GetTickCount", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_GETVERSIONEX + { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, +#else + { "GetVersionExA", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExA ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOA))aSyscall[34].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + SQLITE_WIN32_GETVERSIONEX + { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, +#else + { "GetVersionExW", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExW ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOW))aSyscall[35].pCurrent) + + { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, + +#define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ + SIZE_T))aSyscall[36].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapCreate", (SYSCALL)HeapCreate, 0 }, +#else + { "HeapCreate", (SYSCALL)0, 0 }, +#endif + +#define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ + SIZE_T))aSyscall[37].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, +#else + { "HeapDestroy", (SYSCALL)0, 0 }, +#endif + +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) + + { "HeapFree", (SYSCALL)HeapFree, 0 }, + +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[39].pCurrent) + + { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, + +#define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ + SIZE_T))aSyscall[40].pCurrent) + + { "HeapSize", (SYSCALL)HeapSize, 0 }, + +#define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[41].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapValidate", (SYSCALL)HeapValidate, 0 }, +#else + { "HeapValidate", (SYSCALL)0, 0 }, +#endif + +#define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[42].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "HeapCompact", (SYSCALL)HeapCompact, 0 }, +#else + { "HeapCompact", (SYSCALL)0, 0 }, +#endif + +#define osHeapCompact ((UINT(WINAPI*)(HANDLE,DWORD))aSyscall[43].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, +#else + { "LoadLibraryA", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, +#else + { "LoadLibraryW", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) + +#if !SQLITE_OS_WINRT + { "LocalFree", (SYSCALL)LocalFree, 0 }, +#else + { "LocalFree", (SYSCALL)0, 0 }, +#endif + +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "LockFile", (SYSCALL)LockFile, 0 }, +#else + { "LockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFile +#define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[47].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "LockFileEx", (SYSCALL)LockFileEx, 0 }, +#else + { "LockFileEx", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFileEx +#define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[48].pCurrent) +#endif + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) + { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, +#else + { "MapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + SIZE_T))aSyscall[49].pCurrent) + + { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, + +#define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ + int))aSyscall[50].pCurrent) + + { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, + +#define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ + LARGE_INTEGER*))aSyscall[51].pCurrent) + + { "ReadFile", (SYSCALL)ReadFile, 0 }, + +#define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[52].pCurrent) + + { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, + +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) + +#if !SQLITE_OS_WINRT + { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, +#else + { "SetFilePointer", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ + DWORD))aSyscall[54].pCurrent) + +#if !SQLITE_OS_WINRT + { "Sleep", (SYSCALL)Sleep, 0 }, +#else + { "Sleep", (SYSCALL)0, 0 }, +#endif + +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) + + { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, + +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ + LPFILETIME))aSyscall[56].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "UnlockFile", (SYSCALL)UnlockFile, 0 }, +#else + { "UnlockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osUnlockFile +#define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[57].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "UnlockFileEx", (SYSCALL)UnlockFileEx, 0 }, +#else + { "UnlockFileEx", (SYSCALL)0, 0 }, +#endif + +#define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[58].pCurrent) + +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, +#else + { "UnmapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[59].pCurrent) + + { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, + +#define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ + LPCSTR,LPBOOL))aSyscall[60].pCurrent) + + { "WriteFile", (SYSCALL)WriteFile, 0 }, + +#define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[61].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, +#else + { "CreateEventExW", (SYSCALL)0, 0 }, +#endif + +#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ + DWORD,DWORD))aSyscall[62].pCurrent) + +#if !SQLITE_OS_WINRT + { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, +#else + { "WaitForSingleObject", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ + DWORD))aSyscall[63].pCurrent) + +#if !SQLITE_OS_WINCE + { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, +#else + { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ + BOOL))aSyscall[64].pCurrent) + +#if SQLITE_OS_WINRT + { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, +#else + { "SetFilePointerEx", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ + PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) + +#if SQLITE_OS_WINRT + { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, +#else + { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, +#endif + +#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, +#else + { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ + SIZE_T))aSyscall[67].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateFile2", (SYSCALL)CreateFile2, 0 }, +#else + { "CreateFile2", (SYSCALL)0, 0 }, +#endif + +#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, +#else + { "LoadPackagedLibrary", (SYSCALL)0, 0 }, +#endif + +#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ + DWORD))aSyscall[69].pCurrent) + +#if SQLITE_OS_WINRT + { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, +#else + { "GetTickCount64", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) + +#if SQLITE_OS_WINRT + { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, +#else + { "GetNativeSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetNativeSystemInfo ((VOID(WINAPI*)( \ + LPSYSTEM_INFO))aSyscall[71].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, +#else + { "OutputDebugStringA", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, +#else + { "OutputDebugStringW", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) + + { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, + +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, +#else + { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) + +/* +** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" +** is really just a macro that uses a compiler intrinsic (e.g. x64). +** So do not try to make this is into a redefinable interface. +*/ +#if defined(InterlockedCompareExchange) + { "InterlockedCompareExchange", (SYSCALL)0, 0 }, + +#define osInterlockedCompareExchange InterlockedCompareExchange +#else + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreate", (SYSCALL)UuidCreate, 0 }, +#else + { "UuidCreate", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, +#else + { "UuidCreateSequential", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreateSequential \ + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + +#if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 + { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, +#else + { "FlushViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osFlushViewOfFile \ + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int winSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + osOutputDebugStringA(zDbgBuf); + }else{ + osOutputDebugStringA(zBuf); + } +#elif defined(SQLITE_WIN32_HAS_WIDE) + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + if ( osMultiByteToWideChar( + osAreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, zBuf, + nMin, (LPWSTR)zDbgBuf, SQLITE_WIN32_DBG_BUF_SIZE/sizeof(WCHAR))<=0 ){ + return; + } + osOutputDebugStringW((LPCWSTR)zDbgBuf); +#else + if( nMin>0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + fprintf(stderr, "%s", zDbgBuf); + }else{ + fprintf(stderr, "%s", zBuf); + } +#endif +} + +/* +** The following routine suspends the current thread for at least ms +** milliseconds. This is equivalent to the Win32 Sleep() interface. +*/ +#if SQLITE_OS_WINRT +static HANDLE sleepObj = NULL; +#endif + +void sqlite3_win32_sleep(DWORD milliseconds){ +#if SQLITE_OS_WINRT + if ( sleepObj==NULL ){ + sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, + SYNCHRONIZE); + } + assert( sleepObj!=NULL ); + osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); +#else + osSleep(milliseconds); +#endif +} + +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} +#endif + +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it when running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +*/ + +#if !SQLITE_WIN32_GETVERSIONEX +# define osIsNT() (1) +#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +# define osIsNT() (1) +#elif !defined(SQLITE_WIN32_HAS_WIDE) +# define osIsNT() (0) +#else +# define osIsNT() ((sqlite3_os_type==2) || sqlite3_win32_is_nt()) +#endif + +/* +** This function determines if the machine is running a version of Windows +** based on the NT kernel. +*/ +int sqlite3_win32_is_nt(void){ +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif SQLITE_WIN32_GETVERSIONEX + if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ +#if defined(SQLITE_WIN32_HAS_ANSI) + OSVERSIONINFOA sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExA(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_WIDE) + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#endif + } + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#elif SQLITE_TEST + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#else + /* + ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are + ** deprecated are always assumed to be based on the NT kernel. + */ + return 1; +#endif +} + +#ifdef SQLITE_WIN32_MALLOC +/* +** Allocate nBytes of memory. +*/ +static void *winMemMalloc(int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + assert( nBytes>=0 ); + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapAlloc %u bytes (%lu), heap=%p", + nBytes, osGetLastError(), (void*)hHeap); + } + return p; +} + +/* +** Free memory. +*/ +static void winMemFree(void *pPrior){ + HANDLE hHeap; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ + if( !osHeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapFree block %p (%lu), heap=%p", + pPrior, osGetLastError(), (void*)hHeap); + } +} + +/* +** Change the size of an existing memory allocation +*/ +static void *winMemRealloc(void *pPrior, int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + assert( nBytes>=0 ); + if( !pPrior ){ + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + }else{ + p = osHeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); + } + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to %s %u bytes (%lu), heap=%p", + pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, osGetLastError(), + (void*)hHeap); + } + return p; +} + +/* +** Return the size of an outstanding allocation, in bytes. +*/ +static int winMemSize(void *p){ + HANDLE hHeap; + SIZE_T n; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); +#endif + if( !p ) return 0; + n = osHeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); + if( n==(SIZE_T)-1 ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapSize block %p (%lu), heap=%p", + p, osGetLastError(), (void*)hHeap); + return 0; + } + return (int)n; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int winMemRoundup(int n){ + return n; +} + +/* +** Initialize this module. +*/ +static int winMemInit(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return SQLITE_ERROR; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + +#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE + if( !pWinMemData->hHeap ){ + DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; + DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; + if( dwMaximumSize==0 ){ + dwMaximumSize = SQLITE_WIN32_HEAP_MAX_SIZE; + }else if( dwInitialSize>dwMaximumSize ){ + dwInitialSize = dwMaximumSize; + } + pWinMemData->hHeap = osHeapCreate(SQLITE_WIN32_HEAP_FLAGS, + dwInitialSize, dwMaximumSize); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", + osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, + dwMaximumSize); + return SQLITE_NOMEM_BKPT; + } + pWinMemData->bOwned = TRUE; + assert( pWinMemData->bOwned ); + } +#else + pWinMemData->hHeap = osGetProcessHeap(); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to GetProcessHeap (%lu)", osGetLastError()); + return SQLITE_NOMEM_BKPT; + } + pWinMemData->bOwned = FALSE; + assert( !pWinMemData->bOwned ); +#endif + assert( pWinMemData->hHeap!=0 ); + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void winMemShutdown(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + + if( pWinMemData->hHeap ){ + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + if( pWinMemData->bOwned ){ + if( !osHeapDestroy(pWinMemData->hHeap) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapDestroy (%lu), heap=%p", + osGetLastError(), (void*)pWinMemData->hHeap); + } + pWinMemData->bOwned = FALSE; + } + pWinMemData->hHeap = NULL; + } +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +const sqlite3_mem_methods *sqlite3MemGetWin32(void){ + static const sqlite3_mem_methods winMemMethods = { + winMemMalloc, + winMemFree, + winMemRealloc, + winMemSize, + winMemRoundup, + winMemInit, + winMemShutdown, + &win_mem_data + }; + return &winMemMethods; +} + +void sqlite3MemSetDefault(void){ + sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetWin32()); +} +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** Convert a UTF-8 string to Microsoft Unicode. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static LPWSTR winUtf8ToUnicode(const char *zText){ + int nChar; + LPWSTR zWideText; + + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); + if( zWideText==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, + nChar); + if( nChar==0 ){ + sqlite3_free(zWideText); + zWideText = 0; + } + return zWideText; +} + +/* +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUnicodeToUtf8(LPCWSTR zWideText){ + int nByte; + char *zText; + + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, + 0, 0); + if( nByte == 0 ){ + sqlite3_free(zText); + zText = 0; + } + return zText; +} + +/* +** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM +** code page. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ + int nByte; + LPWSTR zMbcsText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; + + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + 0)*sizeof(WCHAR); + if( nByte==0 ){ + return 0; + } + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + if( zMbcsText==0 ){ + return 0; + } + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nByte); + if( nByte==0 ){ + sqlite3_free(zMbcsText); + zMbcsText = 0; + } + return zMbcsText; +} + +/* +** Convert a Microsoft Unicode string to a multi-byte character string, +** using the ANSI or OEM code page. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ + int nByte; + char *zText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; + + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, + nByte, 0, 0); + if( nByte == 0 ){ + sqlite3_free(zText); + zText = 0; + } + return zText; +} + +/* +** Convert a multi-byte character string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winMbcsToUtf8(const char *zText, int useAnsi){ + char *zTextUtf8; + LPWSTR zTmpWide; + + zTmpWide = winMbcsToUnicode(zText, useAnsi); + if( zTmpWide==0 ){ + return 0; + } + zTextUtf8 = winUnicodeToUtf8(zTmpWide); + sqlite3_free(zTmpWide); + return zTextUtf8; +} + +/* +** Convert a UTF-8 string to a multi-byte character string. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUtf8ToMbcs(const char *zText, int useAnsi){ + char *zTextMbcs; + LPWSTR zTmpWide; + + zTmpWide = winUtf8ToUnicode(zText); + if( zTmpWide==0 ){ + return 0; + } + zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); + sqlite3_free(zTmpWide); + return zTextMbcs; +} + +/* +** This is a public wrapper for the winUtf8ToUnicode() function. +*/ +LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToUnicode(zText); +} + +/* +** This is a public wrapper for the winUnicodeToUtf8() function. +*/ +char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zWideText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUnicodeToUtf8(zWideText); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, useAnsi); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, useAnsi); +} + +/* +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-8 string. +*/ +int sqlite3_win32_set_directory8( + unsigned long type, /* Identifier for directory being set or reset */ + const char *zValue /* New value for directory being set or reset */ +){ + char **ppDirectory = 0; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_data_directory; + }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_temp_directory; + } + assert( !ppDirectory || type==SQLITE_WIN32_DATA_DIRECTORY_TYPE + || type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE + ); + assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) ); + if( ppDirectory ){ + char *zCopy = 0; + if( zValue && zValue[0] ){ + zCopy = sqlite3_mprintf("%s", zValue); + if ( zCopy==0 ){ + return SQLITE_NOMEM_BKPT; + } + } + sqlite3_free(*ppDirectory); + *ppDirectory = zCopy; + return SQLITE_OK; + } + return SQLITE_ERROR; +} + +/* +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-16 string. +*/ +int sqlite3_win32_set_directory16( + unsigned long type, /* Identifier for directory being set or reset */ + const void *zValue /* New value for directory being set or reset */ +){ + int rc; + char *zUtf8 = 0; + if( zValue ){ + zUtf8 = sqlite3_win32_unicode_to_utf8(zValue); + if( zUtf8==0 ) return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_win32_set_directory8(type, zUtf8); + if( zUtf8 ) sqlite3_free(zUtf8); + return rc; +} + +/* +** This function sets the data directory or the temporary directory based on +** the provided arguments. The type argument must be 1 in order to set the +** data directory or 2 in order to set the temporary directory. The zValue +** argument is the name of the directory to use. The return value will be +** SQLITE_OK if successful. +*/ +int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +){ + return sqlite3_win32_set_directory16(type, zValue); +} + +/* +** The return value of winGetLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD dwLen = 0; + char *zOut = 0; + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; + dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + zTempWide, + SQLITE_WIN32_MAX_ERRMSG_CHARS, + 0); +#else + LPWSTR zTempWide = NULL; + dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPWSTR) &zTempWide, + 0, + 0); +#endif + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = winUnicodeToUtf8(zTempWide); + sqlite3EndBenignMalloc(); +#if !SQLITE_OS_WINRT + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTempWide); +#endif + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp = NULL; + dwLen = osFormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + sqlite3EndBenignMalloc(); + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTemp); + } + } +#endif + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + sqlite3_free(zOut); + } + return 0; +} + +/* +** +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). +** +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the associated file-system path, if any. +*/ +#define winLogError(a,b,c,d) winLogErrorAtLine(a,b,c,d,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + DWORD lastErrno, /* Win32 last error */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + + zMsg[0] = 0; + winGetLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%lu) %s(%s) - %s", + iLine, lastErrno, zFunc, zPath, zMsg + ); + + return errcode; +} + +/* +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 +#endif +static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; + +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int winRetryIoerr(int *pnRetry, DWORD *pError){ + DWORD e = osGetLastError(); + if( *pnRetry>=winIoerrRetry ){ + if( pError ){ + *pError = e; + } + return 0; + } + if( winIoerrCanRetry1(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif + if( pError ){ + *pError = e; + } + return 0; +} + +/* +** Log a I/O error retry episode. +*/ +static void winLogIoerr(int nRetry, int lineno){ + if( nRetry ){ + sqlite3_log(SQLITE_NOTICE, + "delayed %dms for lock/sharing conflict at line %d", + winIoerrRetryDelay*nRetry*(nRetry+1)/2, lineno + ); + } +} + +/* +** This #if does not rely on the SQLITE_OS_WINCE define because the +** corresponding section in "date.c" cannot use it. +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So define a substitute. +*/ +#include "libc/time/time.h" +struct tm *__cdecl localtime(const time_t *t) { + static struct tm y; + FILETIME uTm, lTm; + SYSTEMTIME pTm; + sqlite3_int64 t64; + t64 = *t; + t64 = (t64 + 11644473600)*10000000; + uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); + uTm.dwHighDateTime= (DWORD)(t64 >> 32); + osFileTimeToLocalFileTime(&uTm,&lTm); + osFileTimeToSystemTime(&lTm,&pTm); + y.tm_year = pTm.wYear - 1900; + y.tm_mon = pTm.wMonth - 1; + y.tm_wday = pTm.wDayOfWeek; + y.tm_mday = pTm.wDay; + y.tm_hour = pTm.wHour; + y.tm_min = pTm.wMinute; + y.tm_sec = pTm.wSecond; + return &y; +} +#endif + +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ +#define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] + +/* +** Acquire a lock on the handle h +*/ +static void winceMutexAcquire(HANDLE h){ + DWORD dwErr; + do { + dwErr = osWaitForSingleObject(h, INFINITE); + } while (dwErr != WAIT_OBJECT_0 && dwErr != WAIT_ABANDONED); +} +/* +** Release a lock acquired by winceMutexAcquire() +*/ +#define winceMutexRelease(h) ReleaseMutex(h) + +/* +** Create the mutex and shared memory used for locking in the file +** descriptor pFile +*/ +static int winceCreateLock(const char *zFilename, winFile *pFile){ + LPWSTR zTok; + LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; + BOOL bInit = TRUE; + + zName = winUtf8ToUnicode(zFilename); + if( zName==0 ){ + /* out of memory */ + return SQLITE_IOERR_NOMEM_BKPT; + } + + /* Initialize the local lockdata */ + memset(&pFile->local, 0, sizeof(pFile->local)); + + /* Replace the backslashes from the filename and lowercase it + ** to derive a mutex name. */ + zTok = osCharLowerW(zName); + for (;*zTok;zTok++){ + if (*zTok == '\\') *zTok = '_'; + } + + /* Create/open the named mutex */ + pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); + if (!pFile->hMutex){ + pFile->lastErrno = osGetLastError(); + sqlite3_free(zName); + return winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); + } + + /* Acquire the mutex before continuing */ + winceMutexAcquire(pFile->hMutex); + + /* Since the names of named mutexes, semaphores, file mappings etc are + ** case-sensitive, take advantage of that by uppercasing the mutex name + ** and using that as the shared filemapping name. + */ + osCharUpperW(zName); + pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(winceLock), + zName); + + /* Set a flag that indicates we're the first to create the memory so it + ** must be zero-initialized */ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ + bInit = FALSE; + } + + sqlite3_free(zName); + + /* If we succeeded in making the shared memory handle, map it. */ + if( pFile->hShared ){ + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, + FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); + /* If mapping failed, close the shared memory handle and erase it */ + if( !pFile->shared ){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; + osCloseHandle(pFile->hShared); + pFile->hShared = NULL; + } + } + + /* If shared memory could not be created, then close the mutex and fail */ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + return SQLITE_IOERR; + } + + /* Initialize the shared memory if we're supposed to */ + if( bInit ){ + memset(pFile->shared, 0, sizeof(winceLock)); + } + + winceMutexRelease(pFile->hMutex); + return SQLITE_OK; +} + +/* +** Destroy the part of winFile that deals with wince locks +*/ +static void winceDestroyLock(winFile *pFile){ + if (pFile->hMutex){ + /* Acquire the mutex */ + winceMutexAcquire(pFile->hMutex); + + /* The following blocks should probably assert in debug mode, but they + are to cleanup in case any locks remained open */ + if (pFile->local.nReaders){ + pFile->shared->nReaders --; + } + if (pFile->local.bReserved){ + pFile->shared->bReserved = FALSE; + } + if (pFile->local.bPending){ + pFile->shared->bPending = FALSE; + } + if (pFile->local.bExclusive){ + pFile->shared->bExclusive = FALSE; + } + + /* De-reference and close our copy of the shared memory handle */ + osUnmapViewOfFile(pFile->shared); + osCloseHandle(pFile->hShared); + + /* Done with the mutex */ + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + } +} + +/* +** An implementation of the LockFile() API of Windows for CE +*/ +static BOOL winceLockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToLockLow, + DWORD nNumberOfBytesToLockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToLockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Wanting an exclusive lock? */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST + && nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ + if (pFile->shared->nReaders == 0 && pFile->shared->bExclusive == 0){ + pFile->shared->bExclusive = TRUE; + pFile->local.bExclusive = TRUE; + bReturn = TRUE; + } + } + + /* Want a read-only lock? */ + else if (dwFileOffsetLow == (DWORD)SHARED_FIRST && + nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bExclusive == 0){ + pFile->local.nReaders ++; + if (pFile->local.nReaders == 1){ + pFile->shared->nReaders ++; + } + bReturn = TRUE; + } + } + + /* Want a pending lock? */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ + /* If no pending lock has been acquired, then acquire it */ + if (pFile->shared->bPending == 0) { + pFile->shared->bPending = TRUE; + pFile->local.bPending = TRUE; + bReturn = TRUE; + } + } + + /* Want a reserved lock? */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bReserved == 0) { + pFile->shared->bReserved = TRUE; + pFile->local.bReserved = TRUE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} + +/* +** An implementation of the UnlockFile API of Windows for CE +*/ +static BOOL winceUnlockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToUnlockLow, + DWORD nNumberOfBytesToUnlockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToUnlockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Releasing a reader lock or an exclusive lock */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST){ + /* Did we have an exclusive lock? */ + if (pFile->local.bExclusive){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE); + pFile->local.bExclusive = FALSE; + pFile->shared->bExclusive = FALSE; + bReturn = TRUE; + } + + /* Did we just have a reader lock? */ + else if (pFile->local.nReaders){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); + pFile->local.nReaders --; + if (pFile->local.nReaders == 0) + { + pFile->shared->nReaders --; + } + bReturn = TRUE; + } + } + + /* Releasing a pending lock */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bPending){ + pFile->local.bPending = FALSE; + pFile->shared->bPending = FALSE; + bReturn = TRUE; + } + } + /* Releasing a reserved lock */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bReserved) { + pFile->local.bReserved = FALSE; + pFile->shared->bReserved = FALSE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} +/* +** End of the special code for wince +*****************************************************************************/ +#endif /* SQLITE_OS_WINCE */ + +/* +** Lock a file region. +*/ +static BOOL winLockFile( + LPHANDLE phFile, + DWORD flags, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFile. + */ + return winceLockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/* +** Unlock a file region. + */ +static BOOL winUnlockFile( + LPHANDLE phFile, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API UnlockFile. + */ + return winceUnlockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +#if !SQLITE_OS_WINRT + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + DWORD lastErrno; /* Value returned by GetLastError() */ + + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occurred, it is also necessary to call + ** GetLastError(). + */ + dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + + if( (dwRet==INVALID_SET_FILE_POINTER + && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#else + /* + ** Same as above, except that this implementation works for WinRT. + */ + + LARGE_INTEGER x; /* The new offset */ + BOOL bRet; /* Value returned by SetFilePointerEx() */ + + x.QuadPart = iOffset; + bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + + if(!bRet){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS helper methods used for memory mapped files */ +static int winMapfile(winFile*, sqlite3_int64); +static int winUnmapfile(winFile*); +#endif + +/* +** Close a file. +** +** It is reported that an attempt to close a handle might sometimes +** fail. This is a very unreasonable result, but Windows is notorious +** for being unreasonable so I do not doubt that it might happen. If +** the close fails, we pause for 100 milliseconds and try again. As +** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before +** giving up and returning an error. +*/ +#define MX_CLOSE_ATTEMPT 3 +static int winClose(sqlite3_file *id){ + int rc, cnt = 0; + winFile *pFile = (winFile*)id; + + assert( id!=0 ); +#ifndef SQLITE_OMIT_WAL + assert( pFile->pShm==0 ); +#endif + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p\n", + osGetCurrentProcessId(), pFile, pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + winUnmapfile(pFile); +#endif + + do{ + rc = osCloseHandle(pFile->h); + /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); +#if SQLITE_OS_WINCE +#define WINCE_DELETION_ATTEMPTS 3 + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } + if( pFile->zDeleteOnClose ){ + int cnt = 0; + while( + osDeleteFileW(pFile->zDeleteOnClose)==0 + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && cnt++ < WINCE_DELETION_ATTEMPTS + ){ + sqlite3_win32_sleep(100); /* Wait a little before trying again */ + } + sqlite3_free(pFile->zDeleteOnClose); + } +#endif + if( rc ){ + pFile->h = NULL; + } + OpenCounter(-1); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, rc ? "ok" : "failed")); + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), + "winClose", pFile->zPath); +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int winRead( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ + + assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); + SimulateIOError(return SQLITE_IOERR_READ); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( winSeekFile(pFile, offset) ){ + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_FULL; + } + while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif + DWORD lastErrno; + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + pFile->lastErrno = lastErrno; + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, + "winRead", pFile->zPath); + } + winLogIoerr(nRetry, __LINE__); + if( nRead<(DWORD)amt ){ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_SHORT_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_IOERR_SHORT_READ; + } + + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +} + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int winWrite( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ +){ + int rc = 0; /* True if error has occurred, else false */ + winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ + + assert( amt>0 ); + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_WRITE); + SimulateDiskfullError(return SQLITE_FULL); + + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + rc = winSeekFile(pFile, offset); + if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ + +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + + while( nRem>0 ){ +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + break; + } + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ + lastErrno = osGetLastError(); + break; + } +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = lastErrno; + rc = 1; + } + } + + if( rc ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_FULL, pFile->lastErrno, + "winWrite1", pFile->zPath); + } + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_WRITE\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, + "winWrite2", pFile->zPath); + }else{ + winLogIoerr(nRetry, __LINE__); + } + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +} + +/* +** Truncate an open file to a specified size +*/ +static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_int64 oldMmapSize; + if( pFile->nFetchOut>0 ){ + /* File truncation is a no-op if there are outstanding memory mapped + ** pages. This is because truncating the file means temporarily unmapping + ** the file, and that might delete memory out from under existing cursors. + ** + ** This can result in incremental vacuum not truncating the file, + ** if there is an active read cursor when the incremental vacuum occurs. + ** No real harm comes of this - the database file is not corrupted, + ** though some folks might complain that the file is bigger than it + ** needs to be. + ** + ** The only feasible work-around is to defer the truncation until after + ** all references to memory-mapped content are closed. That is doable, + ** but involves adding a few branches in the common write code path which + ** could slow down normal operations slightly. Hence, we have decided for + ** now to simply make trancations a no-op if there are pending reads. We + ** can maybe revisit this decision in the future. + */ + return SQLITE_OK; + } +#endif + + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, size=%lld, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, nByte, pFile->locktype)); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + oldMmapSize = pFile->mmapSize; + }else{ + oldMmapSize = 0; + } + winUnmapfile(pFile); +#endif + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( winSeekFile(pFile, nByte) ){ + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate2", pFile->zPath); + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( rc==SQLITE_OK && oldMmapSize>0 ){ + if( oldMmapSize>nByte ){ + winMapfile(pFile, -1); + }else{ + winMapfile(pFile, oldMmapSize); + } + } +#endif + + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, sqlite3ErrName(rc))); + return rc; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. +*/ +int sqlite3_sync_count = 0; +int sqlite3_fullsync_count = 0; +#endif + +/* +** Make sure all writes to a particular file are committed to disk. +*/ +static int winSync(sqlite3_file *id, int flags){ +#ifndef SQLITE_NO_SYNC + /* + ** Used only when SQLITE_NO_SYNC is not defined. + */ + BOOL rc; +#endif +#if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ + defined(SQLITE_HAVE_OS_TRACE) + /* + ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or + ** OSTRACE() macros. + */ + winFile *pFile = (winFile*)id; +#else + UNUSED_PARAMETER(id); +#endif + + assert( pFile ); + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, flags=%x, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, flags, + pFile->locktype)); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + OSTRACE(("SYNC-NOP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +#else +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + if( osFlushViewOfFile(pFile->pMapRegion, 0) ){ + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_OK\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winSync1", pFile->zPath); + } + } +#endif + rc = osFlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_FSYNC\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, + "winSync2", pFile->zPath); + } +#endif +} + +/* +** Determine the current size of a file in bytes +*/ +static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + + assert( id!=0 ); + assert( pSize!=0 ); + SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + +#if SQLITE_OS_WINRT + { + FILE_STANDARD_INFO info; + if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, + &info, sizeof(info)) ){ + *pSize = info.EndOfFile.QuadPart; + }else{ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#else + { + DWORD upperBits; + DWORD lowerBits; + DWORD lastErrno; + + lowerBits = osGetFileSize(pFile->h, &upperBits); + *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( (lowerBits == INVALID_FILE_SIZE) + && ((lastErrno = osGetLastError())!=NO_ERROR) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); + return rc; +} + +/* +** LOCKFILE_FAIL_IMMEDIATELY is undefined on some Windows systems. +*/ +#ifndef LOCKFILE_FAIL_IMMEDIATELY +# define LOCKFILE_FAIL_IMMEDIATELY 1 +#endif + +#ifndef LOCKFILE_EXCLUSIVE_LOCK +# define LOCKFILE_EXCLUSIVE_LOCK 2 +#endif + +/* +** Historically, SQLite has used both the LockFile and LockFileEx functions. +** When the LockFile function was used, it was always expected to fail +** immediately if the lock could not be obtained. Also, it always expected to +** obtain an exclusive lock. These flags are used with the LockFileEx function +** and reflect those expectations; therefore, they should not be changed. +*/ +#ifndef SQLITE_LOCKFILE_FLAGS +# define SQLITE_LOCKFILE_FLAGS (LOCKFILE_FAIL_IMMEDIATELY | \ + LOCKFILE_EXCLUSIVE_LOCK) +#endif + +/* +** Currently, SQLite never calls the LockFileEx function without wanting the +** call to fail immediately if the lock cannot be obtained. +*/ +#ifndef SQLITE_LOCKFILEEX_FLAGS +# define SQLITE_LOCKFILEEX_FLAGS (LOCKFILE_FAIL_IMMEDIATELY) +#endif + +/* +** Acquire a reader lock. +** Different API routines are called depending on whether or not this +** is Win9x or WinNT. +*/ +static int winGetReadLock(winFile *pFile){ + int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFileEx. + */ + res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); +#else + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + int lk; + sqlite3_randomness(sizeof(lk), &lk); + pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res == 0 ){ + pFile->lastErrno = osGetLastError(); + /* No need to log a failure to lock */ + } + OSTRACE(("READ-LOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Undo a readlock +*/ +static int winUnlockReadLock(winFile *pFile){ + int res; + DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ + res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + res = winUnlockFile(&pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res==0 && ((lastErrno = osGetLastError())!=ERROR_NOT_LOCKED) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, + "winUnlockReadLock", pFile->zPath); + } + OSTRACE(("READ-UNLOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. The winUnlock() routine +** erases all locks at once and returns us immediately to locking level 0. +** It is not possible to lower the locking level one step at a time. You +** must go straight to locking level 0. +*/ +static int winLock(sqlite3_file *id, int locktype){ + int rc = SQLITE_OK; /* Return code from subroutines */ + int res = 1; /* Result of a Windows lock call */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ + int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ + winFile *pFile = (winFile*)id; + DWORD lastErrno = NO_ERROR; + + assert( id!=0 ); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + + /* If there is already a lock of this type or more restrictive on the + ** OsFile, do nothing. Don't use the end_lock: exit path, as + ** sqlite3OsEnterMutex() hasn't been called yet. + */ + if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + + /* Do not allow any kind of write-lock on a read-only database + */ + if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ + return SQLITE_IOERR_LOCK; + } + + /* Make sure the locking sequence is correct + */ + assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); + assert( locktype!=PENDING_LOCK ); + assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); + + /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of + ** the PENDING_LOCK byte is temporary. + */ + newLocktype = pFile->locktype; + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + ){ + int cnt = 3; + while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + PENDING_BYTE, 0, 1, 0))==0 ){ + /* Try 3 times to get the pending lock. This is needed to work + ** around problems caused by indexing and/or anti-virus software on + ** Windows systems. + ** If you are using this code as a model for alternative VFSes, do not + ** copy this retry logic. It is a hack intended for Windows only. + */ + lastErrno = osGetLastError(); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res)); + if( lastErrno==ERROR_INVALID_HANDLE ){ + pFile->lastErrno = lastErrno; + rc = SQLITE_IOERR_LOCK; + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc))); + return rc; + } + if( cnt ) sqlite3_win32_sleep(1); + } + gotPendingLock = res; + if( !res ){ + lastErrno = osGetLastError(); + } + } + + /* Acquire a shared lock + */ + if( locktype==SHARED_LOCK && res ){ + assert( pFile->locktype==NO_LOCK ); + res = winGetReadLock(pFile); + if( res ){ + newLocktype = SHARED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a RESERVED lock + */ + if( locktype==RESERVED_LOCK && res ){ + assert( pFile->locktype==SHARED_LOCK ); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + if( res ){ + newLocktype = RESERVED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a PENDING lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + newLocktype = PENDING_LOCK; + gotPendingLock = 0; + } + + /* Acquire an EXCLUSIVE lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + assert( pFile->locktype>=SHARED_LOCK ); + res = winUnlockReadLock(pFile); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); + if( res ){ + newLocktype = EXCLUSIVE_LOCK; + }else{ + lastErrno = osGetLastError(); + winGetReadLock(pFile); + } + } + + /* If we are holding a PENDING lock that ought to be released, then + ** release it now. + */ + if( gotPendingLock && locktype==SHARED_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + + /* Update the state of the lock has held in the file descriptor then + ** return the appropriate result code. + */ + if( res ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = lastErrno; + rc = SQLITE_BUSY; + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); + } + pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, return +** non-zero, otherwise zero. +*/ +static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ + int res; + winFile *pFile = (winFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); + + assert( id!=0 ); + if( pFile->locktype>=RESERVED_LOCK ){ + res = 1; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); + }else{ + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0); + if( res ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + res = !res; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (remote)\n", pFile->h, res)); + } + *pResOut = res; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Lower the locking level on file descriptor id to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** It is not possible for this routine to fail if the second argument +** is NO_LOCK. If the second argument is SHARED_LOCK then this routine +** might return SQLITE_IOERR; +*/ +static int winUnlock(sqlite3_file *id, int locktype){ + int type; + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + assert( pFile!=0 ); + assert( locktype<=SHARED_LOCK ); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + type = pFile->locktype; + if( type>=EXCLUSIVE_LOCK ){ + winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + /* This should never happen. We should always be able to + ** reacquire the read lock */ + rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), + "winUnlock", pFile->zPath); + } + } + if( type>=RESERVED_LOCK ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + if( locktype==NO_LOCK && type>=SHARED_LOCK ){ + winUnlockReadLock(pFile); + } + if( type>=PENDING_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + +/* +** If *pArg is initially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward references to VFS helper methods used for temporary files */ +static int winGetTempname(sqlite3_vfs *, char **); +static int winIsDir(const void *); +static BOOL winIsLongPathPrefix(const char *); +static BOOL winIsDriveLetterAndColon(const char *); + +/* +** Control and query of the open file handle. +*/ +static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + if( pFile->szChunk>0 ){ + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_PERSIST_WAL: { + winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + winIoerrRetry = a[0]; + }else{ + a[0] = winIoerrRetry; + } + if( a[1]>0 ){ + winIoerrRetryDelay = a[1]; + }else{ + a[1] = winIoerrRetryDelay; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#ifdef SQLITE_TEST + case SQLITE_FCNTL_WIN32_SET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + HANDLE hOldFile = pFile->h; + pFile->h = *phFile; + *phFile = hOldFile; + OSTRACE(("FCNTL oldFile=%p, newFile=%p, rc=SQLITE_OK\n", + hOldFile, pFile->h)); + return SQLITE_OK; + } +#endif + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = 0; + int rc = winGetTempname(pFile->pVfs, &zTFile); + if( rc==SQLITE_OK ){ + *(char**)pArg = zTFile; + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + + /* The value of newLimit may be eventually cast to (SIZE_T) and passed + ** to MapViewOfFile(). Restrict its value to 2GB if (SIZE_T) is not at + ** least a 64-bit type. */ + if( newLimit>0 && sizeof(SIZE_T)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + winUnmapfile(pFile); + rc = winMapfile(pFile, -1); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#endif + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); + return SQLITE_NOTFOUND; +} + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int winSectorSize(sqlite3_file *id){ + (void)id; + return SQLITE_DEFAULT_SECTOR_SIZE; +} + +/* +** Return a vector of device characteristics. +*/ +static int winDeviceCharacteristics(sqlite3_file *id){ + winFile *p = (winFile*)id; + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); +} + +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +static SYSTEM_INFO winSysInfo; + +#ifndef SQLITE_OMIT_WAL + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the winLockInfo objects used by +** this file, all of which may be shared by multiple threads. +** +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** winShmEnterMutex() +** assert( winShmMutexHeld() ); +** winShmLeaveMutex() +*/ +static sqlite3_mutex *winBigLock = 0; +static void winShmEnterMutex(void){ + sqlite3_mutex_enter(winBigLock); +} +static void winShmLeaveMutex(void){ + sqlite3_mutex_leave(winBigLock); +} +#ifndef NDEBUG +static int winShmMutexHeld(void) { + return sqlite3_mutex_held(winBigLock); +} +#endif + +/* +** Object used to represent a single file opened and mmapped to provide +** shared memory. When multiple threads all reference the same +** log-summary, each thread has its own winFile object, but they all +** point to a single instance of this object. In other words, each +** log-summary is opened only once per process. +** +** winShmMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** winShmMutexHeld() is true when reading or writing any other field +** in this structure. +** +*/ +struct winShmNode { + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the file */ + winFile hFile; /* File handle from winOpen */ + + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + + struct ShmRegion { + HANDLE hMap; /* File handle from CreateFileMapping */ + void *pMap; + } *aRegion; + DWORD lastErrno; /* The Windows errno from the last I/O error */ + + int nRef; /* Number of winShm objects pointing to this */ + winShm *pFirst; /* All winShm objects pointing to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + u8 nextShmId; /* Next available winShm.id value */ +#endif +}; + +/* +** A global array of all winShmNode objects. +** +** The winShmMutexHeld() must be true while reading or writing this list. +*/ +static winShmNode *winShmNodeList = 0; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** winShm.pShmNode +** winShm.id +** +** All other fields are read/write. The winShm.pShmNode->mutex must be held +** while accessing any read/write fields. +*/ +struct winShm { + winShmNode *pShmNode; /* The underlying winShmNode object */ + winShm *pNext; /* Next winShm with the same winShmNode */ + u8 hasMutex; /* True if holding the winShmNode mutex */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + u8 id; /* Id of this connection with its winShmNode */ +#endif +}; + +/* +** Constants used for locking +*/ +#define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Apply advisory locks for all n bytes beginning at ofst. +*/ +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 +static int winShmSystemLock( + winShmNode *pFile, /* Apply locks to this open shared-memory segment */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ + int ofst, /* Offset to first byte to be locked/unlocked */ + int nByte /* Number of bytes to lock or unlock */ +){ + int rc = 0; /* Result code form Lock/UnlockFileEx() */ + + /* Access to the winShmNode object is serialized by the caller */ + assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); + + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + + /* Release/Acquire the system-level lock */ + if( lockType==WINSHM_UNLCK ){ + rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); + }else{ + /* Initialize the locking parameters */ + DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); + } + + if( rc!= 0 ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + rc = SQLITE_BUSY; + } + + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); + + return rc; +} + +/* Forward references to VFS methods */ +static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); +static int winDelete(sqlite3_vfs *,const char*,int); + +/* +** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. +*/ +static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ + winShmNode **pp; + winShmNode *p; + assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); + pp = &winShmNodeList; + while( (p = *pp)!=0 ){ + if( p->nRef==0 ){ + int i; + if( p->mutex ){ sqlite3_mutex_free(p->mutex); } + for(i=0; inRegion; i++){ + BOOL bRc = osUnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + bRc = osCloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + } + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ + SimulateIOErrorBenign(1); + winClose((sqlite3_file *)&p->hFile); + SimulateIOErrorBenign(0); + } + if( deleteFlag ){ + SimulateIOErrorBenign(1); + sqlite3BeginBenignMalloc(); + winDelete(pVfs, p->zFilename, 0); + sqlite3EndBenignMalloc(); + SimulateIOErrorBenign(0); + } + *pp = p->pNext; + sqlite3_free(p->aRegion); + sqlite3_free(p); + }else{ + pp = &p->pNext; + } + } +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winShmNode *pShmNode){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + + if( rc==SQLITE_OK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); +} + +/* +** Open the shared-memory area associated with database file pDbFd. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +*/ +static int winOpenSharedMemory(winFile *pDbFd){ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ + + assert( pDbFd->pShm==0 ); /* Not previously opened */ + + /* Allocate space for the new sqlite3_shm object. Also speculatively + ** allocate space for a new winShmNode and filename. + */ + p = sqlite3MallocZero( sizeof(*p) ); + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; + nName = sqlite3Strlen30(pDbFd->zPath); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + if( pNew==0 ){ + sqlite3_free(p); + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + + /* Look to see if there is an existing winShmNode that can be used. + ** If no matching winShmNode currently exists, create a new one. + */ + winShmEnterMutex(); + for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ + /* TBD need to come up with better match here. Perhaps + ** use FILE_ID_BOTH_DIR_INFO Structure. + */ + if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; + } + if( pShmNode ){ + sqlite3_free(pNew); + }else{ + int inFlags = SQLITE_OPEN_WAL; + int outFlags = 0; + + pShmNode = pNew; + pNew = 0; + ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } + } + + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + }else{ + inFlags |= SQLITE_OPEN_READONLY; + } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + inFlags, &outFlags); + if( rc!=SQLITE_OK ){ + rc = winLogError(rc, osGetLastError(), "winOpenShm", + pShmNode->zFilename); + goto shm_open_err; + } + if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; + + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + } + + /* Make the new connection a child of the winShmNode */ + p->pShmNode = pShmNode; +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + winShmLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the winShmEnterMutex() mutex and the pointer from the + ** new (struct winShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; + + /* Jump here on any error */ +shm_open_err: + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ + sqlite3_free(p); + sqlite3_free(pNew); + winShmLeaveMutex(); + return rc; +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +*/ +static int winShmUnmap( + sqlite3_file *fd, /* Database holding shared memory */ + int deleteFlag /* Delete after closing if true */ +){ + winFile *pDbFd; /* Database holding shared-memory */ + winShm *p; /* The connection to be closed */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + winShm **pp; /* For looping over sibling connections */ + + pDbFd = (winFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + winShmEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + winShmPurge(pDbFd->pVfs, deleteFlag); + } + winShmLeaveMutex(); + + return SQLITE_OK; +} + +/* +** Change the lock state for a shared-memory segment. +*/ +static int winShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ + winShm *p = pDbFd->pShm; /* The shared memory being locked */ + winShm *pX; /* For looping over all siblings */ + winShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + + mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void winShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ + winShmLeaveMutex(); +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + winFile *pDbFd = (winFile*)fd; + winShm *pShm = pDbFd->pShm; + winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; + int rc = SQLITE_OK; + + if( !pShm ){ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + pShm = pDbFd->pShm; + assert( pShm!=0 ); + } + pShmNode = pShm->pShmNode; + + sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + + if( pShmNode->nRegion<=iRegion ){ + struct ShmRegion *apNew; /* New aRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + sqlite3_int64 sz; /* Current size of wal-index file */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap1", pDbFd->zPath); + goto shmpage_out; + } + + if( szhFile, nByte); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap2", pDbFd->zPath); + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (struct ShmRegion *)sqlite3_realloc64( + pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shmpage_out; + } + pShmNode->aRegion = apNew; + + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } + + while( pShmNode->nRegion<=iRegion ){ + HANDLE hMap = NULL; /* file-mapping handle */ + void *pMap = 0; /* Mapped memory region */ + +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, + NULL, protect, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pShmNode->hFile.h, + NULL, protect, 0, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA + hMap = osCreateFileMappingA(pShmNode->hFile.h, + NULL, protect, 0, nByte, NULL + ); +#endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); + if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; +#if SQLITE_OS_WINRT + pMap = osMapViewOfFileFromApp(hMap, flags, + iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#else + pMap = osMapViewOfFile(hMap, flags, + 0, iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#endif + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + szRegion, pMap ? "ok" : "failed")); + } + if( !pMap ){ + pShmNode->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_SHMMAP, pShmNode->lastErrno, + "winShmMap3", pDbFd->zPath); + if( hMap ) osCloseHandle(hMap); + goto shmpage_out; + } + + pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; + pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; + pShmNode->nRegion++; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; + char *p = (char *)pShmNode->aRegion[iRegion].pMap; + *pp = (void *)&p[iOffsetShift]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; +} + +#else +# define winShmMap 0 +# define winShmLock 0 +# define winShmBarrier 0 +# define winShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); +#ifdef SQLITE_MMAP_READWRITE + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#endif +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile1", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); +#else + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile2", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + assert( pFd->pMapRegion!=0 ); + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32. +*/ +static const sqlite3_io_methods winIoMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winLock, /* xLock */ + winUnlock, /* xUnlock */ + winCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +#if defined(__CYGWIN__) +/* +** Convert a filename from whatever the underlying operating system +** supports for filenames into UTF-8. Space to hold the result is +** obtained from malloc and must be freed by the calling function. +*/ +static char *winConvertToUtf8Filename(const void *zFilename){ + char *zConverted = 0; + if( osIsNT() ){ + zConverted = winUnicodeToUtf8(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} +#endif + +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ + zConverted = winUtf8ToUnicode(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + +/* +** This function returns non-zero if the specified UTF-8 string buffer +** ends with a directory separator character or one was successfully +** added to it. +*/ +static int winMakeEndInDirSep(int nBuf, char *zBuf){ + if( zBuf ){ + int nLen = sqlite3Strlen30(zBuf); + if( nLen>0 ){ + if( winIsDirSep(zBuf[nLen-1]) ){ + return 1; + }else if( nLen+1mxPathname; nBuf = nMax + 2; + zBuf = sqlite3MallocZero( nBuf ); + if( !zBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM_BKPT; + } + + /* Figure out the effective temporary directory. First, check if one + ** has been explicitly set by the application; otherwise, use the one + ** configured by the operating system. + */ + nDir = nMax - (nPre + 15); + assert( nDir>0 ); + if( sqlite3_temp_directory ){ + int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); + if( nDirLen>0 ){ + if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ + nDirLen++; + } + if( nDirLen>nDir ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); + } + sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); + } + } +#if defined(__CYGWIN__) + else{ + static const char *azDirs[] = { + 0, /* getenv("SQLITE_TMPDIR") */ + 0, /* getenv("TMPDIR") */ + 0, /* getenv("TMP") */ + 0, /* getenv("TEMP") */ + 0, /* getenv("USERPROFILE") */ + "/var/tmp", + "/usr/tmp", + "/tmp", + ".", + 0 /* List terminator */ + }; + unsigned int i; + const char *zDir = 0; + + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = getenv("TMP"); + if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + for(i=0; i/etilqs_XXXXXXXXXXXXXXX\0\0" + ** + ** If not, return SQLITE_ERROR. The number 17 is used here in order to + ** account for the space used by the 15 character random suffix and the + ** two trailing NUL characters. The final directory separator character + ** has already added if it was not already present. + */ + nLen = sqlite3Strlen30(zBuf); + if( (nLen + nPre + 17) > nBuf ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname5", 0); + } + + sqlite3_snprintf(nBuf-16-nLen, zBuf+nLen, SQLITE_TEMP_FILE_PREFIX); + + j = sqlite3Strlen30(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + zBuf[j+1] = 0; + *pzBuf = zBuf; + + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; +} + +/* +** Return TRUE if the named file is really a directory. Return false if +** it is something other than a directory, or if there is any kind of memory +** allocation failure. +*/ +static int winIsDir(const void *zConverted){ + DWORD attr; + int rc = 0; + DWORD lastErrno; + + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( !rc ){ + return 0; /* Invalid name? */ + } + attr = sAttrData.dwFileAttributes; +#if SQLITE_OS_WINCE==0 + }else{ + attr = osGetFileAttributesA((char*)zConverted); +#endif + } + return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); +} + +/* forward reference */ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +); + +/* +** Open a file. +*/ +static int winOpen( + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HANDLE h; + DWORD lastErrno = 0; + DWORD dwDesiredAccess; + DWORD dwShareMode; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = 0; +#if SQLITE_OS_WINCE + int isTemp = 0; +#endif + winVfsAppData *pAppData; + winFile *pFile = (winFile*)id; + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + int cnt = 0; + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char *zTmpname = 0; /* For temporary filename, if necessary. */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and super-journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; + +#if SQLITE_OS_WINRT + if( !zUtf8Name && !sqlite3_temp_directory ){ + sqlite3_log(SQLITE_ERROR, + "sqlite3_temp_directory variable should be set for WinRT"); + } +#endif + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + assert( isDelete && !isOpenJournal ); + rc = winGetTempname(pVfs, &zTmpname); + if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); + return rc; + } + zUtf8Name = zTmpname; + } + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). + */ + assert( (eType!=SQLITE_OPEN_MAIN_DB) || (flags & SQLITE_OPEN_URI) || + zUtf8Name[sqlite3Strlen30(zUtf8Name)+1]==0 ); + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8Name); + if( zConverted==0 ){ + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); + return SQLITE_IOERR_NOMEM_BKPT; + } + + if( winIsDir(zConverted) ){ + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); + return SQLITE_CANTOPEN_ISDIR; + } + + if( isReadWrite ){ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + }else{ + dwDesiredAccess = GENERIC_READ; + } + + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" + ** as it is usually understood. + */ + if( isExclusive ){ + /* Creates a new file, only if it does not already exist. */ + /* If the file exists, it fails. */ + dwCreationDisposition = CREATE_NEW; + }else if( isCreate ){ + /* Open existing file, or create if it doesn't exist */ + dwCreationDisposition = OPEN_ALWAYS; + }else{ + /* Opens a file, only if it exists. */ + dwCreationDisposition = OPEN_EXISTING; + } + + if( 0==sqlite3_uri_boolean(zName, "exclusive", 0) ){ + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + }else{ + dwShareMode = 0; + } + + if( isDelete ){ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; + isTemp = 1; +#else + dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY + | FILE_ATTRIBUTE_HIDDEN + | FILE_FLAG_DELETE_ON_CLOSE; +#endif + }else{ + dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + } + /* Reports from the internet are that performance is always + ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; +#endif + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); + extendedParameters.dwFileAttributes = + dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; + extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + extendedParameters.lpSecurityAttributes = NULL; + extendedParameters.hTemplateFile = NULL; + do{ + h = osCreateFile2((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, + dwCreationDisposition, + &extendedParameters); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); +#else + do{ + h = osCreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + do{ + h = osCreateFileA((LPCSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); + } +#endif + winLogIoerr(cnt, __LINE__); + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + + if( h==INVALID_HANDLE_VALUE ){ + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + if( isReadWrite && !isExclusive ){ + return winOpen(pVfs, zName, id, + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); + }else{ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); + return SQLITE_CANTOPEN_BKPT; + } + } + + if( pOutFlags ){ + if( isReadWrite ){ + *pOutFlags = SQLITE_OPEN_READWRITE; + }else{ + *pOutFlags = SQLITE_OPEN_READONLY; + } + } + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + + pAppData = (winVfsAppData*)pVfs->pAppData; + +#if SQLITE_OS_WINCE + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } + } + if( isTemp ){ + pFile->zDeleteOnClose = zConverted; + }else +#endif + { + sqlite3_free(zConverted); + } + + sqlite3_free(zTmpname); + id->pMethods = pAppData ? pAppData->pMethod : &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + + OpenCounter(+1); + return rc; +} + +/* +** Delete the named file. +** +** Note that Windows does not allow a file to be deleted if some other +** process has it open. Sometimes a virus scanner or indexing program +** will open a journal file shortly after it is created in order to do +** whatever it does. While this other process is holding the +** file open, we will be unable to delete it. To work around this +** problem, we delay 100 milliseconds and try to delete again. Up +** to MX_DELETION_ATTEMPTs deletion attempts are run before giving +** up and returning an error. +*/ +static int winDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + int cnt = 0; + int rc; + DWORD attr; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(syncDir); + + SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + do { +#if SQLITE_OS_WINRT + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, + &sAttrData) ){ + attr = sAttrData.dwFileAttributes; + }else{ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } +#else + attr = osGetFileAttributesW(zConverted); +#endif + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileW(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + do { + attr = osGetFileAttributesA(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileA(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#endif + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ + rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); + }else{ + winLogIoerr(cnt, __LINE__); + } + sqlite3_free(zConverted); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); + return rc; +} + +/* +** Check the existence and status of a file. +*/ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +){ + DWORD attr; + int rc = 0; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( rc ){ + /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file + ** as if it does not exist. + */ + if( flags==SQLITE_ACCESS_EXISTS + && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeLow==0 ){ + attr = INVALID_FILE_ATTRIBUTES; + }else{ + attr = sAttrData.dwFileAttributes; + } + }else{ + winLogIoerr(cnt, __LINE__); + if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", + zFilename); + }else{ + attr = INVALID_FILE_ATTRIBUTES; + } + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + attr = osGetFileAttributesA((char*)zConverted); + } +#endif + sqlite3_free(zConverted); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = attr!=INVALID_FILE_ATTRIBUTES; + break; + case SQLITE_ACCESS_READWRITE: + rc = attr!=INVALID_FILE_ATTRIBUTES && + (attr & FILE_ATTRIBUTE_READONLY)==0; + break; + default: + assert(!"Invalid flags argument"); + } + *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Returns non-zero if the specified path name starts with the "long path" +** prefix. +*/ +static BOOL winIsLongPathPrefix( + const char *zPathname +){ + return ( zPathname[0]=='\\' && zPathname[1]=='\\' + && zPathname[2]=='?' && zPathname[3]=='\\' ); +} + +/* +** Returns non-zero if the specified path name starts with a drive letter +** followed by a colon character. +*/ +static BOOL winIsDriveLetterAndColon( + const char *zPathname +){ + return ( sqlite3Isalpha(zPathname[0]) && zPathname[1]==':' ); +} + +/* +** Returns non-zero if the specified path name should be used verbatim. If +** non-zero is returned from this function, the calling function must simply +** use the provided path name verbatim -OR- resolve it into a full path name +** using the GetFullPathName Win32 API function (if available). +*/ +static BOOL winIsVerbatimPathname( + const char *zPathname +){ + /* + ** If the path name starts with a forward slash or a backslash, it is either + ** a legal UNC name, a volume relative path, or an absolute path name in the + ** "Unix" format on Windows. There is no easy way to differentiate between + ** the final two cases; therefore, we return the safer return value of TRUE + ** so that callers of this function will simply use it verbatim. + */ + if ( winIsDirSep(zPathname[0]) ){ + return TRUE; + } + + /* + ** If the path name starts with a letter and a colon it is either a volume + ** relative path or an absolute path. Callers of this function must not + ** attempt to treat it as a relative path name (i.e. they should simply use + ** it verbatim). + */ + if ( winIsDriveLetterAndColon(zPathname) ){ + return TRUE; + } + + /* + ** If we get to this point, the path name should almost certainly be a purely + ** relative one (i.e. not a UNC name, not absolute, and not volume relative). + */ + return FALSE; +} + +/* +** Turn a relative pathname into a full pathname. Write the full +** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname +** bytes in size. +*/ +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:" or "\\?\", where "X" is any + ** alphabetic character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && (winIsDriveLetterAndColon(zRelative+1) + || winIsLongPathPrefix(zRelative+1)) ){ + zRelative++; + } + +#if defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + assert( nFull>=pVfs->mxPathname ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a slash. + */ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | + CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname1", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM_BKPT; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + }else{ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), + zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname2", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM_BKPT; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + } + return SQLITE_OK; +#endif + +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + /* WinCE has no concept of a relative pathname, or so I am told. */ + /* WinRT has no way to convert a relative path to an absolute one. */ + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + }else{ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zRelative); + } + return SQLITE_OK; +#endif + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zRelative); + if( zConverted==0 ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + LPWSTR zTemp; + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname1", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM_BKPT; + } + nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname2", zRelative); + } + sqlite3_free(zConverted); + zOut = winUnicodeToUtf8(zTemp); + sqlite3_free(zTemp); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp; + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname3", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM_BKPT; + } + nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname4", zRelative); + } + sqlite3_free(zConverted); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + sqlite3_free(zTemp); + } +#endif + if( zOut ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + sqlite3_free(zOut); + return SQLITE_OK; + }else{ + return SQLITE_IOERR_NOMEM_BKPT; + } +#endif +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ + HANDLE h; +#if defined(__CYGWIN__) + int nFull = pVfs->mxPathname+1; + char *zFull = sqlite3MallocZero( nFull ); + void *zConverted = 0; + if( zFull==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ + sqlite3_free(zFull); + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + zConverted = winConvertFromUtf8Filename(zFull); + sqlite3_free(zFull); +#else + void *zConverted = winConvertFromUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); +#endif + if( zConverted==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( osIsNT() ){ +#if SQLITE_OS_WINRT + h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); +#else + h = osLoadLibraryW((LPCWSTR)zConverted); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + h = osLoadLibraryA((char*)zConverted); + } +#endif + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)h)); + sqlite3_free(zConverted); + return (void*)h; +} +static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ + UNUSED_PARAMETER(pVfs); + winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); +} +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ + FARPROC proc; + UNUSED_PARAMETER(pVfs); + proc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, address=%p\n", + (void*)pH, zSym, (void*)proc)); + return (void(*)(void))proc; +} +static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ + UNUSED_PARAMETER(pVfs); + osFreeLibrary((HANDLE)pHandle); + OSTRACE(("DLCLOSE handle=%p\n", (void*)pHandle)); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define winDlOpen 0 + #define winDlError 0 + #define winDlSym 0 + #define winDlClose 0 +#endif + +/* State information for the randomness gatherer. */ +typedef struct EntropyGatherer EntropyGatherer; +struct EntropyGatherer { + unsigned char *a; /* Gather entropy into this buffer */ + int na; /* Size of a[] in bytes */ + int i; /* XOR next input into a[i] */ + int nXor; /* Number of XOR operations done */ +}; + +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) +/* Mix sz bytes of entropy into p. */ +static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ + int j, k; + for(j=0, k=p->i; ja[k++] ^= x[j]; + if( k>=p->na ) k = 0; + } + p->i = k; + p->nXor += sz; +} +#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ + +/* +** Write up to nBuf bytes of randomness into zBuf. +*/ +static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); + return nBuf; +#else + EntropyGatherer e; + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); + e.a = (unsigned char*)zBuf; + e.na = nBuf; + e.nXor = 0; + e.i = 0; + { + SYSTEMTIME x; + osGetSystemTime(&x); + xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); + } + { + DWORD pid = osGetCurrentProcessId(); + xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); + } +#if SQLITE_OS_WINRT + { + ULONGLONG cnt = osGetTickCount64(); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); + } +#else + { + DWORD cnt = osGetTickCount(); + xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); + } +#endif /* SQLITE_OS_WINRT */ + { + LARGE_INTEGER i; + osQueryPerformanceCounter(&i); + xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); + } +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { + UUID id; + memset(&id, 0, sizeof(UUID)); + osUuidCreate(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + memset(&id, 0, sizeof(UUID)); + osUuidCreateSequential(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + } +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ + return e.nXor>nBuf ? nBuf : e.nXor; +#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ +} + + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +static int winSleep(sqlite3_vfs *pVfs, int microsec){ + sqlite3_win32_sleep((microsec+999)/1000); + UNUSED_PARAMETER(pVfs); + return ((microsec+999)/1000)*1000; +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + */ + FILETIME ft; + static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; +#ifdef SQLITE_TEST + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#endif + /* 2^32 - to avoid use of LL and warnings in gcc */ + static const sqlite3_int64 max32BitValue = + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; + +#if SQLITE_OS_WINCE + SYSTEMTIME time; + osGetSystemTime(&time); + /* if SystemTimeToFileTime() fails, it returns zero. */ + if (!osSystemTimeToFileTime(&time,&ft)){ + return SQLITE_ERROR; + } +#else + osGetSystemTimeAsFileTime( &ft ); +#endif + + *piNow = winFiletimeEpoch + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(pVfs); + return SQLITE_OK; +} + +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ + int rc; + sqlite3_int64 i; + rc = winCurrentTimeInt64(pVfs, &i); + if( !rc ){ + *prNow = i/86400000.0; + } + return rc; +} + +/* +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on Windows (or errno and +** strerror_r() on Unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occurred within the calling +** thread. +** +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. +** +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: +** +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. +*/ +static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); + UNUSED_PARAMETER(pVfs); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; +} + +/* +** Initialize and deinitialize the operating system interface. +*/ +int sqlite3_os_init(void){ + static sqlite3_vfs winVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==80 ); + + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); +#if SQLITE_OS_WINRT + osGetNativeSystemInfo(&winSysInfo); +#else + osGetSystemInfo(&winSysInfo); +#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); + + sqlite3_vfs_register(&winVfs, 1); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathVfs, 0); +#endif + + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + + return SQLITE_OK; +} + +int sqlite3_os_end(void){ +#if SQLITE_OS_WINRT + if( sleepObj!=NULL ){ + osCloseHandle(sleepObj); + sleepObj = NULL; + } +#endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = 0; +#endif + + return SQLITE_OK; +} + +#endif /* SQLITE_OS_WIN */ diff --git a/third_party/sqlite3/pager.c b/third_party/sqlite3/pager.c new file mode 100644 index 000000000..88ad7272a --- /dev/null +++ b/third_party/sqlite3/pager.c @@ -0,0 +1,7709 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of the page cache subsystem or "pager". +** +** The pager is used to access a database disk file. It implements +** atomic commit and rollback through the use of a journal file that +** is separate from the database file. The pager also implements file +** locking to prevent two processes from writing the same database +** file simultaneously, or one process from reading the database while +** another is writing. +*/ +#ifndef SQLITE_OMIT_DISKIO +#include "third_party/sqlite3/sqliteInt.inc" +#include "third_party/sqlite3/wal.inc" +/* clang-format off */ + + +/******************* NOTES ON THE DESIGN OF THE PAGER ************************ +** +** This comment block describes invariants that hold when using a rollback +** journal. These invariants do not apply for journal_mode=WAL, +** journal_mode=MEMORY, or journal_mode=OFF. +** +** Within this comment block, a page is deemed to have been synced +** automatically as soon as it is written when PRAGMA synchronous=OFF. +** Otherwise, the page is not synced until the xSync method of the VFS +** is called successfully on the file containing the page. +** +** Definition: A page of the database file is said to be "overwriteable" if +** one or more of the following are true about the page: +** +** (a) The original content of the page as it was at the beginning of +** the transaction has been written into the rollback journal and +** synced. +** +** (b) The page was a freelist leaf page at the start of the transaction. +** +** (c) The page number is greater than the largest page that existed in +** the database file at the start of the transaction. +** +** (1) A page of the database file is never overwritten unless one of the +** following are true: +** +** (a) The page and all other pages on the same sector are overwriteable. +** +** (b) The atomic page write optimization is enabled, and the entire +** transaction other than the update of the transaction sequence +** number consists of a single page change. +** +** (2) The content of a page written into the rollback journal exactly matches +** both the content in the database when the rollback journal was written +** and the content in the database at the beginning of the current +** transaction. +** +** (3) Writes to the database file are an integer multiple of the page size +** in length and are aligned on a page boundary. +** +** (4) Reads from the database file are either aligned on a page boundary and +** an integer multiple of the page size in length or are taken from the +** first 100 bytes of the database file. +** +** (5) All writes to the database file are synced prior to the rollback journal +** being deleted, truncated, or zeroed. +** +** (6) If a super-journal file is used, then all writes to the database file +** are synced prior to the super-journal being deleted. +** +** Definition: Two databases (or the same database at two points it time) +** are said to be "logically equivalent" if they give the same answer to +** all queries. Note in particular the content of freelist leaf +** pages can be changed arbitrarily without affecting the logical equivalence +** of the database. +** +** (7) At any time, if any subset, including the empty set and the total set, +** of the unsynced changes to a rollback journal are removed and the +** journal is rolled back, the resulting database file will be logically +** equivalent to the database file at the beginning of the transaction. +** +** (8) When a transaction is rolled back, the xTruncate method of the VFS +** is called to restore the database file to the same size it was at +** the beginning of the transaction. (In some VFSes, the xTruncate +** method is a no-op, but that does not change the fact the SQLite will +** invoke it.) +** +** (9) Whenever the database file is modified, at least one bit in the range +** of bytes from 24 through 39 inclusive will be changed prior to releasing +** the EXCLUSIVE lock, thus signaling other connections on the same +** database to flush their caches. +** +** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less +** than one billion transactions. +** +** (11) A database file is well-formed at the beginning and at the conclusion +** of every transaction. +** +** (12) An EXCLUSIVE lock is held on the database file when writing to +** the database file. +** +** (13) A SHARED lock is held on the database file while reading any +** content out of the database file. +** +******************************************************************************/ + +/* +** Macros for troubleshooting. Normally turned off +*/ +#if 0 +int sqlite3PagerTrace=1; /* True to enable tracing */ +#define sqlite3DebugPrintf printf +#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } +#else +#define PAGERTRACE(X) +#endif + +/* +** The following two macros are used within the PAGERTRACE() macros above +** to print out file-descriptors. +** +** PAGERID() takes a pointer to a Pager struct as its argument. The +** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file +** struct as its argument. +*/ +#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd)) +#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd)) + +/* +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. +** +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ +** +** +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] +** +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] +** +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** +** +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occurred, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). +*/ +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of an OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + +/* +** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +** This could conceivably cause corruption following a power failure on +** such a system. This is currently an undocumented limit. +*/ +#define MAX_SECTOR_SIZE 0x10000 + + +/* +** An instance of the following structure is allocated for each active +** savepoint and statement transaction in the system. All such structures +** are stored in the Pager.aSavepoint[] array, which is allocated and +** resized using sqlite3Realloc(). +** +** When a savepoint is created, the PagerSavepoint.iHdrOffset field is +** set to 0. If a journal-header is written into the main journal while +** the savepoint is active, then iHdrOffset is set to the byte offset +** immediately following the last journal record written into the main +** journal before the journal-header. This is required during savepoint +** rollback (see pagerPlaybackSavepoint()). +*/ +typedef struct PagerSavepoint PagerSavepoint; +struct PagerSavepoint { + i64 iOffset; /* Starting offset in main journal */ + i64 iHdrOffset; /* See above */ + Bitvec *pInSavepoint; /* Set of pages in this savepoint */ + Pgno nOrig; /* Original number of pages in file */ + Pgno iSubRec; /* Index of first record in sub-journal */ + int bTruncateOnRelease; /* If stmt journal may be truncated on RELEASE */ +#ifndef SQLITE_OMIT_WAL + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ +#endif +}; + +/* +** Bits of the Pager.doNotSpill flag. See further description below. +*/ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ + +/* +** An open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: +** +** eState +** +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. +** +** eLock +** +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. +** +** changeCountDone +** +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. +** +** It is set to true when the change-counter field is updated, which +** can only happen if an exclusive lock is held on the database file. +** It is cleared (set to false) whenever an exclusive lock is +** relinquished on the database file. Each time a transaction is committed, +** The changeCountDone flag is inspected. If it is true, the work of +** updating the change-counter is omitted for the current transaction. +** +** This mechanism means that when running in exclusive mode, a connection +** need only update the change-counter once, for the first transaction +** committed. +** +** setSuper +** +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a super-journal name to be written into the +** journal file before it is synced to disk. +** +** Whether or not a journal file contains a super-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a super-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a super-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain super-journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** super-journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setSuper flag +** is cleared anyway (and the pager will move to ERROR state). +** +** doNotSpill +** +** This variables control the behavior of cache-spills (calls made by +** the pcache module to the pagerStress() routine to write cached data +** to the file-system in order to free up memory). +** +** When bits SPILLFLAG_OFF or SPILLFLAG_ROLLBACK of doNotSpill are set, +** writing to the database from pagerStress() is disabled altogether. +** The SPILLFLAG_ROLLBACK case is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF +** case is a user preference. +** +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from +** pagerStress() is permitted, but syncing the journal file is not. +** This flag is set by sqlite3PagerWrite() when the file-system sector-size +** is larger than the database page-size in order to prevent a journal sync +** from happening in between the journalling of two pages on the same sector. +** +** subjInMemory +** +** This is a boolean variable. If true, then any required sub-journal +** is opened as an in-memory journal file. If false, then in-memory +** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. +** +** syncFlags, walSyncFlags +** +** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). +** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode +** and contains the flags used to sync the checkpoint operations in the +** lower two bits, and sync flags used for transaction commits in the WAL +** file in bits 0x04 and 0x08. In other words, to get the correct sync flags +** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct +** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note +** that with synchronous=NORMAL in WAL mode, transaction commit is not synced +** meaning that the 0x04 and 0x08 bits are both zero. +*/ +struct Pager { + sqlite3_vfs *pVfs; /* OS functions to use for IO */ + u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ + u8 useJournal; /* Use a rollback journal on this file */ + u8 noSync; /* Do not sync the journal if true */ + u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 extraSync; /* sync directory after journal delete */ + u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 walSyncFlags; /* See description above */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ + u8 readOnly; /* True for a read-only database */ + u8 memDb; /* True to inhibit all file I/O */ + + /************************************************************************** + ** The following block contains those class members that change during + ** routine operation. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. + */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ + u8 changeCountDone; /* Set after incrementing the change-counter */ + u8 setSuper; /* Super-jrnl name is written into jrnl */ + u8 doNotSpill; /* Do not spill the cache when non-zero */ + u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ + int errCode; /* One of several kinds of errors */ + int nRec; /* Pages journalled since last j-header written */ + u32 cksumInit; /* Quasi-random value added to every checksum */ + u32 nSubRec; /* Number of records written to sub-journal */ + Bitvec *pInJournal; /* One bit for each page in the database file */ + sqlite3_file *fd; /* File descriptor for database */ + sqlite3_file *jfd; /* File descriptor for main journal */ + sqlite3_file *sjfd; /* File descriptor for sub-journal */ + i64 journalOff; /* Current write offset in the journal file */ + i64 journalHdr; /* Byte offset to previous journal header */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ + PagerSavepoint *aSavepoint; /* Array of active savepoints */ + int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 iDataVersion; /* Changes whenever database content changes */ + char dbFileVers[16]; /* Changes whenever database file changes */ + + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ + + u16 nExtra; /* Add this many bytes to each in-memory page */ + i16 nReserve; /* Number of unused bytes at end of each page */ + u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ + int pageSize; /* Number of bytes in a page */ + Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ + char *zFilename; /* Name of the database file */ + char *zJournal; /* Name of the journal file */ + int (*xBusyHandler)(void*); /* Function to call when busy */ + void *pBusyHandlerArg; /* Context argument for xBusyHandler */ + int aStat[4]; /* Total cache hits, misses, writes, spills */ +#ifdef SQLITE_TEST + int nRead; /* Database pages read */ +#endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ +#ifndef SQLITE_OMIT_WAL + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ + char *zWal; /* File name for write-ahead log */ +#endif +}; + +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 +#define PAGER_STAT_SPILL 3 + +/* +** The following global variables hold counters used for +** testing purposes only. These variables do not exist in +** a non-testing build. These variables are not thread-safe. +*/ +#ifdef SQLITE_TEST +int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ +int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ +int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ +# define PAGER_INCR(v) v++ +#else +# define PAGER_INCR(v) +#endif + + + +/* +** Journal files begin with the following magic string. The data +** was obtained from /dev/random. It is used only as a sanity check. +** +** Since version 2.8.0, the journal format contains additional sanity +** checking information. If the power fails while the journal is being +** written, semi-random garbage data might appear in the journal +** file after power is restored. If an attempt is then made +** to roll the journal back, the database could be corrupted. The additional +** sanity checking data is an attempt to discover the garbage in the +** journal and ignore it. +** +** The sanity checking information for the new journal format consists +** of a 32-bit checksum on each page of data. The checksum covers both +** the page number and the pPager->pageSize bytes of data for the page. +** This cksum is initialized to a 32-bit random value that appears in the +** journal file right after the header. The random initializer is important, +** because garbage data that appears at the end of a journal is likely +** data that was once in other files that have now been deleted. If the +** garbage data came from an obsolete journal file, the checksums might +** be correct. But by initializing the checksum to random value which +** is different for every journal, we minimize that risk. +*/ +static const unsigned char aJournalMagic[] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7, +}; + +/* +** The size of the of each page record in the journal is given by +** the following macro. +*/ +#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) + +/* +** The journal header size for this pager. This is usually the same +** size as a single disk sector. See also setSectorSize(). +*/ +#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) + +/* +** The macro MEMDB is true if we are dealing with an in-memory database. +** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set, +** the value of MEMDB will be a constant and the compiler will optimize +** out code that would never execute. +*/ +#ifdef SQLITE_OMIT_MEMORYDB +# define MEMDB 0 +#else +# define MEMDB pPager->memDb +#endif + +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods!=0) + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if page pgno can be read directly from the database file +** by the b-tree layer. This is the case if: +** +** * the database file is open, +** * there are no dirty pages in the cache, and +** * the desired page is not currently in the wal file. +*/ +int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +#ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; + int rc; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return (rc==SQLITE_OK && iRead==0); + } +#endif + return 1; +} +#endif + +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + +#ifndef NDEBUG +/* +** Usage: +** +** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. +*/ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; + + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); + + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( !isOpen(p->fd) ); + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setSuper==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); + break; + } + + return 1; +} +#endif /* ifndef NDEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +** +** This routine has external linkage in order to suppress compiler warnings +** about an unused function. It is enclosed within SQLITE_DEBUG and so does +** not appear in normal builds. +*/ +char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; +} +#endif + +/* Forward references to the various page getters */ +static int getPageNormal(Pager*,Pgno,DbPage**,int); +static int getPageError(Pager*,Pgno,DbPage**,int); +#if SQLITE_MAX_MMAP_SIZE>0 +static int getPageMMap(Pager*,Pgno,DbPage**,int); +#endif + +/* +** Set the Pager.xGet method for the appropriate routine used to fetch +** content from the pager. +*/ +static void setGetterMethod(Pager *pPager){ + if( pPager->errCode ){ + pPager->xGet = getPageError; +#if SQLITE_MAX_MMAP_SIZE>0 + }else if( USEFETCH(pPager) ){ + pPager->xGet = getPageMMap; +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ + pPager->xGet = getPageNormal; + } +} + +/* +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: +** +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. +*/ +static int subjRequiresPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + PagerSavepoint *p; + Pgno pgno = pPg->pgno; + int i; + for(i=0; inSavepoint; i++){ + p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ + for(i=i+1; inSavepoint; i++){ + pPager->aSavepoint[i].bTruncateOnRelease = 0; + } + return 1; + } + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** Return true if the page is already in the journal file. +*/ +static int pageInJournal(Pager *pPager, PgHdr *pPg){ + return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); +} +#endif + +/* +** Read a 32-bit integer from the given file descriptor. Store the integer +** that is read in *pRes. Return SQLITE_OK if everything worked, or an +** error code is something goes wrong. +** +** All values are stored on disk as big-endian. +*/ +static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ + unsigned char ac[4]; + int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); + if( rc==SQLITE_OK ){ + *pRes = sqlite3Get4byte(ac); + } + return rc; +} + +/* +** Write a 32-bit integer into a string buffer in big-endian byte order. +*/ +#define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + + +/* +** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK +** on success or an error code is something goes wrong. +*/ +static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ + char ac[4]; + put32bits(ac, val); + return sqlite3OsWrite(fd, ac, 4, offset); +} + +/* +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. +*/ +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + pPager->changeCountDone = pPager->tempFile; /* ticket fb3b3024ea238d5c */ + return rc; +} + +/* +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. +*/ +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } + } + return rc; +} + +/* +** This function determines whether or not the atomic-write or +** atomic-batch-write optimizations can be used with this pager. The +** atomic-write optimization can be used if: +** +** (a) the value returned by OsDeviceCharacteristics() indicates that +** a database page may be written atomically, and +** (b) the value returned by OsSectorSize() is less than or equal +** to the page size. +** +** If it can be used, then the value returned is the size of the journal +** file when it contains rollback data for exactly one page. +** +** The atomic-batch-write optimization can be used if OsDeviceCharacteristics() +** returns a value with the SQLITE_IOCAP_BATCH_ATOMIC bit set. -1 is +** returned in this case. +** +** If neither optimization can be used, 0 is returned. +*/ +static int jrnlBufferSize(Pager *pPager){ + assert( !MEMDB ); + +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int dc; /* Device characteristics */ + + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); +#else + UNUSED_PARAMETER(pPager); +#endif + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->dbSize>0 && (dc&SQLITE_IOCAP_BATCH_ATOMIC) ){ + return -1; + } +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int nSector = pPager->sectorSize; + int szPage = pPager->pageSize; + + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ + return 0; + } + } + + return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); +#endif + + return 0; +} + +/* +** If SQLITE_CHECK_PAGES is defined then we do some sanity checking +** on the cache using a hash function. This is used for testing +** and debugging only. +*/ +#ifdef SQLITE_CHECK_PAGES +/* +** Return a 32-bit hash of the page data for pPage. +*/ +static u32 pager_datahash(int nByte, unsigned char *pData){ + u32 hash = 0; + int i; + for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); +} +static void pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); +} + +/* +** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES +** is defined, and NDEBUG is not defined, an assert() statement checks +** that the page is either dirty or still matches the calculated page-hash. +*/ +#define CHECK_PAGE(x) checkPage(x) +static void checkPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); +} + +#else +#define pager_datahash(X,Y) 0 +#define pager_pagehash(X) 0 +#define pager_set_pagehash(X) +#define CHECK_PAGE(x) +#endif /* SQLITE_CHECK_PAGES */ + +/* +** When this is called the journal file for pager pPager must be open. +** This function attempts to read a super-journal file name from the +** end of the file and, if successful, copies it into memory supplied +** by the caller. See comments above writeSuperJournal() for the format +** used to store a super-journal file name at the end of a journal file. +** +** zSuper must point to a buffer of at least nSuper bytes allocated by +** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is +** enough space to write the super-journal name). If the super-journal +** name in the journal is longer than nSuper bytes (including a +** nul-terminator), then this is handled as if no super-journal name +** were present in the journal. +** +** If a super-journal file name is present at the end of the journal +** file, then it is copied into the buffer pointed to by zSuper. A +** nul-terminator byte is appended to the buffer following the +** super-journal file name. +** +** If it is determined that no super-journal file name is present +** zSuper[0] is set to 0 and SQLITE_OK returned. +** +** If an error occurs while reading from the journal file, an SQLite +** error code is returned. +*/ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ + int rc; /* Return code */ + u32 len; /* Length in bytes of super-journal name */ + i64 szJ; /* Total size in bytes of journal file pJrnl */ + u32 cksum; /* MJ checksum value read from journal */ + u32 u; /* Unsigned loop counter */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + zSuper[0] = '\0'; + + if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) + || szJ<16 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) + || len>=nSuper + || len>szJ-16 + || len==0 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) + || memcmp(aMagic, aJournalMagic, 8) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zSuper, len, szJ-16-len)) + ){ + return rc; + } + + /* See if the checksum matches the super-journal name */ + for(u=0; ujournalOff, assuming a sector +** size of pPager->sectorSize bytes. +** +** i.e for a sector size of 512: +** +** Pager.journalOff Return value +** --------------------------------------- +** 0 0 +** 512 512 +** 100 512 +** 2000 2048 +** +*/ +static i64 journalHdrOffset(Pager *pPager){ + i64 offset = 0; + i64 c = pPager->journalOff; + if( c ){ + offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager); + } + assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); + assert( offset>=c ); + assert( (offset-c)jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); + if( pPager->journalOff ){ + const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ + + IOTRACE(("JZEROHDR %p\n", pPager)) + if( doTruncate || iLimit==0 ){ + rc = sqlite3OsTruncate(pPager->jfd, 0); + }else{ + static const char zeroHdr[28] = {0}; + rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); + } + if( rc==SQLITE_OK && !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); + } + + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for + ** the persistent journal and the journal file currently consumes more + ** space than that limit allows for, truncate it now. There is no need + ** to sync the file following this operation. + */ + if( rc==SQLITE_OK && iLimit>0 ){ + i64 sz; + rc = sqlite3OsFileSize(pPager->jfd, &sz); + if( rc==SQLITE_OK && sz>iLimit ){ + rc = sqlite3OsTruncate(pPager->jfd, iLimit); + } + } + } + return rc; +} + +/* +** The journal file must be open when this routine is called. A journal +** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the +** current location. +** +** The format for the journal header is as follows: +** - 8 bytes: Magic identifying journal format. +** - 4 bytes: Number of records in journal, or -1 no-sync mode is on. +** - 4 bytes: Random number used for page hash. +** - 4 bytes: Initial database page count. +** - 4 bytes: Sector size used by the process that wrote this journal. +** - 4 bytes: Database page size. +** +** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. +*/ +static int writeJournalHdr(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ + u32 nWrite; /* Bytes of header sector written */ + int ii; /* Loop counter */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + if( nHeader>JOURNAL_HDR_SZ(pPager) ){ + nHeader = JOURNAL_HDR_SZ(pPager); + } + + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the + ** PagerSavepoint.iHdrOffset fields now. + */ + for(ii=0; iinSavepoint; ii++){ + if( pPager->aSavepoint[ii].iHdrOffset==0 ){ + pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; + } + } + + pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); + + /* + ** Write the nRec Field - the number of page records that follow this + ** journal header. Normally, zero is written to this value at this time. + ** After the records are added to the journal (and the journal synced, + ** if in full-sync mode), the zero is overwritten with the true number + ** of records (see syncJournal()). + ** + ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When + ** reading the journal this value tells SQLite to assume that the + ** rest of the journal file contains valid page records. This assumption + ** is dangerous, as if a failure occurred whilst writing to the journal + ** file it may contain some garbage data. There are two scenarios + ** where this risk can be ignored: + ** + ** * When the pager is in no-sync mode. Corruption can follow a + ** power failure in this case anyway. + ** + ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees + ** that garbage data is never appended to the journal file. + */ + assert( isOpen(pPager->fd) || pPager->noSync ); + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); + }else{ + memset(zHeader, 0, sizeof(aJournalMagic)+4); + } + + /* The random check-hash initializer */ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); + /* The assumed sector size for this process */ + put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); + + /* The page size */ + put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); + + /* Initializing the tail of the buffer is not necessary. Everything + ** works find if the following memset() is omitted. But initializing + ** the memory prevents valgrind from complaining, so we are willing to + ** take the performance hit. + */ + memset(&zHeader[sizeof(aJournalMagic)+20], 0, + nHeader-(sizeof(aJournalMagic)+20)); + + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + ** record is written to the following sector (leaving a gap in the file + ** that will be implicitly filled in by the OS). + ** + ** However it has been discovered that on some systems this pattern can + ** be significantly slower than contiguously writing data to the file, + ** even if that means explicitly writing data to the block of + ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what + ** is done. + ** + ** The loop is required here in case the sector-size is larger than the + ** database page size. Since the zHeader buffer is only Pager.pageSize + ** bytes in size, more than one call to sqlite3OsWrite() may be required + ** to populate the entire journal header sector. + */ + for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) + rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); + assert( pPager->journalHdr <= pPager->journalOff ); + pPager->journalOff += nHeader; + } + + return rc; +} + +/* +** The journal file must be open when this is called. A journal header file +** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal +** file. The current location in the journal file is given by +** pPager->journalOff. See comments above function writeJournalHdr() for +** a description of the journal header format. +** +** If the header is read successfully, *pNRec is set to the number of +** page records following this header and *pDbSize is set to the size of the +** database before the transaction began, in pages. Also, pPager->cksumInit +** is set to the value read from the journal header. SQLITE_OK is returned +** in this case. +** +** If the journal header file appears to be corrupted, SQLITE_DONE is +** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes +** cannot be read from the journal file an error code is returned. +*/ +static int readJournalHdr( + Pager *pPager, /* Pager object */ + int isHot, + i64 journalSize, /* Size of the open journal file in bytes */ + u32 *pNRec, /* OUT: Value read from the nRec field */ + u32 *pDbSize /* OUT: Value of original database size field */ +){ + int rc; /* Return code */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + i64 iHdrOff; /* Offset of journal header being read */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + /* Advance Pager.journalOff to the start of the next sector. If the + ** journal file is too small for there to be a header stored at this + ** point, return SQLITE_DONE. + */ + pPager->journalOff = journalHdrOffset(pPager); + if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ + return SQLITE_DONE; + } + iHdrOff = pPager->journalOff; + + /* Read in the first 8 bytes of the journal header. If they do not match + ** the magic string found at the start of each journal header, return + ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise, + ** proceed. + */ + if( isHot || iHdrOff!=pPager->journalHdr ){ + rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff); + if( rc ){ + return rc; + } + if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ + return SQLITE_DONE; + } + } + + /* Read the first three 32-bit fields of the journal header: The nRec + ** field, the checksum-initializer and the database size at the start + ** of the transaction. Return an error code if anything goes wrong. + */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize)) + ){ + return rc; + } + + if( pPager->journalOff==0 ){ + u32 iPageSize; /* Page-size field of journal header */ + u32 iSectorSize; /* Sector-size field of journal header */ + + /* Read the page-size and sector-size journal header fields. */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) + ){ + return rc; + } + + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + + /* Check that the values read from the page-size and sector-size fields + ** are within range. To be 'in range', both values need to be a power + ** of two greater than or equal to 512 or 32, and not greater than their + ** respective compile time maximum limits. + */ + if( iPageSize<512 || iSectorSize<32 + || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + ){ + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading + ** the journal file here. + */ + return SQLITE_DONE; + } + + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within + ** PagerSetPagesize() is tested. + */ + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); + testcase( rc!=SQLITE_OK ); + + /* Update the assumed sector-size to match the value used by + ** the process that created this journal. If this journal was + ** created by a process other than this one, then this routine + ** is being called from within pager_playback(). The local value + ** of Pager.sectorSize is restored at the end of that routine. + */ + pPager->sectorSize = iSectorSize; + } + + pPager->journalOff += JOURNAL_HDR_SZ(pPager); + return rc; +} + + +/* +** Write the supplied super-journal name into the journal file for pager +** pPager at the current location. The super-journal name must be the last +** thing written to a journal file. If the pager is in full-sync mode, the +** journal file descriptor is advanced to the next sector boundary before +** anything is written. The format is: +** +** + 4 bytes: PAGER_MJ_PGNO. +** + N bytes: super-journal filename in utf-8. +** + 4 bytes: N (length of super-journal name in bytes, no nul-terminator). +** + 4 bytes: super-journal name checksum. +** + 8 bytes: aJournalMagic[]. +** +** The super-journal page checksum is the sum of the bytes in thesuper-journal +** name, where each byte is interpreted as a signed 8-bit integer. +** +** If zSuper is a NULL pointer (occurs for a single database transaction), +** this call is a no-op. +*/ +static int writeSuperJournal(Pager *pPager, const char *zSuper){ + int rc; /* Return code */ + int nSuper; /* Length of string zSuper */ + i64 iHdrOff; /* Offset of header in journal file */ + i64 jrnlSize; /* Size of journal file on disk */ + u32 cksum = 0; /* Checksum of string zSuper */ + + assert( pPager->setSuper==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zSuper + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || !isOpen(pPager->jfd) + ){ + return SQLITE_OK; + } + pPager->setSuper = 1; + assert( pPager->journalHdr <= pPager->journalOff ); + + /* Calculate the length in bytes and the checksum of zSuper */ + for(nSuper=0; zSuper[nSuper]; nSuper++){ + cksum += zSuper[nSuper]; + } + + /* If in full-sync mode, advance to the next disk sector before writing + ** the super-journal name. This is in case the previous page written to + ** the journal has already been synced. + */ + if( pPager->fullSync ){ + pPager->journalOff = journalHdrOffset(pPager); + } + iHdrOff = pPager->journalOff; + + /* Write the super-journal data to the end of the journal file. If + ** an error occurs, return the error code to the caller. + */ + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, zSuper, nSuper, iHdrOff+4))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper, nSuper))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper+4, cksum))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, + iHdrOff+4+nSuper+8))) + ){ + return rc; + } + pPager->journalOff += (nSuper+20); + + /* If the pager is in peristent-journal mode, then the physical + ** journal-file may extend past the end of the super-journal name + ** and 8 bytes of magic data just written to the file. This is + ** dangerous because the code to rollback a hot-journal file + ** will not be able to find the super-journal name to determine + ** whether or not the journal is hot. + ** + ** Easiest thing to do in this scenario is to truncate the journal + ** file to the required size. + */ + if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) + && jrnlSize>pPager->journalOff + ){ + rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff); + } + return rc; +} + +/* +** Discard the entire contents of the in-memory page-cache. +*/ +static void pager_reset(Pager *pPager){ + pPager->iDataVersion++; + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); +} + +/* +** Return the pPager->iDataVersion value +*/ +u32 sqlite3PagerDataVersion(Pager *pPager){ + return pPager->iDataVersion; +} + +/* +** Free all structures in the Pager.aSavepoint[] array and set both +** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal +** if it is open and the pager is not in exclusive mode. +*/ +static void releaseAllSavepoints(Pager *pPager){ + int ii; /* Iterator for looping through Pager.aSavepoint */ + for(ii=0; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ + sqlite3OsClose(pPager->sjfd); + } + sqlite3_free(pPager->aSavepoint); + pPager->aSavepoint = 0; + pPager->nSavepoint = 0; + pPager->nSubRec = 0; +} + +/* +** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** bitvecs of all open savepoints. Return SQLITE_OK if successful +** or SQLITE_NOMEM if a malloc failure occurs. +*/ +static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ + int ii; /* Loop counter */ + int rc = SQLITE_OK; /* Result code */ + + for(ii=0; iinSavepoint; ii++){ + PagerSavepoint *p = &pPager->aSavepoint[ii]; + if( pgno<=p->nOrig ){ + rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + } + } + return rc; +} + +/* +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. +** +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). +*/ +static void pager_unlock(Pager *pPager){ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ + int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; + + /* If the operating system support deletion of open files, then + ** close the journal file when dropping the database lock. Otherwise + ** another connection with journal_mode=delete might delete the file + ** out from under us. + */ + assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 ); + assert( (PAGER_JOURNALMODE_OFF & 5)!=1 ); + assert( (PAGER_JOURNALMODE_WAL & 5)!=1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) + || 1!=(pPager->journalMode & 5) + ){ + sqlite3OsClose(pPager->jfd); + } + + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } + + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); + pPager->eState = PAGER_OPEN; + } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); + if( pPager->errCode ){ + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; + setGetterMethod(pPager); + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setSuper = 0; +} + +/* +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. +** +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. +** +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding +** the contents of the pager-cache. If a transaction was active when +** the persistent error occurred, then the rollback journal may need +** to be replayed to restore the contents of the database file (as if +** it were a hot-journal). +*/ +static int pager_error(Pager *pPager, int rc){ + int rc2 = rc & 0xff; + assert( rc==SQLITE_OK || !MEMDB ); + assert( + pPager->errCode==SQLITE_FULL || + pPager->errCode==SQLITE_OK || + (pPager->errCode & 0xff)==SQLITE_IOERR + ); + if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ + pPager->errCode = rc; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } + return rc; +} + +static int pager_truncate(Pager *pPager, Pgno nPage); + +/* +** The write transaction open on pPager is being committed (bCommit==1) +** or rolled back (bCommit==0). +** +** Return TRUE if and only if all dirty pages should be flushed to disk. +** +** Rules: +** +** * For non-TEMP databases, always sync to disk. This is necessary +** for transactions to be durable. +** +** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing +** file has been created already (via a spill on pagerStress()) and +** when the number of dirty pages in memory exceeds 25% of the total +** cache size. +*/ +static int pagerFlushOnCommit(Pager *pPager, int bCommit){ + if( pPager->tempFile==0 ) return 1; + if( !bCommit ) return 0; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + +/* +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called +** after rollback of a hot-journal, or if an error occurs while opening +** the journal file or writing the very first journal-header of a +** database transaction. +** +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. +** +** Otherwise, any active savepoints are released. +** +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a +** transaction. Nor will it be considered to be a hot-journal by this +** or any other database connection. Exactly how a journal is finalized +** depends on whether or not the pager is running in exclusive mode and +** the current journal-mode (Pager.journalMode value), as follows: +** +** journalMode==MEMORY +** Journal file descriptor is simply closed. This destroys an +** in-memory journal. +** +** journalMode==TRUNCATE +** Journal file is truncated to zero bytes in size. +** +** journalMode==PERSIST +** The first 28 bytes of the journal file are zeroed. This invalidates +** the first journal header in the file, and hence the entire journal +** file. An invalid journal file cannot be rolled back. +** +** journalMode==DELETE +** The journal file is closed and deleted using sqlite3OsDelete(). +** +** If the pager is running in exclusive mode, this method of finalizing +** the journal file is never used. Instead, if the journalMode is +** DELETE and the pager is in exclusive mode, the method described under +** journalMode==PERSIST is used instead. +** +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. +** +** SQLITE_OK is returned if no error occurs. If an error occurs during +** any of the IO operations to finalize the journal file or unlock the +** database then the IO error code is returned to the user. If the +** operation to finalize the journal file fails, then the code still +** tries to unlock the database file if not in exclusive mode. If the +** unlock operation fails as well, then the first error code related +** to the first error encountered (the journal finalization one) is +** returned. +*/ +static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ + int rc = SQLITE_OK; /* Error code from journal finalization operation */ + int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ + + /* Do nothing if the pager does not have an open write transaction + ** or at least a RESERVED lock. This function may be called when there + ** is no write-transaction active but a RESERVED or greater lock is + ** held under two circumstances: + ** + ** 1. After a successful hot-journal rollback, it is called with + ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. + ** + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** lock switches back to locking_mode=normal and then executes a + ** read-transaction, this function is called with eState==PAGER_READER + ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. + */ + assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + if( isOpen(pPager->jfd) ){ + assert( !pagerUseWal(pPager) ); + + /* Finalize the journal file. */ + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ + sqlite3OsClose(pPager->jfd); + }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ + if( pPager->journalOff==0 ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } + } + pPager->journalOff = 0; + }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST + || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + ){ + rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile); + pPager->journalOff = 0; + }else{ + /* This branch may be executed with Pager.journalMode==MEMORY if + ** a hot-journal was just rolled back. In this case the journal + ** file should be closed and deleted. If this connection writes to + ** the database file, it will do so using an in-memory journal. + */ + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + sqlite3OsClose(pPager->jfd); + if( bDelete ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); + } + } + } + +#ifdef SQLITE_CHECK_PAGES + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = sqlite3PagerLookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnrefNotNull(p); + } + } +#endif + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + pPager->nRec = 0; + if( rc==SQLITE_OK ){ + if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } + + if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ + rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); + assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); + } + + if( rc==SQLITE_OK && bCommit ){ + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + } + + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); + } + pPager->eState = PAGER_READER; + pPager->setSuper = 0; + + return (rc==SQLITE_OK?rc2:rc); +} + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0, 0); + } + } + pager_unlock(pPager); +} + +/* +** Parameter aData must point to a buffer of pPager->pageSize bytes +** of data. Compute and return a checksum based ont the contents of the +** page of data and the current value of pPager->cksumInit. +** +** This is not a real checksum. It is really just the sum of the +** random initial value (pPager->cksumInit) and every 200th byte +** of the page data, starting with byte offset (pPager->pageSize%200). +** Each byte is interpreted as an 8-bit unsigned integer. +** +** Changing the formula used to compute this checksum results in an +** incompatible journal file format. +** +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. +** It is much less likely that the two ends of the journal record will be +** correct and the middle be corrupt. Thus, this "checksum" scheme, +** though fast and simple, catches the mostly likely kind of corruption. +*/ +static u32 pager_cksum(Pager *pPager, const u8 *aData){ + u32 cksum = pPager->cksumInit; /* Checksum value to return */ + int i = pPager->pageSize-200; /* Loop counter */ + while( i>0 ){ + cksum += aData[i]; + i -= 200; + } + return cksum; +} + +/* +** Read a single page from either the journal file (if isMainJrnl==1) or +** from the sub-journal (if isMainJrnl==0) and playback that page. +** The page begins at offset *pOffset into the file. The *pOffset +** value is increased to the start of the next page in the journal. +** +** The main rollback journal uses checksums - the statement journal does +** not. +** +** If the page number of the page record read from the (sub-)journal file +** is greater than the current value of Pager.dbSize, then playback is +** skipped and SQLITE_OK is returned. +** +** If pDone is not NULL, then it is a record of pages that have already +** been played back. If the page at *pOffset has already been played back +** (if the corresponding pDone bit is set) then skip the playback. +** Make sure the pDone bit corresponding to the *pOffset page is set +** prior to returning. +** +** If the page record is successfully read from the (sub-)journal file +** and played back, then SQLITE_OK is returned. If an IO error occurs +** while reading the record from the (sub-)journal file or while writing +** to the database file, then the IO error code is returned. If data +** is successfully read from the (sub-)journal file but appears to be +** corrupted, SQLITE_DONE is returned. Data is considered corrupted in +** two circumstances: +** +** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** * If the record is being rolled back from the main journal file +** and the checksum field does not match the record content. +** +** Neither of these two scenarios are possible during a savepoint rollback. +** +** If this is a savepoint rollback, then memory may have to be dynamically +** allocated by this function. If this is the case and an allocation fails, +** SQLITE_NOMEM is returned. +*/ +static int pager_playback_one_page( + Pager *pPager, /* The pager being played back */ + i64 *pOffset, /* Offset of record to playback */ + Bitvec *pDone, /* Bitvec of pages already played back */ + int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ + int isSavepnt /* True for a savepoint rollback */ +){ + int rc; + PgHdr *pPg; /* An existing page in the cache */ + Pgno pgno; /* The page number of a page in journal */ + u32 cksum; /* Checksum used for sanity checking */ + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ + + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ + assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ + assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ + + aData = pPager->pTmpSpace; + assert( aData ); /* Temp storage must have already been allocated */ + assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + + /* Read the page number and page data from the journal or sub-journal + ** file. Return an error code to the caller if an IO error occurs. + */ + jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; + rc = read32bits(jfd, *pOffset, &pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); + if( rc!=SQLITE_OK ) return rc; + *pOffset += pPager->pageSize + 4 + isMainJrnl*4; + + /* Sanity checking on the page. This is more important that I originally + ** thought. If a power failure occurs while the journal is being written, + ** it could cause invalid data to be written into the journal. We need to + ** detect this invalid data (with high probability) and ignore it. + */ + if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + assert( !isSavepnt ); + return SQLITE_DONE; + } + if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ + return SQLITE_OK; + } + if( isMainJrnl ){ + rc = read32bits(jfd, (*pOffset)-4, &cksum); + if( rc ) return rc; + if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ + return SQLITE_DONE; + } + } + + /* If this page has already been played back before during the current + ** rollback, then don't bother to play it back again. + */ + if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ + return rc; + } + + /* When playing back page 1, restore the nReserve setting + */ + if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ + pPager->nReserve = ((u8*)aData)[20]; + } + + /* If the pager is in CACHEMOD state, then there must be a copy of this + ** page in the pager cache. In this case just update the pager cache, + ** not the database file. The page is left marked dirty in this case. + ** + ** An exception to the above rule: If the database is in no-sync mode + ** and a page is moved during an incremental vacuum then the page may + ** not be in the pager cache. Later: if a malloc() or IO error occurs + ** during a Movepage() call, then the page may not be in the cache + ** either. So the condition described in the above paragraph is not + ** assert()able. + ** + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. + ** + ** Ticket #1171: The statement journal might contain page content that is + ** different from the page content at the start of the transaction. + ** This occurs when a page is changed prior to the start of a statement + ** then changed again within the statement. When rolling back such a + ** statement we must not write to the original database unless we know + ** for certain that original page contents are synced into the main rollback + ** journal. Otherwise, a power loss might leave modified data in the + ** database file without an entry in the rollback journal that can + ** restore the database to its original form. Two conditions must be + ** met before writing to the database files. (1) the database must be + ** locked. (2) we know that the original page content is fully synced + ** in the main journal either because the page is not in cache or else + ** the page is marked as needSync==0. + ** + ** 2008-04-14: When attempting to vacuum a corrupt database file, it + ** is possible to fail a statement on a database that does not yet exist. + ** Do not attempt to write if database file has never been opened. + */ + if( pagerUseWal(pPager) ){ + pPg = 0; + }else{ + pPg = sqlite3PagerLookup(pPager, pgno); + } + assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); + PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), + (isMainJrnl?"main-journal":"sub-journal") + )); + if( isMainJrnl ){ + isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); + }else{ + isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); + } + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + && isSynced + ){ + i64 ofst = (pgno-1)*(i64)pPager->pageSize; + testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); + assert( !pagerUseWal(pPager) ); + + /* Write the data read from the journal back into the database file. + ** This is usually safe even for an encrypted database - as the data + ** was encrypted before it was written to the journal file. The exception + ** is if the data was just read from an in-memory sub-journal. In that + ** case it must be encrypted here before it is copied into the database + ** file. */ + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + } + }else if( !isMainJrnl && pPg==0 ){ + /* If this is a rollback of a savepoint and data was not written to + ** the database and the page is not in-memory, there is a potential + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. + ** + ** There are a couple of different ways this can happen. All are quite + ** obscure. When running in synchronous mode, this can only happen + ** if the page is on the free-list at the start of the transaction, then + ** populated, then moved using sqlite3PagerMovepage(). + ** + ** The solution is to add an in-memory page to the cache containing + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as + ** requiring a journal-sync before it is written. + */ + assert( isSavepnt ); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); + pPager->doNotSpill |= SPILLFLAG_ROLLBACK; + rc = sqlite3PagerGet(pPager, pgno, &pPg, 1); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; + if( rc!=SQLITE_OK ) return rc; + sqlite3PcacheMakeDirty(pPg); + } + if( pPg ){ + /* No page should ever be explicitly rolled back that is in use, except + ** for page 1 which is held in use in order to keep the lock on the + ** database active. However such a page may be rolled back as a result + ** of an internal error resulting in an automatic call to + ** sqlite3PagerRollback(). + */ + void *pData; + pData = pPg->pData; + memcpy(pData, (u8*)aData, pPager->pageSize); + pPager->xReiniter(pPg); + /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But + ** that call was dangerous and had no detectable benefit since the cache + ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so + ** has been removed. */ + pager_set_pagehash(pPg); + + /* If this was page 1, then restore the value of Pager.dbFileVers. + ** Do this before any decoding. */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); + } + sqlite3PcacheRelease(pPg); + } + return rc; +} + +/* +** Parameter zSuper is the name of a super-journal file. A single journal +** file that referred to the super-journal file has just been rolled back. +** This routine checks if it is possible to delete the super-journal file, +** and does so if it is. +** +** Argument zSuper may point to Pager.pTmpSpace. So that buffer is not +** available for use within this function. +** +** When a super-journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a +** nul-terminator byte (0x00). i.e. the entire contents of a super-journal +** file for a transaction involving two databases might be: +** +** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" +** +** A super-journal file may only be deleted once all of its child +** journals have been rolled back. +** +** This function reads the contents of the super-journal file into +** memory and loops through each of the child journal names. For +** each child journal, it checks if: +** +** * if the child journal exists, and if so +** * if the child journal contains a reference to super-journal +** file zSuper +** +** If a child journal can be found that matches both of the criteria +** above, this function returns without doing anything. Otherwise, if +** no such child journal can be found, file zSuper is deleted from +** the file-system using sqlite3OsDelete(). +** +** If an IO error within this function, an error code is returned. This +** function allocates memory by calling sqlite3Malloc(). If an allocation +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** occur, SQLITE_OK is returned. +** +** TODO: This function allocates a single block of memory to load +** the entire contents of the super-journal file. This could be +** a couple of kilobytes or so - potentially larger than the page +** size. +*/ +static int pager_delsuper(Pager *pPager, const char *zSuper){ + sqlite3_vfs *pVfs = pPager->pVfs; + int rc; /* Return code */ + sqlite3_file *pSuper; /* Malloc'd super-journal file descriptor */ + sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ + char *zSuperJournal = 0; /* Contents of super-journal file */ + i64 nSuperJournal; /* Size of super-journal file */ + char *zJournal; /* Pointer to one journal within MJ file */ + char *zSuperPtr; /* Space to hold super-journal filename */ + char *zFree = 0; /* Free this buffer */ + int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ + + /* Allocate space for both the pJournal and pSuper file descriptors. + ** If successful, open the super-journal file for reading. + */ + pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + if( !pSuper ){ + rc = SQLITE_NOMEM_BKPT; + pJournal = 0; + }else{ + const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_SUPER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zSuper, pSuper, flags, 0); + pJournal = (sqlite3_file *)(((u8 *)pSuper) + pVfs->szOsFile); + } + if( rc!=SQLITE_OK ) goto delsuper_out; + + /* Load the entire super-journal file into space obtained from + ** sqlite3_malloc() and pointed to by zSuperJournal. Also obtain + ** sufficient space (in zSuperPtr) to hold the names of super-journal + ** files extracted from regular rollback-journals. + */ + rc = sqlite3OsFileSize(pSuper, &nSuperJournal); + if( rc!=SQLITE_OK ) goto delsuper_out; + nSuperPtr = pVfs->mxPathname+1; + zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); + if( !zFree ){ + rc = SQLITE_NOMEM_BKPT; + goto delsuper_out; + } + zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; + zSuperJournal = &zFree[4]; + zSuperPtr = &zSuperJournal[nSuperJournal+2]; + rc = sqlite3OsRead(pSuper, zSuperJournal, (int)nSuperJournal, 0); + if( rc!=SQLITE_OK ) goto delsuper_out; + zSuperJournal[nSuperJournal] = 0; + zSuperJournal[nSuperJournal+1] = 0; + + zJournal = zSuperJournal; + while( (zJournal-zSuperJournal)pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS +** xTruncate() method to truncate it. +** +** Or, it might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to +** the end of the new file instead. +** +** If successful, return SQLITE_OK. If an IO error occurs while modifying +** the database file, return the error code to the caller. +*/ +static int pager_truncate(Pager *pPager, Pgno nPage){ + int rc = SQLITE_OK; + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + i64 currentSize, newSize; + int szPage = pPager->pageSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); + /* TODO: Is it safe to use Pager.dbFileSize here? */ + rc = sqlite3OsFileSize(pPager->fd, ¤tSize); + newSize = szPage*(i64)nPage; + if( rc==SQLITE_OK && currentSize!=newSize ){ + if( currentSize>newSize ){ + rc = sqlite3OsTruncate(pPager->fd, newSize); + }else if( (currentSize+szPage)<=newSize ){ + char *pTmp = pPager->pTmpSpace; + memset(pTmp, 0, szPage); + testcase( (newSize-szPage) == currentSize ); + testcase( (newSize-szPage) > currentSize ); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); + } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } + } + } + return rc; +} + +/* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + +/* +** Set the value of the Pager.sectorSize variable for the given +** pager based on the value returned by the xSectorSize method +** of the open database file. The sector size will be used +** to determine the size and alignment of journal header and +** super-journal pointers within created journal files. +** +** For temporary files the effective sector size is always 512 bytes. +** +** Otherwise, for non-temporary files, the effective sector size is +** the value returned by the xSectorSize() method rounded up to 32 if +** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it +** is greater than MAX_SECTOR_SIZE. +** +** If the file has the SQLITE_IOCAP_POWERSAFE_OVERWRITE property, then set +** the effective sector size to its minimum value (512). The purpose of +** pPager->sectorSize is to define the "blast radius" of bytes that +** might change if a crash occurs while writing to a single byte in +** that range. But with POWERSAFE_OVERWRITE, the blast radius is zero +** (that is what POWERSAFE_OVERWRITE means), so we minimize the sector +** size. For backwards compatibility of the rollback journal file format, +** we cannot reduce the effective sector size below 512. +*/ +static void setSectorSize(Pager *pPager){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + + if( pPager->tempFile + || (sqlite3OsDeviceCharacteristics(pPager->fd) & + SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 + ){ + /* Sector size doesn't matter for temporary files. Also, the file + ** may not have been opened yet, in which case the OsSectorSize() + ** call will segfault. */ + pPager->sectorSize = 512; + }else{ + pPager->sectorSize = sqlite3SectorSize(pPager->fd); + } +} + +/* +** Playback the journal and thus restore the database file to +** the state it was in before we started making changes. +** +** The journal file format is as follows: +** +** (1) 8 byte prefix. A copy of aJournalMagic[]. +** (2) 4 byte big-endian integer which is the number of valid page records +** in the journal. If this value is 0xffffffff, then compute the +** number of page records from the journal size. +** (3) 4 byte big-endian integer which is the initial value for the +** sanity checksum. +** (4) 4 byte integer which is the number of pages to truncate the +** database to during a rollback. +** (5) 4 byte big-endian integer which is the sector size. The header +** is this many bytes in size. +** (6) 4 byte big-endian integer which is the page size. +** (7) zero padding out to the next sector size. +** (8) Zero or more pages instances, each as follows: +** + 4 byte page number. +** + pPager->pageSize bytes of data. +** + 4 byte checksum +** +** When we speak of the journal header, we mean the first 7 items above. +** Each entry in the journal is an instance of the 8th item. +** +** Call the value from the second bullet "nRec". nRec is the number of +** valid page entries in the journal. In most cases, you can compute the +** value of nRec from the size of the journal file. But if a power +** failure occurred while the journal was being written, it could be the +** case that the size of the journal file had already been increased but +** the extra entries had not yet made it safely to disk. In such a case, +** the value of nRec computed from the file size would be too large. For +** that reason, we always use the nRec value in the header. +** +** If the nRec value is 0xffffffff it means that nRec should be computed +** from the file size. This value is used when the user selects the +** no-sync option for the journal. A power failure could lead to corruption +** in this case. But for things like temporary table (which will be +** deleted when the power is restored) we don't care. +** +** If the file opened as the journal file is not a well-formed +** journal file then all pages up to the first corrupted page are rolled +** back (or no pages if the journal header is corrupted). The journal file +** is then deleted and SQLITE_OK returned, just as if no corruption had +** been encountered. +** +** If an I/O or malloc() error occurs, the journal-file is not deleted +** and an error code is returned. +** +** The isHot parameter indicates that we are trying to rollback a journal +** that might be a hot journal. Or, it could be that the journal is +** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. +** If the journal really is hot, reset the pager cache prior rolling +** back any content. If the journal is merely persistent, no reset is +** needed. +*/ +static int pager_playback(Pager *pPager, int isHot){ + sqlite3_vfs *pVfs = pPager->pVfs; + i64 szJ; /* Size of the journal file in bytes */ + u32 nRec; /* Number of Records in the journal */ + u32 u; /* Unsigned loop counter */ + Pgno mxPg = 0; /* Size of the original file in pages */ + int rc; /* Result code of a subroutine */ + int res = 1; /* Value returned by sqlite3OsAccess() */ + char *zSuper = 0; /* Name of super-journal file if any */ + int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ + u32 savedPageSize = pPager->pageSize; + + /* Figure out how many records are in the journal. Abort early if + ** the journal is empty. + */ + assert( isOpen(pPager->jfd) ); + rc = sqlite3OsFileSize(pPager->jfd, &szJ); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + + /* Read the super-journal name from the journal, if it is present. + ** If a super-journal file name is specified, but the file is not + ** present on disk, then the journal is not hot and does not need to be + ** played back. + ** + ** TODO: Technically the following is an error because it assumes that + ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that + ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, + ** mxPathname is 512, which is the same as the minimum allowable value + ** for pageSize. + */ + zSuper = pPager->pTmpSpace; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zSuper[0] ){ + rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); + } + zSuper = 0; + if( rc!=SQLITE_OK || !res ){ + goto end_playback; + } + pPager->journalOff = 0; + needPagerReset = isHot; + + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. + */ + while( 1 ){ + /* Read the next journal header from the journal file. If there are + ** not enough bytes left in the journal file for a complete header, or + ** it is corrupted, then a process must have failed while writing it. + ** This indicates nothing more needs to be rolled back. + */ + rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + goto end_playback; + } + + /* If nRec is 0xffffffff, then this journal was created by a process + ** working in no-sync mode. This means that the rest of the journal + ** file consists of pages, there are no more journal headers. Compute + ** the value of nRec based on this assumption. + */ + if( nRec==0xffffffff ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); + nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); + } + + /* If nRec is 0 and this rollback is of a transaction created by this + ** process and if this is the final header in the journal, then it means + ** that this part of the journal was being filled but has not yet been + ** synced to disk. Compute the number of pages based on the remaining + ** size of the file. + ** + ** The third term of the test was added to fix ticket #2565. + ** When rolling back a hot journal, nRec==0 always means that the next + ** chunk of the journal contains zero pages to be rolled back. But + ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in + ** the journal, it means that the journal might contain additional + ** pages that need to be rolled back and that the number of pages + ** should be computed based on the journal file size. + */ + if( nRec==0 && !isHot && + pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ + nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); + } + + /* If this is the first header read from the journal, truncate the + ** database file back to its original size. + */ + if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ + rc = pager_truncate(pPager, mxPg); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + pPager->dbSize = mxPg; + } + + /* Copy original pages out of the journal and back into the + ** database file and/or page cache. + */ + for(u=0; ujournalOff,0,1,0); + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ + if( rc==SQLITE_DONE ){ + pPager->journalOff = szJ; + break; + }else if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If the journal has been truncated, simply stop reading and + ** processing the journal. This might happen if the journal was + ** not completely written and synced prior to a crash. In that + ** case, the database should have never been written in the + ** first place so it is OK to simply abandon the rollback. */ + rc = SQLITE_OK; + goto end_playback; + }else{ + /* If we are unable to rollback, quit and return the error + ** code. This will cause the pager to enter the error state + ** so that no further harm will be done. Perhaps the next + ** process to come along will be able to rollback the database. + */ + goto end_playback; + } + } + } + } + /*NOTREACHED*/ + assert( 0 ); + +end_playback: + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSetPagesize(pPager, &savedPageSize, -1); + } + /* Following a rollback, the database file should be back in its original + ** state prior to the start of the transaction, so invoke the + ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the + ** assertion that the transaction counter was modified. + */ +#ifdef SQLITE_DEBUG + sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); +#endif + + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive + ** mode, then subsequent transactions performed by the connection will not + ** update the change-counter at all. This may lead to cache inconsistency + ** problems for other processes at some point in the future. So, just + ** in case this has happened, clear the changeCountDone flag now. + */ + pPager->changeCountDone = pPager->tempFile; + + if( rc==SQLITE_OK ){ + /* Leave 4 bytes of space before the super-journal filename in memory. + ** This is because it may end up being passed to sqlite3OsOpen(), in + ** which case it requires 4 0x00 bytes in memory immediately before + ** the filename. */ + zSuper = &pPager->pTmpSpace[4]; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + rc = sqlite3PagerSync(pPager, 0); + } + if( rc==SQLITE_OK ){ + rc = pager_end_transaction(pPager, zSuper[0]!='\0', 0); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK && zSuper[0] && res ){ + /* If there was a super-journal and this routine will return success, + ** see if it is possible to delete the super-journal. + */ + assert( zSuper==&pPager->pTmpSpace[4] ); + memset(&zSuper[-4], 0, 4); + rc = pager_delsuper(pPager, zSuper); + testcase( rc!=SQLITE_OK ); + } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } + + /* The Pager.sectorSize variable may have been updated while rolling + ** back a journal created by a process with a different sector size + ** value. Reset it to the correct value for this process. + */ + setSectorSize(pPager); + return rc; +} + + +/* +** Read the content for page pPg out of the database file (or out of +** the WAL if that is where the most recent copy if found) into +** pPg->pData. A shared lock or greater must be held on the database +** file before this function is called. +** +** If page 1 is read, then the value of Pager.dbFileVers[] is set to +** the value read from the database file. +** +** If an IO error occurs, then the IO error is returned to the caller. +** Otherwise, SQLITE_OK is returned. +*/ +static int readDbPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ + int rc = SQLITE_OK; /* Return code */ + +#ifndef SQLITE_OMIT_WAL + u32 iFrame = 0; /* Frame of WAL containing pgno */ + + assert( pPager->eState>=PAGER_READER && !MEMDB ); + assert( isOpen(pPager->fd) ); + + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc ) return rc; + } + if( iFrame ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData); + }else +#endif + { + i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + + if( pPg->pgno==1 ){ + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero or the size of the database in page. Bytes 32..35 and 35..39 + ** should be page numbers which are never 0xffffffff. So filling + ** pPager->dbFileVers[] with all 0xff bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noise equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. + */ + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, pager_pagehash(pPg))); + + return rc; +} + +/* +** Update the value of the change-counter at offsets 24 and 92 in +** the header and the sqlite version number at offset 96. +** +** This is an unconditional update. See also the pager_incr_changecounter() +** routine which only updates the change-counter if the update is actually +** needed, as determined by the pPager->changeCountDone state variable. +*/ +static void pager_write_changecounter(PgHdr *pPg){ + u32 change_counter; + + /* Increment the value just read and write it back to byte 24. */ + change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; + put32bits(((char*)pPg->pData)+24, change_counter); + + /* Also store the SQLite version number in bytes 96..99 and in + ** bytes 92..95 store the change counter for which the version number + ** is valid. */ + put32bits(((char*)pPg->pData)+92, change_counter); + put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); +} + +#ifndef SQLITE_OMIT_WAL +/* +** This function is invoked once for each page that has already been +** written into the log file when a WAL transaction is rolled back. +** Parameter iPg is the page number of said page. The pCtx argument +** is actually a pointer to the Pager structure. +** +** If page iPg is present in the cache, and has no outstanding references, +** it is discarded. Otherwise, if there are one or more outstanding +** references, the page content is reloaded from the database. If the +** attempt to reload content from the database is required and fails, +** return an SQLite error code. Otherwise, SQLITE_OK. +*/ +static int pagerUndoCallback(void *pCtx, Pgno iPg){ + int rc = SQLITE_OK; + Pager *pPager = (Pager *)pCtx; + PgHdr *pPg; + + assert( pagerUseWal(pPager) ); + pPg = sqlite3PagerLookup(pPager, iPg); + if( pPg ){ + if( sqlite3PcachePageRefcount(pPg)==1 ){ + sqlite3PcacheDrop(pPg); + }else{ + rc = readDbPage(pPg); + if( rc==SQLITE_OK ){ + pPager->xReiniter(pPg); + } + sqlite3PagerUnrefNotNull(pPg); + } + } + + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + sqlite3BackupRestart(pPager->pBackup); + + return rc; +} + +/* +** This function is called to rollback a transaction on a WAL database. +*/ +static int pagerRollbackWal(Pager *pPager){ + int rc; /* Return Code */ + PgHdr *pList; /* List of dirty pages to revert */ + + /* For all pages in the cache that are currently dirty or have already + ** been written (but not committed) to the log file, do one of the + ** following: + ** + ** + Discard the cached page (if refcount==0), or + ** + Reload page content from the database (if refcount>0). + */ + pPager->dbSize = pPager->dbOrigSize; + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); + pList = sqlite3PcacheDirtyList(pPager->pPCache); + while( pList && rc==SQLITE_OK ){ + PgHdr *pNext = pList->pDirty; + rc = pagerUndoCallback((void *)pPager, pList->pgno); + pList = pNext; + } + + return rc; +} + +/* +** This function is a wrapper around sqlite3WalFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +** +** The list of pages passed into this routine is always sorted by page number. +** Hence, if page 1 appears anywhere on the list, it will be the first page. +*/ +static int pagerWalFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit /* True if this is a commit */ +){ + int rc; /* Return code */ + int nList; /* Number of pages in pList */ + PgHdr *p; /* For looping over pages */ + + assert( pPager->pWal ); + assert( pList ); +#ifdef SQLITE_DEBUG + /* Verify that the page list is in accending order */ + for(p=pList; p && p->pDirty; p=p->pDirty){ + assert( p->pgno < p->pDirty->pgno ); + } +#endif + + assert( pList->pDirty==0 || isCommit ); + if( isCommit ){ + /* If a WAL transaction is being committed, there is no point in writing + ** any pages with page numbers greater than nTruncate into the WAL file. + ** They will never be read by any client. So remove them from the pDirty + ** list here. */ + PgHdr **ppNext = &pList; + nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } + } + assert( pList ); + }else{ + nList = 1; + } + pPager->aStat[PAGER_STAT_WRITE] += nList; + + if( pList->pgno==1 ) pager_write_changecounter(pList); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + +#ifdef SQLITE_CHECK_PAGES + pList = sqlite3PcacheDirtyList(pPager->pPCache); + for(p=pList; p; p=p->pDirty){ + pager_set_pagehash(p); + } +#endif + + return rc; +} + +/* +** Begin a read transaction on the WAL. +** +** This routine used to be called "pagerOpenSnapshot()" because it essentially +** makes a snapshot of the database at the current point in time and preserves +** that snapshot for use by the reader in spite of concurrently changes by +** other writers or checkpointers. +*/ +static int pagerBeginReadTransaction(Pager *pPager){ + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ + + assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + + /* sqlite3WalEndReadTransaction() was not called for the previous + ** transaction in locking_mode=EXCLUSIVE. So call it now. If we + ** are in locking_mode=NORMAL and EndRead() was previously called, + ** the duplicate call is harmless. + */ + sqlite3WalEndReadTransaction(pPager->pWal); + + rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + } + + return rc; +} +#endif + +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the number of pages in the database is not available from the + ** WAL sub-system, determine the page count based on the size of + ** the database file. If the size of the database file is not an + ** integer multiple of the page-size, round up the result. + */ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ + i64 n = 0; /* Size of db file in bytes */ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_WAL +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists if the database is not empy, or verify that the *-wal file does +** not exist (by deleting it) if the database file is empty. +** +** If the database is not empty and the *-wal file exists, open the pager +** in WAL mode. If the database is empty or if no *-wal file exists and +** if no error occurs, make sure Pager.journalMode is not set to +** PAGER_JOURNALMODE_WAL. +** +** Return SQLITE_OK or an error code. +** +** The caller must hold a SHARED lock on the database file to call this +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some +** other connection. +*/ +static int pagerOpenWalIfPresent(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + + if( !pPager->tempFile ){ + int isWal; /* True if WAL file exists */ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal + ); + if( rc==SQLITE_OK ){ + if( isWal ){ + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); + if( rc ) return rc; + if( nPage==0 ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + }else{ + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); + rc = sqlite3PagerOpenWal(pPager, 0); + } + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } + } + } + return rc; +} +#endif + +/* +** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback +** the entire super-journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** savepoint. +** +** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** being rolled back), then the rollback consists of up to three stages, +** performed in the order specified: +** +** * Pages are played back from the main journal starting at byte +** offset PagerSavepoint.iOffset and continuing to +** PagerSavepoint.iHdrOffset, or to the end of the main journal +** file if PagerSavepoint.iHdrOffset is zero. +** +** * If PagerSavepoint.iHdrOffset is not zero, then pages are played +** back starting from the journal header immediately following +** PagerSavepoint.iHdrOffset to the end of the main journal file. +** +** * Pages are then played back from the sub-journal file, starting +** with the PagerSavepoint.iSubRec and continuing to the end of +** the journal file. +** +** Throughout the rollback process, each time a page is rolled back, the +** corresponding bit is set in a bitvec structure (variable pDone in the +** implementation below). This is used to ensure that a page is only +** rolled back the first time it is encountered in either journal. +** +** If pSavepoint is NULL, then pages are only played back from the main +** journal file. There is no need for a bitvec in this case. +** +** In either case, before playback commences the Pager.dbSize variable +** is reset to the value that it held at the start of the savepoint +** (or transaction). No page with a page-number greater than this value +** is played back. If one is encountered it is simply skipped. +*/ +static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ + i64 szJ; /* Effective size of the main journal */ + i64 iHdrOff; /* End of first segment of main-journal records */ + int rc = SQLITE_OK; /* Return code */ + Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ + + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + + /* Allocate a bitvec to use to store the set of pages rolled back */ + if( pSavepoint ){ + pDone = sqlite3BitvecCreate(pSavepoint->nOrig); + if( !pDone ){ + return SQLITE_NOMEM_BKPT; + } + } + + /* Set the database size back to the value it was before the savepoint + ** being reverted was opened. + */ + pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; + pPager->changeCountDone = pPager->tempFile; + + if( !pSavepoint && pagerUseWal(pPager) ){ + return pagerRollbackWal(pPager); + } + + /* Use pPager->journalOff as the effective size of the main rollback + ** journal. The actual file might be larger than this in + ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything + ** past pPager->journalOff is off-limits to us. + */ + szJ = pPager->journalOff; + assert( pagerUseWal(pPager)==0 || szJ==0 ); + + /* Begin by rolling back records from the main journal starting at + ** PagerSavepoint.iOffset and continuing to the next journal header. + ** There might be records in the main journal that have a page number + ** greater than the current database size (pPager->dbSize) but those + ** will be skipped automatically. Pages are added to pDone as they + ** are played back. + */ + if( pSavepoint && !pagerUseWal(pPager) ){ + iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; + pPager->journalOff = pSavepoint->iOffset; + while( rc==SQLITE_OK && pPager->journalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + }else{ + pPager->journalOff = 0; + } + + /* Continue rolling back records out of the main journal starting at + ** the first journal header seen and continuing until the effective end + ** of the main journal file. Continue to skip out-of-range pages and + ** continue adding pages rolled back to pDone. + */ + while( rc==SQLITE_OK && pPager->journalOffjournalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" + ** test is related to ticket #2565. See the discussion in the + ** pager_playback() function for additional information. + */ + if( nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff + ){ + nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); + } + for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + } + assert( rc!=SQLITE_OK || pPager->journalOff>=szJ ); + + /* Finally, rollback pages from the sub-journal. Page that were + ** previously rolled back out of the main journal (and are hence in pDone) + ** will be skipped. Out-of-range pages are also skipped. + */ + if( pSavepoint ){ + u32 ii; /* Loop counter */ + i64 offset = (i64)pSavepoint->iSubRec*(4+pPager->pageSize); + + if( pagerUseWal(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); + } + for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ + assert( offset==(i64)ii*(4+pPager->pageSize) ); + rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); + } + assert( rc!=SQLITE_DONE ); + } + + sqlite3BitvecDestroy(pDone); + if( rc==SQLITE_OK ){ + pPager->journalOff = szJ; + } + + return rc; +} + +/* +** Change the maximum number of in-memory pages that are allowed +** before attempting to recycle clean and unused pages. +*/ +void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); +} + +/* +** Change the maximum number of in-memory pages that are allowed +** before attempting to spill pages to journal. +*/ +int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){ + return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage); +} + +/* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) && fd->pMethods->iVersion>=3 ){ + sqlite3_int64 sz; + sz = pPager->szMmap; + pPager->bUseFetch = (sz>0); + setGetterMethod(pPager); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + +/* +** Free as much memory as possible from the pager. +*/ +void sqlite3PagerShrink(Pager *pPager){ + sqlite3PcacheShrink(pPager->pPCache); +} + +/* +** Adjust settings of the pager to those specified in the pgFlags parameter. +** +** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness +** of the database to damage due to OS crashes or power failures by +** changing the number of syncs()s when writing the journals. +** There are four levels: +** +** OFF sqlite3OsSync() is never called. This is the default +** for temporary and transient files. +** +** NORMAL The journal is synced once before writes begin on the +** database. This is normally adequate protection, but +** it is theoretically possible, though very unlikely, +** that an inopertune power failure could leave the journal +** in a state which would cause damage to the database +** when it is rolled back. +** +** FULL The journal is synced twice before writes begin on the +** database (with some additional information - the nRec field +** of the journal header - being written in between the two +** syncs). If we assume that writing a +** single disk sector is atomic, then this mode provides +** assurance that the journal will not be corrupted to the +** point of causing damage to the database during rollback. +** +** EXTRA This is like FULL except that is also syncs the directory +** that contains the rollback journal after the rollback +** journal is unlinked. +** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. There is no difference between FULL +** and EXTRA for WAL mode. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. +** +** Numeric values associated with these states are OFF==1, NORMAL=2, +** and FULL=3. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +void sqlite3PagerSetFlags( + Pager *pPager, /* The pager to set safety level for */ + unsigned pgFlags /* Various flags */ +){ + unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; + if( pPager->tempFile ){ + pPager->noSync = 1; + pPager->fullSync = 0; + pPager->extraSync = 0; + }else{ + pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; + pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + } + if( pPager->noSync ){ + pPager->syncFlags = 0; + }else if( pgFlags & PAGER_FULLFSYNC ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + } + pPager->walSyncFlags = (pPager->syncFlags<<2); + if( pPager->fullSync ){ + pPager->walSyncFlags |= pPager->syncFlags; + } + if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ + pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); + } + if( pgFlags & PAGER_CACHESPILL ){ + pPager->doNotSpill &= ~SPILLFLAG_OFF; + }else{ + pPager->doNotSpill |= SPILLFLAG_OFF; + } +} +#endif + +/* +** The following global variable is incremented whenever the library +** attempts to open a temporary file. This information is used for +** testing and analysis only. +*/ +#ifdef SQLITE_TEST +int sqlite3_opentemp_count = 0; +#endif + +/* +** Open a temporary file. +** +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically +** delete the temporary file when it is closed. +** +** The flags passed to the VFS layer xOpen() call are those specified +** by parameter vfsFlags ORed with the following: +** +** SQLITE_OPEN_READWRITE +** SQLITE_OPEN_CREATE +** SQLITE_OPEN_EXCLUSIVE +** SQLITE_OPEN_DELETEONCLOSE +*/ +static int pagerOpentemp( + Pager *pPager, /* The pager object */ + sqlite3_file *pFile, /* Write the file descriptor here */ + int vfsFlags /* Flags passed through to the VFS */ +){ + int rc; /* Return code */ + +#ifdef SQLITE_TEST + sqlite3_opentemp_count++; /* Used for testing and analysis only */ +#endif + + vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); + assert( rc!=SQLITE_OK || isOpen(pFile) ); + return rc; +} + +/* +** Set the busy handler function. +** +** The pager invokes the busy-handler if sqlite3OsLock() returns +** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** lock. It does *not* invoke the busy handler when upgrading from +** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE +** (which occurs during hot-journal rollback). Summary: +** +** Transition | Invokes xBusyHandler +** -------------------------------------------------------- +** NO_LOCK -> SHARED_LOCK | Yes +** SHARED_LOCK -> RESERVED_LOCK | No +** SHARED_LOCK -> EXCLUSIVE_LOCK | No +** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes +** +** If the busy-handler callback returns non-zero, the lock is +** retried. If it returns zero, then the SQLITE_BUSY error is +** returned to the caller of the pager API function. +*/ +void sqlite3PagerSetBusyHandler( + Pager *pPager, /* Pager object */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ +){ + void **ap; + pPager->xBusyHandler = xBusyHandler; + pPager->pBusyHandlerArg = pBusyHandlerArg; + ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); +} + +/* +** Change the page size used by the Pager object. The new page size +** is passed in *pPageSize. +** +** If the pager is in the error state when this function is called, it +** is a no-op. The value returned is the error state error code (i.e. +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). +** +** Otherwise, if all of the following are true: +** +** * the new page size (value of *pPageSize) is valid (a power +** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and +** +** * there are no outstanding page references, and +** +** * the database is either not an in-memory database or it is +** an in-memory database that currently consists of zero pages. +** +** then the pager object page size is set to *pPageSize. +** +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** In all other cases, SQLITE_OK is returned. +** +** If the page size is not changed, either because one of the enumerated +** conditions above is not true, the pager was in error state when this +** function was called, or because the memory allocation attempt failed, +** then *pPageSize is set to the old, retained page size before returning. +*/ +int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; + + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + } + if( rc==SQLITE_OK ){ + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + if( rc==SQLITE_OK ){ + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + }else{ + sqlite3PageFree(pNew); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ + if( nReserve<0 ) nReserve = pPager->nReserve; + assert( nReserve>=0 && nReserve<1000 ); + pPager->nReserve = (i16)nReserve; + pagerFixMaplimit(pPager); + } + return rc; +} + +/* +** Return a pointer to the "temporary page" buffer held internally +** by the pager. This is a buffer that is big enough to hold the +** entire content of a database page. This buffer is used internally +** during rollback and will be overwritten whenever a rollback +** occurs. But other modules are free to use it too, as long as +** no rollbacks are happening. +*/ +void *sqlite3PagerTempSpace(Pager *pPager){ + return pPager->pTmpSpace; +} + +/* +** Attempt to set the maximum database page count if mxPage is positive. +** Make no changes if mxPage is zero or negative. And never reduce the +** maximum page count below the current size of the database. +** +** Regardless of mxPage, return the current maximum page count. +*/ +Pgno sqlite3PagerMaxPageCount(Pager *pPager, Pgno mxPage){ + if( mxPage>0 ){ + pPager->mxPgno = mxPage; + } + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + /* assert( pPager->mxPgno>=pPager->dbSize ); */ + /* OP_MaxPgcnt ensures that the parameter passed to this function is not + ** less than the total number of valid pages in the database. But this + ** may be less than Pager.dbSize, and so the assert() above is not valid */ + return pPager->mxPgno; +} + +/* +** The following set of routines are used to disable the simulated +** I/O error mechanism. These routines are used to avoid simulated +** errors in places where we do not care about errors. +** +** Unless -DSQLITE_TEST=1 is used, these routines are all no-ops +** and generate no code. +*/ +#ifdef SQLITE_TEST +extern int sqlite3_io_error_pending; +extern int sqlite3_io_error_hit; +static int saved_cnt; +void disable_simulated_io_errors(void){ + saved_cnt = sqlite3_io_error_pending; + sqlite3_io_error_pending = -1; +} +void enable_simulated_io_errors(void){ + sqlite3_io_error_pending = saved_cnt; +} +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +/* +** Read the first N bytes from the beginning of the file into memory +** that pDest points to. +** +** If the pager was opened on a transient file (zFilename==""), or +** opened on a file less than N bytes in size, the output buffer is +** zeroed and SQLITE_OK returned. The rationale for this is that this +** function is used to read database headers, and a new transient or +** zero sized database has a header than consists entirely of zeroes. +** +** If any IO error apart from SQLITE_IOERR_SHORT_READ is encountered, +** the error code is returned to the caller and the contents of the +** output buffer undefined. +*/ +int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ + int rc = SQLITE_OK; + memset(pDest, 0, N); + assert( isOpen(pPager->fd) || pPager->tempFile ); + + /* This routine is only called by btree immediately after creating + ** the Pager object. There has not been an opportunity to transition + ** to WAL mode yet. + */ + assert( !pagerUseWal(pPager) ); + + if( isOpen(pPager->fd) ){ + IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) + rc = sqlite3OsRead(pPager->fd, pDest, N, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + return rc; +} + +/* +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** +** However, if the file is between 1 and bytes in size, then +** this is considered a 1 page file. +*/ +void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; +} + + +/* +** Try to obtain a lock of type locktype on the database file. If +** a similar or greater lock is already held, this function is a no-op +** (returning SQLITE_OK immediately). +** +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to +** obtain the lock succeeds. +** +** Return SQLITE_OK on success and an error code if we cannot obtain +** the lock. If the lock is obtained successfully, set the Pager.state +** variable to locktype before returning. +*/ +static int pager_wait_on_lock(Pager *pPager, int locktype){ + int rc; /* Return code */ + + /* Check that this is either a no-op (because the requested lock is + ** already held), or one of the transitions that the busy-handler + ** may be invoked during, according to the comment above + ** sqlite3PagerSetBusyhandler(). + */ + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) + ); + + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + return rc; +} + +/* +** Function assertTruncateConstraint(pPager) checks that one of the +** following is true for all dirty pages currently in the page-cache: +** +** a) The page number is less than or equal to the size of the +** current database image, in pages, OR +** +** b) if the page content were written at this time, it would not +** be necessary to write the current content out to the sub-journal +** (as determined by function subjRequiresPage()). +** +** If the condition asserted by this function were not true, and the +** dirty page were to be discarded from the cache via the pagerStress() +** routine, pagerStress() would not write the current page content to +** the database file. If a savepoint transaction were rolled back after +** this happened, the correct behavior would be to restore the current +** content of the page. However, since this content is not present in either +** the database file or the portion of the rollback journal and +** sub-journal rolled back the content could not be restored and the +** database image would become corrupt. It is therefore fortunate that +** this circumstance cannot arise. +*/ +#if defined(SQLITE_DEBUG) +static void assertTruncateConstraintCb(PgHdr *pPg){ + assert( pPg->flags&PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); +} +static void assertTruncateConstraint(Pager *pPager){ + sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); +} +#else +# define assertTruncateConstraint(pPager) +#endif + +/* +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. +*/ +void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSize>=nPage ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + pPager->dbSize = nPage; + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ +} + + +/* +** This function is called before attempting a hot-journal rollback. It +** syncs the journal file to disk, then sets pPager->journalHdr to the +** size of the journal file so that the pager_playback() routine knows +** that the entire journal file has been synced. +** +** Syncing a hot-journal to disk before attempting to roll it back ensures +** that if a power-failure occurs during the rollback, the process that +** attempts rollback following system recovery sees the same journal +** content as this process. +** +** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** an SQLite error code. +*/ +static int pagerSyncHotJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_NORMAL); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileSize(pPager->jfd, &pPager->journalHdr); + } + return rc; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + assert( pPager->nExtra>=8 ); + memset(p->pExtra, 0, 8); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM_BKPT; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} +#endif + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still where it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + + +/* +** Shutdown the page cache. Free all memory and close all files. +** +** If a transaction was in progress when this routine is called, that +** transaction is rolled back. All outstanding pages are invalidated +** and their memory is freed. Any attempt to use a page associated +** with this page cache after this function returns will likely +** result in a coredump. +** +** This function always succeeds. If a transaction is active an attempt +** is made to roll it back. If an error occurs during the rollback +** a hot journal may be left in the filesystem but no error is returned +** to the caller. +*/ +int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + u8 *pTmp = (u8*)pPager->pTmpSpace; + assert( db || pagerUseWal(pPager)==0 ); + assert( assert_pager_state(pPager) ); + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); + /* pPager->errCode = 0; */ + pPager->exclusiveMode = 0; +#ifndef SQLITE_OMIT_WAL + { + u8 *a = 0; + assert( db || pPager->pWal==0 ); + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + && SQLITE_OK==databaseIsUnmoved(pPager) + ){ + a = pTmp; + } + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } +#endif + pager_reset(pPager); + if( MEMDB ){ + pager_unlock(pPager); + }else{ + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. + */ + if( isOpen(pPager->jfd) ){ + pager_error(pPager, pagerSyncHotJournal(pPager)); + } + pagerUnlockAndRollback(pPager); + } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); + IOTRACE(("CLOSE %p\n", pPager)) + sqlite3OsClose(pPager->jfd); + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + + sqlite3_free(pPager); + return SQLITE_OK; +} + +#if !defined(NDEBUG) || defined(SQLITE_TEST) +/* +** Return the page number for page pPg. +*/ +Pgno sqlite3PagerPagenumber(DbPage *pPg){ + return pPg->pgno; +} +#endif + +/* +** Increment the reference count for page pPg. +*/ +void sqlite3PagerRef(DbPage *pPg){ + sqlite3PcacheRef(pPg); +} + +/* +** Sync the journal. In other words, make sure all the pages that have +** been written to the journal have actually reached the surface of the +** disk and can be restored in the event of a hot-journal rollback. +** +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the file-system, as follows: +** +** * If the journal file is an in-memory journal file, no action need +** be taken. +** +** * Otherwise, if the device does not support the SAFE_APPEND property, +** then the nRec field of the most recently written journal header +** is updated to contain the number of journal records that have +** been written following it. If the pager is operating in full-sync +** mode, then the journal file is synced before this field is updated. +** +** * If the device does not support the SEQUENTIAL property, then +** journal file is synced. +** +** Or, in pseudo-code: +** +** if( NOT ){ +** if( NOT SAFE_APPEND ){ +** if( ) xSync(); +** +** } +** if( NOT SEQUENTIAL ) xSync(); +** } +** +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** page currently held in memory before returning SQLITE_OK. If an IO +** error is encountered, then the IO error code is returned to the caller. +*/ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ + assert( !pPager->tempFile ); + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + assert( isOpen(pPager->jfd) ); + + if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + /* This block deals with an obscure problem. If the last connection + ** that wrote to this database was operating in persistent-journal + ** mode, then the journal file may at this point actually be larger + ** than Pager.journalOff bytes. If the next thing in the journal + ** file happens to be a journal-header (written as part of the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the + ** hot-journal rollback following recovery. It may roll back all + ** of this connections data, then proceed to rolling back the old, + ** out-of-date data that follows it. Database corruption. + ** + ** To work around this, if the journal file does appear to contain + ** a valid header following Pager.journalOff, then write a 0x00 + ** byte to the start of it to prevent it from being recognized. + ** + ** Variable iNextHdrOffset is set to the offset at which this + ** problematic header will occur, if it exists. aMagic is used + ** as a temporary buffer to inspect the first couple of bytes of + ** the potential journal header. + */ + i64 iNextHdrOffset; + u8 aMagic[8]; + u8 zHeader[sizeof(aJournalMagic)+4]; + + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + + iNextHdrOffset = journalHdrOffset(pPager); + rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); + if( rc==SQLITE_OK && 0==memcmp(aMagic, aJournalMagic, 8) ){ + static const u8 zerobyte = 0; + rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, iNextHdrOffset); + } + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + return rc; + } + + /* Write the nRec value into the journal file header. If in + ** full-synchronous mode, sync the journal first. This ensures that + ** all data has really hit the disk before nRec is updated to mark + ** it as a candidate for rollback. + ** + ** This is not required if the persistent media supports the + ** SAFE_APPEND property. Because in this case it is not possible + ** for garbage data to be appended to the file, the nRec field + ** is populated with 0xFFFFFFFF when the journal header is written + ** and never needs to be updated. + */ + if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + if( rc!=SQLITE_OK ) return rc; + } + IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); + rc = sqlite3OsWrite( + pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr + ); + if( rc!=SQLITE_OK ) return rc; + } + if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + ); + if( rc!=SQLITE_OK ) return rc; + } + + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } + } + + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); + return SQLITE_OK; +} + +/* +** The argument is the first in a linked list of dirty pages connected +** by the PgHdr.pDirty pointer. This function writes each one of the +** in-memory pages in the list to the database file. The argument may +** be NULL, representing an empty list. In this case this function is +** a no-op. +** +** The pager must hold at least a RESERVED lock when this function +** is called. Before writing anything to the database file, this lock +** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, +** SQLITE_BUSY is returned and no data is written to the database file. +** +** If the pager is a temp-file pager and the actual file-system file +** is not yet open, it is created and opened before any data is +** written out. +** +** Once the lock has been upgraded and, if necessary, the file opened, +** the pages are written out to the database file in list order. Writing +** a page is skipped if it meets either of the following criteria: +** +** * The page number is greater than Pager.dbSize, or +** * The PGHDR_DONT_WRITE flag is set on the page. +** +** If writing out a page causes the database file to grow, Pager.dbFileSize +** is updated accordingly. If page 1 is written out, then the value cached +** in Pager.dbFileVers[] is updated to match the new value stored in +** the database file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot +** be obtained, SQLITE_BUSY is returned. +*/ +static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ + int rc = SQLITE_OK; /* Return code */ + + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ + assert( !pagerUseWal(pPager) ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); + + /* If the file is a temp-file has not yet been opened, open it now. It + ** is not possible for rc to be other than SQLITE_OK if this branch + ** is taken, as pager_wait_on_lock() is a no-op for temp-files. + */ + if( !isOpen(pPager->fd) ){ + assert( pPager->tempFile && rc==SQLITE_OK ); + rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + } + + /* Before the first write, give the VFS a hint of what the final + ** file size will be. + */ + assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); + if( rc==SQLITE_OK + && pPager->dbHintSizedbSize + && (pList->pDirty || pList->pgno>pPager->dbHintSize) + ){ + sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; + } + + while( rc==SQLITE_OK && pList ){ + Pgno pgno = pList->pgno; + + /* If there are dirty pages in the page cache with page numbers greater + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to + ** make the file smaller (presumably by auto-vacuum code). Do not write + ** any such pages to the file. + ** + ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag + ** set (set by sqlite3PagerDontWrite()). + */ + if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ + i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ + char *pData; /* Data to write */ + + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); + + pData = pList->pData; + + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + + /* If page 1 was just written, update Pager.dbFileVers to match + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. + */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); + } + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + pPager->aStat[PAGER_STAT_WRITE]++; + + /* Update any backup objects copying the contents of this pager. */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); + + PAGERTRACE(("STORE %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pList))); + IOTRACE(("PGOUT %p %d\n", pPager, pgno)); + PAGER_INCR(sqlite3_pager_writedb_count); + }else{ + PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); + } + pager_set_pagehash(pList); + pList = pList->pDirty; + } + + return rc; +} + +/* +** Ensure that the sub-journal file is open. If it is already open, this +** function is a no-op. +** +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** fails. +*/ +static int openSubJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nStmtSpill = sqlite3Config.nStmtSpill; + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ + nStmtSpill = -1; + } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); + } + return rc; +} + +/* +** Append a record of the current state of page pPg to the sub-journal. +** +** If successful, set the bit corresponding to pPg->pgno in the bitvecs +** for all open savepoints before returning. +** +** This function returns SQLITE_OK if everything is successful, an IO +** error code if the attempt to write to the sub-journal fails, or +** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint +** bitvec. +*/ +static int subjournalPage(PgHdr *pPg){ + int rc = SQLITE_OK; + Pager *pPager = pPg->pPager; + if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + + /* Open the sub-journal, if it has not already been opened */ + assert( pPager->useJournal ); + assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); + assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); + assert( pagerUseWal(pPager) + || pageInJournal(pPager, pPg) + || pPg->pgno>pPager->dbOrigSize + ); + rc = openSubJournal(pPager); + + /* If the sub-journal was opened successfully (or was already open), + ** write the journal record into the file. */ + if( rc==SQLITE_OK ){ + void *pData = pPg->pData; + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); + char *pData2; + pData2 = pData; + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); + } + } + } + if( rc==SQLITE_OK ){ + pPager->nSubRec++; + assert( pPager->nSavepoint>0 ); + rc = addToSavepointBitvecs(pPager, pPg->pgno); + } + return rc; +} +static int subjournalPageIfRequired(PgHdr *pPg){ + if( subjRequiresPage(pPg) ){ + return subjournalPage(pPg); + }else{ + return SQLITE_OK; + } +} + +/* +** This function is called by the pcache layer when it has reached some +** soft memory limit. The first argument is a pointer to a Pager object +** (cast as a void*). The pager is always 'purgeable' (not an in-memory +** database). The second argument is a reference to a page that is +** currently dirty but has no outstanding references. The page +** is always associated with the Pager object passed as the first +** argument. +** +** The job of this function is to make pPg clean by writing its contents +** out to the database file, if possible. This may involve syncing the +** journal file. +** +** If successful, sqlite3PcacheMakeClean() is called on the page and +** SQLITE_OK returned. If an IO error occurs while trying to make the +** page clean, the IO error code is returned. If the page cannot be +** made clean for some other reason, but no error occurs, then SQLITE_OK +** is returned by sqlite3PcacheMakeClean() is not called. +*/ +static int pagerStress(void *p, PgHdr *pPg){ + Pager *pPager = (Pager *)p; + int rc = SQLITE_OK; + + assert( pPg->pPager==pPager ); + assert( pPg->flags&PGHDR_DIRTY ); + + /* The doNotSpill NOSYNC bit is set during times when doing a sync of + ** journal (and adding a new header) is not allowed. This occurs + ** during calls to sqlite3PagerWrite() while trying to journal multiple + ** pages belonging to the same sector. + ** + ** The doNotSpill ROLLBACK and OFF bits inhibits all cache spilling + ** regardless of whether or not a sync is required. This is set during + ** a rollback or by user request, respectively. + ** + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementation it + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. + */ + if( NEVER(pPager->errCode) ) return SQLITE_OK; + testcase( pPager->doNotSpill & SPILLFLAG_ROLLBACK ); + testcase( pPager->doNotSpill & SPILLFLAG_OFF ); + testcase( pPager->doNotSpill & SPILLFLAG_NOSYNC ); + if( pPager->doNotSpill + && ((pPager->doNotSpill & (SPILLFLAG_ROLLBACK|SPILLFLAG_OFF))!=0 + || (pPg->flags & PGHDR_NEED_SYNC)!=0) + ){ + return SQLITE_OK; + } + + pPager->aStat[PAGER_STAT_SPILL]++; + pPg->pDirty = 0; + if( pagerUseWal(pPager) ){ + /* Write a single frame for this page to the log. */ + rc = subjournalPageIfRequired(pPg); + if( rc==SQLITE_OK ){ + rc = pagerWalFrames(pPager, pPg, 0, 0); + } + }else{ + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->tempFile==0 ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) return pager_error(pPager, rc); + } +#endif + + /* Sync the journal file if required. */ + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); + } + + /* Write the contents of the page out to the database file. */ + if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); + rc = pager_write_pagelist(pPager, pPg); + } + } + + /* Mark the page as clean. */ + if( rc==SQLITE_OK ){ + PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); + sqlite3PcacheMakeClean(pPg); + } + + return pager_error(pPager, rc); +} + +/* +** Flush all unreferenced dirty pages to disk. +*/ +int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} + +/* +** Allocate and initialize a new Pager object and put a pointer to it +** in *ppPager. The pager should eventually be freed by passing it +** to sqlite3PagerClose(). +** +** The zFilename argument is the path to the database file to open. +** If zFilename is NULL then a randomly-named temporary file is created +** and used as the file to be cached. Temporary files are be deleted +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. +** This can be used to implement an in-memory database. +** +** The nExtra parameter specifies the number of bytes of space allocated +** along with each page reference. This space is available to the user +** via the sqlite3PagerGetExtra() API. When a new page is allocated, the +** first 8 bytes of this space are zeroed but the remainder is uninitialized. +** (The extra space is used by btree as the MemPage object.) +** +** The flags argument is used to specify properties that affect the +** operation of the pager. It should be passed some bitwise combination +** of the PAGER_* flags. +** +** The vfsFlags parameter is a bitmask to pass to the flags parameter +** of the xOpen() method of the supplied VFS when opening files. +** +** If the pager object is allocated and the specified file opened +** successfully, SQLITE_OK is returned and *ppPager set to point to +** the new pager object. If an error occurs, *ppPager is set to NULL +** and error code returned. This function may return SQLITE_NOMEM +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** various SQLITE_IO_XXX errors. +*/ +int sqlite3PagerOpen( + sqlite3_vfs *pVfs, /* The virtual file system to use */ + Pager **ppPager, /* OUT: Return the Pager structure here */ + const char *zFilename, /* Name of the database file to open */ + int nExtra, /* Extra bytes append to each in-memory page */ + int flags, /* flags controlling this file */ + int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */ + void (*xReinit)(DbPage*) /* Function to reinitialize pages */ +){ + u8 *pPtr; + Pager *pPager = 0; /* Pager object to allocate and return */ + int rc = SQLITE_OK; /* Return code */ + int tempFile = 0; /* True for temp files (incl. in-memory files) */ + int memDb = 0; /* True if this is an in-memory file */ +#ifdef SQLITE_ENABLE_DESERIALIZE + int memJM = 0; /* Memory journal mode */ +#else +# define memJM 0 +#endif + int readOnly = 0; /* True if this is a read-only file */ + int journalFileSize; /* Bytes to allocate for each journal fd */ + char *zPathname = 0; /* Full path to database file */ + int nPathname = 0; /* Number of bytes in zPathname */ + int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ + int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + int nUri = 0; /* Number of URI parameters */ + + /* Figure out how much space is required for each journal file-handle + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + + /* Set the output variable to NULL in case an error occurs. */ + *ppPager = 0; + +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + if( zFilename && zFilename[0] ){ + zPathname = sqlite3DbStrDup(0, zFilename); + if( zPathname==0 ) return SQLITE_NOMEM_BKPT; + nPathname = sqlite3Strlen30(zPathname); + zFilename = 0; + } + } +#endif + + /* Compute and store the full pathname in an allocated buffer pointed + ** to by zPathname, length nPathname. Or, if this is a temporary file, + ** leave both nPathname and zPathname set to 0. + */ + if( zFilename && zFilename[0] ){ + const char *z; + nPathname = pVfs->mxPathname+1; + zPathname = sqlite3DbMallocRaw(0, nPathname*2); + if( zPathname==0 ){ + return SQLITE_NOMEM_BKPT; + } + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK_SYMLINK ){ + if( vfsFlags & SQLITE_OPEN_NOFOLLOW ){ + rc = SQLITE_CANTOPEN_SYMLINK; + }else{ + rc = SQLITE_OK; + } + } + } + nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += strlen(z)+1; + z += strlen(z)+1; + nUri++; + } + nUriByte = (int)(&z[1] - zUri); + assert( nUriByte>=1 ); + if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ + /* This branch is taken when the journal path required by + ** the database being opened will be more than pVfs->mxPathname + ** bytes in length. This means the database cannot be opened, + ** as it will not be possible to open the journal file or even + ** check for a hot-journal before reading. + */ + rc = SQLITE_CANTOPEN_BKPT; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(0, zPathname); + return rc; + } + } + + /* Allocate memory for the Pager structure, PCache object, the + ** three file descriptors, the database file name and the journal + ** file name. The layout in memory is as follows: + ** + ** Pager object (sizeof(Pager) bytes) + ** PCache object (sqlite3PcacheSize() bytes) + ** Database file handle (pVfs->szOsFile bytes) + ** Sub-journal file handle (journalFileSize bytes) + ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) + ** \0\0\0\0 database prefix (4 bytes) + ** Database file name (nPathname+1 bytes) + ** URI query parameters (nUriByte bytes) + ** Journal filename (nPathname+8+1 bytes) + ** WAL filename (nPathname+4+1 bytes) + ** \0\0\0 terminator (3 bytes) + ** + ** Some 3rd-party software, over which we have no control, depends on + ** the specific order of the filenames and the \0 separators between them + ** so that it can (for example) find the database filename given the WAL + ** filename without using the sqlite3_filename_database() API. This is a + ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party + ** software is in widespread use, so we try to avoid changing the filename + ** order and formatting if possible. In particular, the details of the + ** filename format expected by 3rd-party software should be as follows: + ** + ** - Main Database Path + ** - \0 + ** - Multiple URI components consisting of: + ** - Key + ** - \0 + ** - Value + ** - \0 + ** - \0 + ** - Journal Path + ** - \0 + ** - WAL Path (zWALName) + ** - \0 + ** + ** The sqlite3_create_filename() interface and the databaseFilename() utility + ** that is used by sqlite3_filename_database() and kin also depend on the + ** specific formatting and order of the various filenames, so if the format + ** changes here, be sure to change it there as well. + */ + pPtr = (u8 *)sqlite3MallocZero( + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + sizeof(pPager) + /* Space to hold a pointer */ + 4 + /* Database prefix */ + nPathname + 1 + /* database filename */ + nUriByte + /* query parameters */ + nPathname + 8 + 1 + /* Journal filename */ +#ifndef SQLITE_OMIT_WAL + nPathname + 4 + 1 + /* WAL filename */ +#endif + 3 /* Terminator */ + ); + assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); + if( !pPtr ){ + sqlite3DbFree(0, zPathname); + return SQLITE_NOMEM_BKPT; + } + pPager = (Pager*)pPtr; pPtr += ROUND8(sizeof(*pPager)); + pPager->pPCache = (PCache*)pPtr; pPtr += ROUND8(pcacheSize); + pPager->fd = (sqlite3_file*)pPtr; pPtr += ROUND8(pVfs->szOsFile); + pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + + /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ + pPtr += 4; /* Skip zero prefix */ + pPager->zFilename = (char*)pPtr; + if( nPathname>0 ){ + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname + 1; + if( zUri ){ + memcpy(pPtr, zUri, nUriByte); pPtr += nUriByte; + }else{ + pPtr++; + } + } + + + /* Fill in Pager.zJournal */ + if( nPathname>0 ){ + pPager->zJournal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-journal",8); pPtr += 8 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename,pPager->zJournal); + pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1); +#endif + }else{ + pPager->zJournal = 0; + } + +#ifndef SQLITE_OMIT_WAL + /* Fill in Pager.zWal */ + if( nPathname>0 ){ + pPager->zWal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-wal", 4); pPtr += 4 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename, pPager->zWal); + pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1); +#endif + }else{ + pPager->zWal = 0; + } +#endif + + if( nPathname ) sqlite3DbFree(0, zPathname); + pPager->pVfs = pVfs; + pPager->vfsFlags = vfsFlags; + + /* Open the pager file. + */ + if( zFilename && zFilename[0] ){ + int fout = 0; /* VFS flags returned by xOpen() */ + rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); +#ifdef SQLITE_ENABLE_DESERIALIZE + memJM = (fout&SQLITE_OPEN_MEMORY)!=0; +#endif + readOnly = (fout&SQLITE_OPEN_READONLY)!=0; + + /* If the file was successfully opened for read/write access, + ** choose a default page size in case we have to create the + ** database file. The default page size is the maximum of: + ** + ** + SQLITE_DEFAULT_PAGE_SIZE, + ** + The value returned by sqlite3OsSectorSize() + ** + The largest page size that can be written atomically. + */ + if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + if( !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } + } + } +#endif + } + pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } + } + }else{ + /* If a temporary file is requested, it is not opened immediately. + ** In this case we accept the default page size and delay actually + ** opening the file until the first call to OsWrite(). + ** + ** This branch is also run for an in-memory database. An in-memory + ** database is the same as a temp-file that is never written out to + ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. + */ +act_like_temp_file: + tempFile = 1; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ + pPager->noLock = 1; /* Do no locking */ + readOnly = (vfsFlags&SQLITE_OPEN_READONLY); + } + + /* The following call to PagerSetPagesize() serves to set the value of + ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. + */ + if( rc==SQLITE_OK ){ + assert( pPager->memDb==0 ); + rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1); + testcase( rc!=SQLITE_OK ); + } + + /* Initialize the PCache object. */ + if( rc==SQLITE_OK ){ + nExtra = ROUND8(nExtra); + assert( nExtra>=8 && nExtra<1000 ); + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } + + /* If an error occurred above, free the Pager structure and close the file. + */ + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pPager->pTmpSpace); + sqlite3_free(pPager); + return rc; + } + + PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); + IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) + + pPager->useJournal = (u8)useJournal; + /* pPager->stmtOpen = 0; */ + /* pPager->stmtInUse = 0; */ + /* pPager->nRef = 0; */ + /* pPager->stmtSize = 0; */ + /* pPager->stmtJSize = 0; */ + /* pPager->nPage = 0; */ + pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; + /* pPager->state = PAGER_UNLOCK; */ + /* pPager->errMask = 0; */ + pPager->tempFile = (u8)tempFile; + assert( tempFile==PAGER_LOCKINGMODE_NORMAL + || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); + pPager->exclusiveMode = (u8)tempFile; + pPager->changeCountDone = pPager->tempFile; + pPager->memDb = (u8)memDb; + pPager->readOnly = (u8)readOnly; + assert( useJournal || pPager->tempFile ); + pPager->noSync = pPager->tempFile; + if( pPager->noSync ){ + assert( pPager->fullSync==0 ); + assert( pPager->extraSync==0 ); + assert( pPager->syncFlags==0 ); + assert( pPager->walSyncFlags==0 ); + }else{ + pPager->fullSync = 1; + pPager->extraSync = 0; + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); + } + /* pPager->pFirst = 0; */ + /* pPager->pFirstSynced = 0; */ + /* pPager->pLast = 0; */ + pPager->nExtra = (u16)nExtra; + pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; + assert( isOpen(pPager->fd) || tempFile ); + setSectorSize(pPager); + if( !useJournal ){ + pPager->journalMode = PAGER_JOURNALMODE_OFF; + }else if( memDb || memJM ){ + pPager->journalMode = PAGER_JOURNALMODE_MEMORY; + } + /* pPager->xBusyHandler = 0; */ + /* pPager->pBusyHandlerArg = 0; */ + pPager->xReiniter = xReinit; + setGetterMethod(pPager); + /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ + + *ppPager = pPager; + return SQLITE_OK; +} + +/* +** Return the sqlite3_file for the main database given the name +** of the corresonding WAL or Journal name as passed into +** xOpen. +*/ +sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + return pPager->fd; +} + + +/* +** This function is called after transitioning from PAGER_UNLOCK to +** PAGER_SHARED state. It tests if there is a hot journal present in +** the file-system for the given pager. A hot journal is one that +** needs to be played back. According to this function, a hot-journal +** file exists if the following criteria are met: +** +** * The journal file exists in the file system, and +** * No process holds a RESERVED or greater lock on the database file, and +** * The database file itself is greater than 0 bytes in size, and +** * The first byte of the journal file exists and is not 0x00. +** +** If the current size of the database file is 0 but a journal file +** exists, that is probably an old journal left over from a prior +** database with the same name. In this case the journal file is +** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK +** is returned. +** +** This routine does not check if there is a super-journal filename +** at the end of the file. If there is, and that super-journal file +** does not exist, then the journal file is not really hot. In this +** case this routine will return a false-positive. The pager_playback() +** routine will discover that the journal file is not really hot and +** will not roll it back. +** +** If a hot-journal file is found to exist, *pExists is set to 1 and +** SQLITE_OK returned. If no hot-journal file is present, *pExists is +** set to 0 and SQLITE_OK returned. If an IO error occurs while trying +** to determine whether or not a hot-journal file exists, the IO error +** code is returned and the value of *pExists is undefined. +*/ +static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int rc = SQLITE_OK; /* Return code */ + int exists = 1; /* True if a journal file is present */ + int jrnlOpen = !!isOpen(pPager->jfd); + + assert( pPager->useJournal ); + assert( isOpen(pPager->fd) ); + assert( pPager->eState==PAGER_OPEN ); + + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN + )); + + *pExists = 0; + if( !jrnlOpen ){ + rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); + } + if( rc==SQLITE_OK && exists ){ + int locked = 0; /* True if some process holds a RESERVED lock */ + + /* Race condition here: Another process might have been holding the + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** call above, but then delete the journal and drop the lock before + ** we get to the following sqlite3OsCheckReservedLock() call. If that + ** is the case, this routine might think there is a hot journal when + ** in fact there is none. This results in a false-positive which will + ** be dealt with by the playback routine. Ticket #3883. + */ + rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); + if( rc==SQLITE_OK && !locked ){ + Pgno nPage; /* Number of pages in database file */ + + assert( pPager->tempFile==0 ); + rc = pagerPagecount(pPager, &nPage); + if( rc==SQLITE_OK ){ + /* If the database is zero pages in size, that means that either (1) the + ** journal is a remnant from a prior database with the same name where + ** the database file but not the journal was deleted, or (2) the initial + ** transaction that populates a new database is being rolled back. + ** In either case, the journal file can be deleted. However, take care + ** not to delete the journal file if it is already open due to + ** journal_mode=PERSIST. + */ + if( nPage==0 && !jrnlOpen ){ + sqlite3BeginBenignMalloc(); + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + sqlite3EndBenignMalloc(); + }else{ + /* The journal file exists and no other connection has a reserved + ** or greater lock on the database file. Now check that there is + ** at least one non-zero bytes at the start of the journal file. + ** If there is, then we consider this journal to be hot. If not, + ** it can be ignored. + */ + if( !jrnlOpen ){ + int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); + } + if( rc==SQLITE_OK ){ + u8 first = 0; + rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + if( !jrnlOpen ){ + sqlite3OsClose(pPager->jfd); + } + *pExists = (first!=0); + }else if( rc==SQLITE_CANTOPEN ){ + /* If we cannot open the rollback journal file in order to see if + ** it has a zero header, that might be due to an I/O error, or + ** it might be due to the race condition described above and in + ** ticket #3883. Either way, assume that the journal is hot. + ** This might be a false positive. But if it is, then the + ** automatic journal playback and recovery mechanism will deal + ** with it under an EXCLUSIVE lock where we do not need to + ** worry so much with race conditions. + */ + *pExists = 1; + rc = SQLITE_OK; + } + } + } + } + } + + return rc; +} + +/* +** This function is called to obtain a shared lock on the database file. +** It is illegal to call sqlite3PagerGet() until after this function +** has been successfully called. If a shared-lock is already held when +** this function is called, it is a no-op. +** +** The following operations are also performed by this function. +** +** 1) If the pager is currently in PAGER_OPEN state (no lock held +** on the database file), then an attempt is made to obtain a +** SHARED lock on the database file. Immediately after obtaining +** the SHARED lock, the file-system is checked for a hot-journal, +** which is played back if present. Following any hot-journal +** rollback, the contents of the cache are validated by checking +** the 'change-counter' field of the database file header and +** discarded if they are found to be invalid. +** +** 2) If the pager is running in exclusive-mode, and there are currently +** no outstanding references to any pages, and is in the error state, +** then an attempt is made to clear the error state by discarding +** the contents of the page cache and rolling back any open journal +** file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. +*/ +int sqlite3PagerSharedLock(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine is only called from b-tree and only when there are no + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. */ + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + assert( pPager->errCode==SQLITE_OK ); + + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ + + assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); + + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; + } + + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); + } + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling the + ** hot-journal back. + ** + ** Because the intermediate RESERVED lock is not requested, any + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + goto failed; + } + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. Purge the cache before + ** playing back the hot-journal so that we don't end up with + ** an inconsistent cache. Sync the hot journal before playing + ** it back since the process that crashed and left the hot journal + ** probably did not sync it and we are required to always sync + ** the journal before playing it back. + */ + if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); + rc = pagerSyncHotJournal(pPager); + if( rc==SQLITE_OK ){ + rc = pager_playback(pPager, !pPager->tempFile); + pPager->eState = PAGER_OPEN; + } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) + ); + } + + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The hasHeldSharedLock flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. + ** + ** Database changes are detected by looking at 15 bytes beginning + ** at offset 24 into the file. The first 4 of these 16 bytes are + ** a 32-bit counter that is incremented with each change. The + ** other bytes change randomly with each file change when + ** a codec is in use. + ** + ** There is a vanishingly small chance that a change will not be + ** detected. The chance of an undetected change is so small that + ** it can be neglected. + */ + char dbFileVers[sizeof(pPager->dbFileVers)]; + + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_IOERR_SHORT_READ ){ + goto failed; + } + memset(dbFileVers, 0, sizeof(dbFileVers)); + } + + if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ + pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } + } + } + + /* If there is a WAL file in the file-system, open this database in WAL + ** mode. Otherwise, the following function call is a no-op. + */ + rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); + } + + failed: + if( rc!=SQLITE_OK ){ + assert( !MEMDB ); + pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; + } + return rc; +} + +/* +** If the reference count has reached zero, rollback any active +** transaction and unlock the pager. +** +** Except, in locking_mode=EXCLUSIVE when there is nothing to in +** the rollback journal, the unlock is not performed and there is +** nothing to rollback, so this routine is a no-op. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){ + assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ + pagerUnlockAndRollback(pPager); + } +} + +/* +** The page getter methods each try to acquire a reference to a +** page with page number pgno. If the requested reference is +** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. +** +** There are different implementations of the getter method depending +** on the current state of the pager. +** +** getPageNormal() -- The normal getter +** getPageError() -- Used if the pager is in an error state +** getPageMmap() -- Used if memory-mapped I/O is enabled +** +** If the requested page is already in the cache, it is returned. +** Otherwise, a new page object is allocated and populated with data +** read from the database file. In some cases, the pcache module may +** choose not to allocate a new page object and may reuse an existing +** object with no outstanding references. +** +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is +** already in the cache when this function is called, then the extra +** data is left as it was when the page object was last used. +** +** If the database image is smaller than the requested page or if +** the flags parameter contains the PAGER_GET_NOCONTENT bit and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. +** +** If PAGER_GET_NOCONTENT is true, it means that we do not care about +** the contents of the page. This occurs in two scenarios: +** +** a) When reading a free-list leaf page from the database, and +** +** b) When a savepoint is being rolled back and we need to load +** a new page into the cache to be filled with the data read +** from the savepoint journal. +** +** If PAGER_GET_NOCONTENT is true, then the data returned is zeroed instead +** of being read from the database. Additionally, the bits corresponding +** to pgno in Pager.pInJournal (bitvec of pages already written to the +** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open +** savepoints are set. This means if the page is made writable at any +** point in the future, using a call to sqlite3PagerWrite(), its contents +** will not be journaled. This saves IO. +** +** The acquisition might fail for several reasons. In all cases, +** an appropriate error code is returned and *ppPage is set to NULL. +** +** See also sqlite3PagerLookup(). Both this routine and Lookup() attempt +** to find a page in the in-memory cache first. If the page is not already +** in memory, this routine goes to disk to read it in whereas Lookup() +** just returns 0. This routine acquires a read-lock the first time it +** has to go to disk, and could also playback an old journal if necessary. +** Since Lookup() never goes to disk, it never has to deal with locks +** or journal files. +*/ +static int getPageNormal( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg; + u8 noContent; /* True if PAGER_GET_NOCONTENT is set */ + sqlite3_pcache_page *pBase; + + assert( pPager->errCode==SQLITE_OK ); + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + + if( pgno==0 ) return SQLITE_CORRUPT_BKPT; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + pPg = 0; + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto pager_acquire_err; + } + } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); + assert( pPg==(*ppPage) ); + assert( pPg->pgno==pgno ); + assert( pPg->pPager==pPager || pPg->pPager==0 ); + + noContent = (flags & PAGER_GET_NOCONTENT)!=0; + if( pPg->pPager && !noContent ){ + /* In this case the pcache already contains an initialized copy of + ** the page. Return without further ado. */ + assert( pgno!=PAGER_MJ_PGNO(pPager) ); + pPager->aStat[PAGER_STAT_HIT]++; + return SQLITE_OK; + + }else{ + /* The pager cache has created a new page. Its content needs to + ** be initialized. But first some error checks: + ** + ** (*) obsolete. Was: maximum page number is 2^31 + ** (2) Never try to fetch the locking page + */ + if( pgno==PAGER_MJ_PGNO(pPager) ){ + rc = SQLITE_CORRUPT_BKPT; + goto pager_acquire_err; + } + + pPg->pPager = pPager; + + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ + rc = SQLITE_FULL; + goto pager_acquire_err; + } + if( noContent ){ + /* Failure to set the bits in the InJournal bit-vectors is benign. + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set + ** a bit in a bit vector. + */ + sqlite3BeginBenignMalloc(); + if( pgno<=pPager->dbOrigSize ){ + TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pgno); + testcase( rc==SQLITE_NOMEM ); + } + TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); + testcase( rc==SQLITE_NOMEM ); + sqlite3EndBenignMalloc(); + } + memset(pPg->pData, 0, pPager->pageSize); + IOTRACE(("ZERO %p %d\n", pPager, pgno)); + }else{ + assert( pPg->pPager==pPager ); + pPager->aStat[PAGER_STAT_MISS]++; + rc = readDbPage(pPg); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + pager_set_pagehash(pPg); + } + return SQLITE_OK; + +pager_acquire_err: + assert( rc!=SQLITE_OK ); + if( pPg ){ + sqlite3PcacheDrop(pPg); + } + pagerUnlockIfUnused(pPager); + *ppPage = 0; + return rc; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* The page getter for when memory-mapped I/O is enabled */ +static int getPageMMap( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno>1 + && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) + ); + + assert( USEFETCH(pPager) ); + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + assert( pPager->errCode==SQLITE_OK ); + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + if( bMmapOk && iFrame==0 ){ + void *pData = 0; + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ + pPg = sqlite3PagerLookup(pPager, pgno); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + return getPageNormal(pPager, pgno, ppPage, flags); +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* The page getter method for when the pager is an error state */ +static int getPageError( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + UNUSED_PARAMETER(pgno); + UNUSED_PARAMETER(flags); + assert( pPager->errCode!=SQLITE_OK ); + *ppPage = 0; + return pPager->errCode; +} + + +/* Dispatch all page fetch requests to the appropriate getter method. +*/ +int sqlite3PagerGet( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + return pPager->xGet(pPager, pgno, ppPage, flags); +} + +/* +** Acquire a page if it is already in the in-memory cache. Do +** not read the page from disk. Return a pointer to the page, +** or 0 if the page is not in cache. +** +** See also sqlite3PagerGet(). The difference between this routine +** and sqlite3PagerGet() is that _get() will go to the disk and read +** in the page if the page is not already in cache. This routine +** returns NULL if the page is not in cache or if a disk I/O error +** has ever happened. +*/ +DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ + sqlite3_pcache_page *pPage; + assert( pPager!=0 ); + assert( pgno!=0 ); + assert( pPager->pPCache!=0 ); + pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasHeldSharedLock ); + if( pPage==0 ) return 0; + return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); +} + +/* +** Release a page reference. +** +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be +** used if we know that the page being released is not the last page. +** The btree layer always holds page1 open until the end, so these first +** to routines can be used to release any page other than BtShared.pPage1. +** +** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine +** checks the total number of outstanding pages and if the number of +** pages reaches zero it drops the database lock. +*/ +void sqlite3PagerUnrefNotNull(DbPage *pPg){ + TESTONLY( Pager *pPager = pPg->pPager; ) + assert( pPg!=0 ); + if( pPg->flags & PGHDR_MMAP ){ + assert( pPg->pgno!=1 ); /* Page1 is never memory mapped */ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } + /* Do not use this routine to release the last reference to page1 */ + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); +} +void sqlite3PagerUnref(DbPage *pPg){ + if( pPg ) sqlite3PagerUnrefNotNull(pPg); +} +void sqlite3PagerUnrefPageOne(DbPage *pPg){ + Pager *pPager; + assert( pPg!=0 ); + assert( pPg->pgno==1 ); + assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ + pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); +} + +/* +** This function is called at the start of every write transaction. +** There must already be a RESERVED or EXCLUSIVE lock on the database +** file when this routine is called. +** +** Open the journal file for pager pPager and write a journal header +** to the start of it. If there are active savepoints, open the sub-journal +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used +** when opening a hot journal file to roll it back. +** +** If the journal file is already open (as it may be in exclusive mode), +** then this function just writes a journal header to the start of the +** already open file. +** +** Whether or not the journal file is opened by this function, the +** Pager.pInJournal bitvec structure is allocated. +** +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** an IO error code if opening or writing the journal file fails. +*/ +static int pager_open_journal(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ + + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( pPager->pInJournal==0 ); + + /* If already in the error state, this function is a no-op. But on + ** the other hand, this routine is never called if we are already in + ** an error state. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM_BKPT; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + + /* Verify that the database still has the same name as it did when + ** it was originally opened. */ + rc = databaseIsUnmoved(pPager); + if( rc==SQLITE_OK ){ + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill + ); + } + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setSuper = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; + } + + return rc; +} + +/* +** Begin a write-transaction on the specified pager object. If a +** write-transaction has already been opened, this function is a no-op. +** +** If the exFlag argument is false, then acquire at least a RESERVED +** lock on the database file. If exFlag is true, then acquire at least +** an EXCLUSIVE lock. If such a lock is already held, no locking +** functions need be called. +** +** If the subjInMemory argument is non-zero, then any sub-journal opened +** within this transaction will be opened as an in-memory file. This +** has no effect if the sub-journal is already opened (as it may be when +** running in exclusive mode) or if the transaction does not require a +** sub-journal. If the subjInMemory argument is zero, then any required +** sub-journal is implemented in-memory if pPager is an in-memory database, +** or using a temporary file otherwise. +*/ +int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ + int rc = SQLITE_OK; + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + + if( ALWAYS(pPager->eState==PAGER_READER) ){ + assert( pPager->pInJournal==0 ); + + if( pagerUseWal(pPager) ){ + /* If the pager is configured to use locking_mode=exclusive, and an + ** exclusive lock on the database is not already held, obtain it now. + */ + if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + return rc; + } + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); + } + + /* Grab the write lock on the log file. If successful, upgrade to + ** PAGER_RESERVED state. Otherwise, return an error code to the caller. + ** The busy-handler is not invoked if another connection already + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The + ** busy-handler callback can be used when upgrading to the EXCLUSIVE + ** lock, but not when obtaining the RESERVED lock. + */ + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + } + + PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); + return rc; +} + +/* +** Write page pPg onto the end of the rollback journal. +*/ +static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc; + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + pData2 = pPg->pData; + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + return rc; +} + +/* +** Mark a single data page as writeable. The page is written into the +** main journal or sub-journal as required. If the page is written into +** one of the journals, the corresponding bit is set in the +** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs +** of any open savepoints as appropriate. +*/ +static int pager_write(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc = SQLITE_OK; + + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. + */ + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( pPager->errCode==0 ); + assert( pPager->readOnly==0 ); + CHECK_PAGE(pPg); + + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); + + /* Mark the page that is about to be modified as dirty. */ + sqlite3PcacheMakeDirty(pPg); + + /* If a rollback journal is in use, them make sure the page that is about + ** to change is in the rollback journal, or if the page is a new page off + ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + */ + assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); + if( pPager->pInJournal!=0 + && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 + ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize ){ + rc = pagerAddPageToRollbackJournal(pPg); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + } + } + + /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list + ** and before writing the page into the rollback journal. Wait until now, + ** after the page has been successfully journalled, before setting the + ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. + */ + pPg->flags |= PGHDR_WRITEABLE; + + /* If the statement journal is open and the page is not in it, + ** then write the page into the statement journal. + */ + if( pPager->nSavepoint>0 ){ + rc = subjournalPageIfRequired(pPg); + } + + /* Update the database size and return. */ + if( pPager->dbSizepgno ){ + pPager->dbSize = pPg->pgno; + } + return rc; +} + +/* +** This is a variant of sqlite3PagerWrite() that runs when the sector size +** is larger than the page size. SQLite makes the (reasonable) assumption that +** all bytes of a sector are written together by hardware. Hence, all bytes of +** a sector need to be journalled in case of a power loss in the middle of +** a write. +** +** Usually, the sector size is less than or equal to the page size, in which +** case pages can be individually written. This routine only runs in the +** exceptional case where the page size is smaller than the sector size. +*/ +static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage, 0); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = sqlite3PagerLookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + return rc; +} + +/* +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless +** this routine returns SQLITE_OK. +** +** The difference between this function and pager_write() is that this +** function also deals with the special case where 2 or more pages +** fit on a single disk sector. In this case all co-resident pages +** must have been written to the journal file before returning. +** +** If an error occurs, SQLITE_NOMEM or an IO error code is returned +** as appropriate. Otherwise, SQLITE_OK. +*/ +int sqlite3PagerWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( (pPg->flags & PGHDR_MMAP)==0 ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); + return SQLITE_OK; + }else if( pPager->errCode ){ + return pPager->errCode; + }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); + return pagerWriteLargeSector(pPg); + }else{ + return pager_write(pPg); + } +} + +/* +** Return TRUE if the page given in the argument was previously passed +** to sqlite3PagerWrite(). In other words, return TRUE if it is ok +** to change the content of the page. +*/ +#ifndef NDEBUG +int sqlite3PagerIswriteable(DbPage *pPg){ + return pPg->flags & PGHDR_WRITEABLE; +} +#endif + +/* +** A call to this routine tells the pager that it is not necessary to +** write the information on page pPg back to the disk, even though +** that page might be marked as dirty. This happens, for example, when +** the page has been added as a leaf of the freelist and so its +** content no longer matters. +** +** The overlying software layer calls this routine when all of the data +** on the given page is unused. The pager marks the page as clean so +** that it does not get written to disk. +** +** Tests show that this optimization can quadruple the speed of large +** DELETE operations. +** +** This optimization cannot be used with a temp-file, as the page may +** have been dirty at the start of the transaction. In that case, if +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the +** current transaction is rolled back. +*/ +void sqlite3PagerDontWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); + IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) + pPg->flags |= PGHDR_DONT_WRITE; + pPg->flags &= ~PGHDR_WRITEABLE; + testcase( pPg->flags & PGHDR_NEED_SYNC ); + pager_set_pagehash(pPg); + } +} + +/* +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at +** byte offset 24 of the pager file. The secondary change counter at +** 92 is also updated, as is the SQLite version number at offset 96. +** +** But this only happens if the pPager->changeCountDone flag is false. +** To avoid excess churning of page 1, the update only happens once. +** See also the pager_write_changecounter() routine that does an +** unconditional update of the change counters. +** +** If the isDirectMode flag is zero, then this is done by calling +** sqlite3PagerWrite() on page 1, then modifying the contents of the +** page data. In this case the file will be updated when the current +** transaction is committed. +** +** The isDirectMode flag may only be non-zero if the library was compiled +** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, +** if isDirect is non-zero, then the database file is updated directly +** by writing an updated version of page 1 using a call to the +** sqlite3OsWrite() function. +*/ +static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ + int rc = SQLITE_OK; + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* Declare and initialize constant integer 'isDirect'. If the + ** atomic-write optimization is enabled in this build, then isDirect + ** is initialized to the value passed as the isDirectMode parameter + ** to this function. Otherwise, it is always set to zero. + ** + ** The idea is that if the atomic-write optimization is not + ** enabled at compile time, the compiler can omit the tests of + ** 'isDirect' below, as well as the block enclosed in the + ** "if( isDirect )" condition. + */ +#ifndef SQLITE_ENABLE_ATOMIC_WRITE +# define DIRECT_MODE 0 + assert( isDirectMode==0 ); + UNUSED_PARAMETER(isDirectMode); +#else +# define DIRECT_MODE isDirectMode +#endif + + if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + PgHdr *pPgHdr; /* Reference to page 1 */ + + assert( !pPager->tempFile && isOpen(pPager->fd) ); + + /* Open page 1 of the file for writing. */ + rc = sqlite3PagerGet(pPager, 1, &pPgHdr, 0); + assert( pPgHdr==0 || rc==SQLITE_OK ); + + /* If page one was fetched successfully, and this function is not + ** operating in direct-mode, make page 1 writable. When not in + ** direct mode, page 1 is always held in cache and hence the PagerGet() + ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. + */ + if( !DIRECT_MODE && ALWAYS(rc==SQLITE_OK) ){ + rc = sqlite3PagerWrite(pPgHdr); + } + + if( rc==SQLITE_OK ){ + /* Actually do the update of the change counter */ + pager_write_changecounter(pPgHdr); + + /* If running in direct mode, write the contents of page 1 to the file. */ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); + zBuf = pPgHdr->pData; + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; + } + if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); + pPager->changeCountDone = 1; + } + }else{ + pPager->changeCountDone = 1; + } + } + + /* Release the page reference. */ + sqlite3PagerUnref(pPgHdr); + } + return rc; +} + +/* +** Sync the database file to disk. This is a no-op for in-memory databases +** or pages with the Pager.noSync flag set. +** +** If successful, or if called on a pager for which it is a no-op, this +** function returns SQLITE_OK. Otherwise, an IO error code is returned. +*/ +int sqlite3PagerSync(Pager *pPager, const char *zSuper){ + int rc = SQLITE_OK; + void *pArg = (void*)zSuper; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc==SQLITE_OK && !pPager->noSync ){ + assert( !MEMDB ); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); + } + return rc; +} + +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = pPager->errCode; + assert( assert_pager_state(pPager) ); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + return rc; +} + +/* +** Sync the database file for the pager pPager. zSuper points to the name +** of a super-journal file that should be written into the individual +** journal file. zSuper may be NULL, which is interpreted as no +** super-journal (a single database transaction). +** +** This routine ensures that: +** +** * The database file change-counter is updated, +** * the journal is synced (unless the atomic-write optimization is used), +** * all dirty pages are written to the database file, +** * the database file is truncated (if required), and +** * the database file synced. +** +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or +** delete the super-journal file if specified). +** +** Note that if zSuper==NULL, this does not overwrite a previous value +** passed to an sqlite3PagerCommitPhaseOne() call. +** +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. +*/ +int sqlite3PagerCommitPhaseOne( + Pager *pPager, /* Pager object */ + const char *zSuper, /* If not NULL, the super-journal name */ + int noSync /* True to omit the xSync on the db file */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); + + /* If a prior error occurred, report that error again. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + /* Provide the ability to easily simulate an I/O error during testing */ + if( sqlite3FaultSim(400) ) return SQLITE_IOERR; + + PAGERTRACE(("DATABASE SYNC: File=%s zSuper=%s nSize=%d\n", + pPager->zFilename, zSuper, pPager->dbSize)); + + /* If no database changes have been made, return early. */ + if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager, 1) ){ + /* If this is an in-memory db, or no pages have been written to, or this + ** function has already been called, it is mostly a no-op. However, any + ** backup in progress needs to be restarted. */ + sqlite3BackupRestart(pPager->pBackup); + }else{ + PgHdr *pList; + if( pagerUseWal(pPager) ){ + PgHdr *pPageOne = 0; + pList = sqlite3PcacheDirtyList(pPager->pPCache); + if( pList==0 ){ + /* Must have at least one page for the WAL commit flag. + ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ + rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0); + pList = pPageOne; + pList->pDirty = 0; + } + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1); + } + sqlite3PagerUnref(pPageOne); + if( rc==SQLITE_OK ){ + sqlite3PcacheCleanAll(pPager->pPCache); + } + }else{ + /* The bBatch boolean is true if the batch-atomic-write commit method + ** should be used. No rollback journal is created if batch-atomic-write + ** is enabled. + */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + sqlite3_file *fd = pPager->fd; + int bBatch = zSuper==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) + && !pPager->noSync + && sqlite3JournalIsInMemory(pPager->jfd); +#else +# define bBatch 0 +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + /* The following block updates the change-counter. Exactly how it + ** does this depends on whether or not the atomic-update optimization + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: + ** + ** * The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** * This commit is not part of a multi-file transaction, and + ** * Exactly one page has been modified and store in the journal file. + ** + ** If the optimization was not enabled at compile time, then the + ** pager_incr_changecounter() function is called to update the change + ** counter in 'indirect-mode'. If the optimization is compiled in but + ** is not applicable to this transaction, call sqlite3JournalCreate() + ** to make sure the journal file has actually been created, then call + ** pager_incr_changecounter() to update the change-counter in indirect + ** mode. + ** + ** Otherwise, if the optimization is both enabled and applicable, + ** then call pager_incr_changecounter() to update the change-counter + ** in 'direct' mode. In this case the journal file will never be + ** created for this transaction. + */ + if( bBatch==0 ){ + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zSuper && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } + } + } +#else /* SQLITE_ENABLE_ATOMIC_WRITE */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( zSuper ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + assert( bBatch==0 ); + } +#endif + rc = pager_incr_changecounter(pPager, 0); +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Write the super-journal name into the journal file. If a + ** super-journal file name has already been written to the journal file, + ** or if zSuper is NULL (no super-journal), then this call is a no-op. + */ + rc = writeSuperJournal(pPager, zSuper); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. + */ + rc = syncJournal(pPager, 0); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( bBatch ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); + if( rc==SQLITE_OK ){ + rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } + } + + if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + goto commit_phase_one_exit; + } + bBatch = 0; + }else{ + sqlite3OsClose(pPager->jfd); + } + } +#endif /* SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + if( bBatch==0 ){ + rc = pager_write_pagelist(pPager, pList); + } + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + goto commit_phase_one_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + rc = pager_truncate(pPager, nNew); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + + /* Finally, sync the database file. */ + if( !noSync ){ + rc = sqlite3PagerSync(pPager, zSuper); + } + IOTRACE(("DBSYNC %p\n", pPager)) + } + } + +commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } + return rc; +} + + +/* +** When this function is called, the database file has been completely +** updated to reflect the changes made by the current transaction and +** synced to disk. The journal file still exists in the file-system +** though, and if a failure occurs at this point it will eventually +** be used as a hot-journal and the current transaction rolled back. +** +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used +** for hot-journal rollback. Once this is done the transaction is +** irrevocably committed. +** +** If an error occurs, an IO error code is returned and the pager +** moves into the error state. Otherwise, SQLITE_OK is returned. +*/ +int sqlite3PagerCommitPhaseTwo(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine should not be called if a prior error has occurred. + ** But if (due to a coding error elsewhere in the system) it does get + ** called, just return the same error code without doing anything. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + pPager->iDataVersion++; + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); + + /* An optimization. If the database was not actually modified during + ** this transaction, the pager is running in exclusive-mode and is + ** using persistent journals, then this function is a no-op. + ** + ** The start of the journal file currently contains a single journal + ** header with the nRec field set to 0. If such a journal is used as + ** a hot-journal during hot-journal rollback, 0 changes will be made + ** to the database file. So there is no need to zero the journal + ** header. Since the pager is in exclusive mode, there is no need + ** to drop any locks either. + */ + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode + && pPager->journalMode==PAGER_JOURNALMODE_PERSIST + ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; + return SQLITE_OK; + } + + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + rc = pager_end_transaction(pPager, pPager->setSuper, 1); + return pager_error(pPager, rc); +} + +/* +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. +** +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: +** +** 1) It rolls back the journal file, restoring all database file and +** in-memory cache pages to the state they were in when the transaction +** was opened, and +** +** 2) It finalizes the journal file, so that it is not used for hot +** rollback at any point in the future. +** +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. +** +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. +*/ +int sqlite3PagerRollback(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + + if( pagerUseWal(pPager) ){ + int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); + rc2 = pager_end_transaction(pPager, pPager->setSuper, 0); + if( rc==SQLITE_OK ) rc = rc2; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + int eState = pPager->eState; + rc = pager_end_transaction(pPager, 0, 0); + if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ + /* This can happen using journal_mode=off. Move the pager to the error + ** state to indicate that the contents of the cache may not be trusted. + ** Any active readers will get SQLITE_ABORT. + */ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + return rc; + } + }else{ + rc = pager_playback(pPager, 0); + } + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR + || rc==SQLITE_CANTOPEN + ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); +} + +/* +** Return TRUE if the database file is opened read-only. Return FALSE +** if the database is (in theory) writable. +*/ +u8 sqlite3PagerIsreadonly(Pager *pPager){ + return pPager->readOnly; +} + +#ifdef SQLITE_DEBUG +/* +** Return the sum of the reference counts for all pages held by pPager. +*/ +int sqlite3PagerRefcount(Pager *pPager){ + return sqlite3PcacheRefCount(pPager->pPCache); +} +#endif + +/* +** Return the approximate number of bytes of memory currently +** used by the pager and its associated cache. +*/ +int sqlite3PagerMemUsed(Pager *pPager){ + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); + return perPageSize*sqlite3PcachePagecount(pPager->pPCache) + + sqlite3MallocSize(pPager) + + pPager->pageSize; +} + +/* +** Return the number of references to the specified page. +*/ +int sqlite3PagerPageRefcount(DbPage *pPage){ + return sqlite3PcachePageRefcount(pPage); +} + +#ifdef SQLITE_TEST +/* +** This routine is used for testing and analysis only. +*/ +int *sqlite3PagerStats(Pager *pPager){ + static int a[11]; + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; + a[5] = pPager->errCode; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[8] = 0; /* Used to be pPager->nOvfl */ + a[9] = pPager->nRead; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; + return a; +} +#endif + +/* +** Parameter eStat must be one of SQLITE_DBSTATUS_CACHE_HIT, _MISS, _WRITE, +** or _WRITE+1. The SQLITE_DBSTATUS_CACHE_WRITE+1 case is a translation +** of SQLITE_DBSTATUS_CACHE_SPILL. The _SPILL case is not contiguous because +** it was added later. +** +** Before returning, *pnVal is incremented by the +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before +** returning. +*/ +void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ + + assert( eStat==SQLITE_DBSTATUS_CACHE_HIT + || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE + || eStat==SQLITE_DBSTATUS_CACHE_WRITE+1 + ); + + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 + && PAGER_STAT_WRITE==2 && PAGER_STAT_SPILL==3 ); + + eStat -= SQLITE_DBSTATUS_CACHE_HIT; + *pnVal += pPager->aStat[eStat]; + if( reset ){ + pPager->aStat[eStat] = 0; + } +} + +/* +** Return true if this is an in-memory or temp-file backed pager. +*/ +int sqlite3PagerIsMemdb(Pager *pPager){ + return pPager->tempFile; +} + +/* +** Check that there are at least nSavepoint savepoints open. If there are +** currently less than nSavepoints open, then open one or more savepoints +** to make up the difference. If the number of savepoints is already +** equal to nSavepoint, then this function is a no-op. +** +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** occurs while opening the sub-journal file, then an IO error code is +** returned. Otherwise, SQLITE_OK. +*/ +static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ + int rc = SQLITE_OK; /* Return code */ + int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ + + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( nSavepoint>nCurrent && pPager->useJournal ); + + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM_BKPT; + } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; + + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + aNew[ii].bTruncateOnRelease = 1; + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM_BKPT; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + } + pPager->nSavepoint = ii+1; + } + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); + return rc; +} +int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + return pagerOpenSavepoint(pPager, nSavepoint); + }else{ + return SQLITE_OK; + } +} + + +/* +** This function is called to rollback or release (commit) a savepoint. +** The savepoint to release or rollback need not be the most recently +** created savepoint. +** +** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. +** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with +** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes +** that have occurred since the specified savepoint was created. +** +** The savepoint to rollback or release is identified by parameter +** iSavepoint. A value of 0 means to operate on the outermost savepoint +** (the first created). A value of (Pager.nSavepoint-1) means operate +** on the most recently created savepoint. If iSavepoint is greater than +** (Pager.nSavepoint-1), then this function is a no-op. +** +** If a negative value is passed to this function, then the current +** transaction is rolled back. This is different to calling +** sqlite3PagerRollback() because this function does not terminate +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. +** +** In any case, all savepoints with an index greater than iSavepoint +** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), +** then savepoint iSavepoint is also destroyed. +** +** This function may return SQLITE_NOMEM if a memory allocation fails, +** or an IO error code if an IO error occurs while rolling back a +** savepoint. If no errors occur, SQLITE_OK is returned. +*/ +int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif + + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); + + if( rc==SQLITE_OK && iSavepointnSavepoint ){ + int ii; /* Iterator variable */ + int nNew; /* Number of remaining savepoints after this op. */ + + /* Figure out how many savepoints will still be active after this + ** operation. Store this value in nNew. Then free resources associated + ** with any savepoints that are destroyed by this operation. + */ + nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); + for(ii=nNew; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + pPager->nSavepoint = nNew; + + /* If this is a release of the outermost savepoint, truncate + ** the sub-journal to zero bytes in size. */ + if( op==SAVEPOINT_RELEASE ){ + PagerSavepoint *pRel = &pPager->aSavepoint[nNew]; + if( pRel->bTruncateOnRelease && isOpen(pPager->sjfd) ){ + /* Only truncate if it is an in-memory sub-journal. */ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ + i64 sz = (pPager->pageSize+4)*pRel->iSubRec; + rc = sqlite3OsTruncate(pPager->sjfd, sz); + assert( rc==SQLITE_OK ); + } + pPager->nSubRec = pRel->iSubRec; + } + } + /* Else this is a rollback operation, playback the specified savepoint. + ** If this is a temp-file, it is possible that the journal file has + ** not yet been opened. In this case there have been no changes to + ** the database file, so the playback operation can be skipped. + */ + else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ + PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; + rc = pagerPlaybackSavepoint(pPager, pSavepoint); + assert(rc!=SQLITE_DONE); + } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } +#endif + } + + return rc; +} + +/* +** Return the full pathname of the database file. +** +** Except, if the pager is in-memory only, then return an empty string if +** nullIfMemDb is true. This routine is called with nullIfMemDb==1 when +** used to report the filename to the user, for compatibility with legacy +** behavior. But when the Btree needs to know the filename for matching to +** shared cache, it uses nullIfMemDb==0 so that in-memory databases can +** participate in shared-cache. +** +** The return value to this routine is always safe to use with +** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. +*/ +const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ + static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; +} + +/* +** Return the VFS structure for the pager. +*/ +sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ + return pPager->pVfs; +} + +/* +** Return the file handle for the database file associated +** with the pager. This might return NULL if the file has +** not yet been opened. +*/ +sqlite3_file *sqlite3PagerFile(Pager *pPager){ + return pPager->fd; +} + +/* +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. +*/ +sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif +} + +/* +** Return the full pathname of the journal file. +*/ +const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Move the page pPg to location pgno in the file. +** +** There must be no references to the page previously located at +** pgno (which we call pPgOld) though that page is allowed to be +** in cache. If the page previously located at pgno is not already +** in the rollback journal, it is not put there by by this routine. +** +** References to the page pPg remain valid. Updating any +** meta-data associated with pPg (i.e. data stored in the nExtra bytes +** allocated along with the page) is the responsibility of the caller. +** +** A transaction must be active when this routine is called. It used to be +** required that a statement transaction was not active, but this restriction +** has been removed (CREATE INDEX needs to move a page when a statement +** transaction is active). +** +** If the fourth argument, isCommit, is non-zero, then this page is being +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page +** pPg refers to will not be written to again within this transaction. +** +** This function may return SQLITE_NOMEM or an IO error code if an error +** occurs. Otherwise, it returns SQLITE_OK. +*/ +int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ + PgHdr *pPgOld; /* The page being overwritten. */ + Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */ + int rc; /* Return code */ + Pgno origPgno; /* The original page number */ + + assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* In order to be able to rollback, an in-memory database must journal + ** the page we are moving from. + */ + assert( pPager->tempFile || !MEMDB ); + if( pPager->tempFile ){ + rc = sqlite3PagerWrite(pPg); + if( rc ) return rc; + } + + /* If the page being moved is dirty and has not been saved by the latest + ** savepoint, then save the current contents of the page into the + ** sub-journal now. This is required to handle the following scenario: + ** + ** BEGIN; + ** + ** SAVEPOINT one; + ** + ** ROLLBACK TO one; + ** + ** If page X were not written to the sub-journal here, it would not + ** be possible to restore its contents when the "ROLLBACK TO one" + ** statement were is processed. + ** + ** subjournalPage() may need to allocate space to store pPg->pgno into + ** one or more savepoint bitvecs. This is the reason this function + ** may return SQLITE_NOMEM. + */ + if( (pPg->flags & PGHDR_DIRTY)!=0 + && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) + ){ + return rc; + } + + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); + IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) + + /* If the journal needs to be sync()ed before page pPg->pgno can + ** be written to, store pPg->pgno in local variable needSyncPgno. + ** + ** If the isCommit flag is set, there is no need to remember that + ** the journal needs to be sync()ed before database page pPg->pgno + ** can be written to. The caller has already promised not to write to it. + */ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + needSyncPgno = pPg->pgno; + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPg->flags&PGHDR_DIRTY ); + } + + /* If the cache contains a page with page-number pgno, remove it + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained + ** for the page moved there. + */ + pPg->flags &= ~PGHDR_NEED_SYNC; + pPgOld = sqlite3PagerLookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); + if( pPgOld ){ + if( pPgOld->nRef>1 ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + if( pPager->tempFile ){ + /* Do not discard pages from an in-memory database since we might + ** need to rollback later. Just move the page out of the way. */ + sqlite3PcacheMove(pPgOld, pPager->dbSize+1); + }else{ + sqlite3PcacheDrop(pPgOld); + } + } + + origPgno = pPg->pgno; + sqlite3PcacheMove(pPg, pgno); + sqlite3PcacheMakeDirty(pPg); + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( pPager->tempFile && pPgOld ){ + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnrefNotNull(pPgOld); + } + + if( needSyncPgno ){ + /* If needSyncPgno is non-zero, then the journal file needs to be + ** sync()ed before any data is written to database file page needSyncPgno. + ** Currently, no such page exists in the page-cache and the + ** "is journaled" bitvec flag has been set. This needs to be remedied by + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC + ** flag. + ** + ** If the attempt to load the page into the page-cache fails, (due + ** to a malloc() or IO failure), clear the bit in the pInJournal[] + ** array. Otherwise, if the page is loaded and written again in + ** this transaction, it may be written to the database file before + ** it is synced into the journal file. This way, it may end up in + ** the journal file twice, but that is not a problem. + */ + PgHdr *pPgHdr; + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr, 0); + if( rc!=SQLITE_OK ){ + if( needSyncPgno<=pPager->dbOrigSize ){ + assert( pPager->pTmpSpace!=0 ); + sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); + } + return rc; + } + pPgHdr->flags |= PGHDR_NEED_SYNC; + sqlite3PcacheMakeDirty(pPgHdr); + sqlite3PagerUnrefNotNull(pPgHdr); + } + + return SQLITE_OK; +} +#endif + +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ + assert( pPg->pgno!=iNew ); + pPg->flags = flags; + sqlite3PcacheMove(pPg, iNew); +} + +/* +** Return a pointer to the data for the specified page. +*/ +void *sqlite3PagerGetData(DbPage *pPg){ + assert( pPg->nRef>0 || pPg->pPager->memDb ); + return pPg->pData; +} + +/* +** Return a pointer to the Pager.nExtra bytes of "extra" space +** allocated along with the specified page. +*/ +void *sqlite3PagerGetExtra(DbPage *pPg){ + return pPg->pExtra; +} + +/* +** Get/set the locking-mode for this pager. Parameter eMode must be one +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then +** the locking-mode is set to the value specified. +** +** The returned value is either PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE, indicating the current (possibly updated) +** locking-mode. +*/ +int sqlite3PagerLockingMode(Pager *pPager, int eMode){ + assert( eMode==PAGER_LOCKINGMODE_QUERY + || eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_QUERY<0 ); + assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ + pPager->exclusiveMode = (u8)eMode; + } + return (int)pPager->exclusiveMode; +} + +/* +** Set the journal-mode for this pager. Parameter eMode must be one of: +** +** PAGER_JOURNALMODE_DELETE +** PAGER_JOURNALMODE_TRUNCATE +** PAGER_JOURNALMODE_PERSIST +** PAGER_JOURNALMODE_OFF +** PAGER_JOURNALMODE_MEMORY +** PAGER_JOURNALMODE_WAL +** +** The journalmode is set to the value specified if the change is allowed. +** The change may be disallowed for the following reasons: +** +** * An in-memory database can only have its journal_mode set to _OFF +** or _MEMORY. +** +** * Temporary databases cannot have _WAL journalmode. +** +** The returned indicate the current (possibly updated) journal-mode. +*/ +int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ + u8 eOld = pPager->journalMode; /* Prior journalmode */ + + /* The eMode parameter is always valid */ + assert( eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL + || eMode==PAGER_JOURNALMODE_MEMORY ); + + /* This routine is only called from the OP_JournalMode opcode, and + ** the logic there will never allow a temporary file to be changed + ** to WAL mode. + */ + assert( pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL ); + + /* Do allow the journalmode of an in-memory database to be set to + ** anything other than MEMORY or OFF + */ + if( MEMDB ){ + assert( eOld==PAGER_JOURNALMODE_MEMORY || eOld==PAGER_JOURNALMODE_OFF ); + if( eMode!=PAGER_JOURNALMODE_MEMORY && eMode!=PAGER_JOURNALMODE_OFF ){ + eMode = eOld; + } + } + + if( eMode!=eOld ){ + + /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); + pPager->journalMode = (u8)eMode; + + /* When transistioning from TRUNCATE or PERSIST to any other journal + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, + ** delete the journal file. + */ + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); + assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); + assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); + assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); + + assert( isOpen(pPager->fd) || pPager->exclusiveMode ); + if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + + /* In this case we would like to delete the journal file. If it is + ** not possible, then that is not a problem. Deleting the journal file + ** here is an optimization only. + ** + ** Before deleting the journal file, obtain a RESERVED lock on the + ** database file. This ensures that the journal file is not deleted + ** while it is in use by some other client. + */ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); + } + }else if( eMode==PAGER_JOURNALMODE_OFF ){ + sqlite3OsClose(pPager->jfd); + } + } + + /* Return the new journal mode */ + return (int)pPager->journalMode; +} + +/* +** Return the current journal mode. +*/ +int sqlite3PagerGetJournalMode(Pager *pPager){ + return (int)pPager->journalMode; +} + +/* +** Return TRUE if the pager is in a state where it is OK to change the +** journalmode. Journalmode changes can only happen when the database +** is unmodified. +*/ +int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; + if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; + return 1; +} + +/* +** Get/set the size-limit used for persistent journal files. +** +** Setting the size limit to -1 means no limit is enforced. +** An attempt to set a limit smaller than -1 is a no-op. +*/ +i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ + if( iLimit>=-1 ){ + pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); + } + return pPager->journalSizeLimit; +} + +/* +** Return a pointer to the pPager->pBackup variable. The backup module +** in backup.c maintains the content of this variable. This module +** uses it opaquely as an argument to sqlite3BackupRestart() and +** sqlite3BackupUpdate() only. +*/ +sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ + return &pPager->pBackup; +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Unless this is an in-memory or temporary database, clear the pager cache. +*/ +void sqlite3PagerClearCache(Pager *pPager){ + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); +} +#endif + + +#ifndef SQLITE_OMIT_WAL +/* +** This function is called when the user invokes "PRAGMA wal_checkpoint", +** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() +** or wal_blocking_checkpoint() API functions. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +int sqlite3PagerCheckpoint( + Pager *pPager, /* Checkpoint on this pager */ + sqlite3 *db, /* Db handle used to check for interrupts */ + int eMode, /* Type of checkpoint */ + int *pnLog, /* OUT: Final number of frames in log */ + int *pnCkpt /* OUT: Final number of checkpointed frames */ +){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt + ); + } + return rc; +} + +int sqlite3PagerWalCallback(Pager *pPager){ + return sqlite3WalCallback(pPager->pWal); +} + +/* +** Return true if the underlying VFS for the given pager supports the +** primitives necessary for write-ahead logging. +*/ +int sqlite3PagerWalSupported(Pager *pPager){ + const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); +} + +/* +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. +*/ +static int pagerExclusiveLock(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + /* If the attempt to grab the exclusive lock failed, release the + ** pending lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + return rc; +} + +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); + } + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal + ); + } + pagerFixMaplimit(pPager); + + return rc; +} + + +/* +** The caller must be holding a SHARED lock on the database file to call +** this function. +** +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. +** +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. +*/ +int sqlite3PagerOpenWal( + Pager *pPager, /* Pager object */ + int *pbOpen /* OUT: Set to true if call is a no-op */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + + if( !pPager->tempFile && !pPager->pWal ){ + if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + + rc = pagerOpenWal(pPager); + if( rc==SQLITE_OK ){ + pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; + } + }else{ + *pbOpen = 1; + } + + return rc; +} + +/* +** This function is called to close the connection to the log file prior +** to switching from WAL to rollback mode. +** +** Before closing the log file, this function attempts to take an +** EXCLUSIVE lock on the database file. If this cannot be obtained, an +** error (SQLITE_BUSY) is returned and the log connection is not closed. +** If successful, the EXCLUSIVE lock is not released before returning. +*/ +int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ + int rc = SQLITE_OK; + + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + + /* If the log file is not already open, but does exist in the file-system, + ** it may need to be checkpointed before the connection can switch to + ** rollback mode. Open it now so this can happen. + */ + if( !pPager->pWal ){ + int logexists = 0; + rc = pagerLockDb(pPager, SHARED_LOCK); + if( rc==SQLITE_OK ){ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists + ); + } + if( rc==SQLITE_OK && logexists ){ + rc = pagerOpenWal(pPager); + } + } + + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on + ** the database file, the log and log-summary files will be deleted. + */ + if( rc==SQLITE_OK && pPager->pWal ){ + rc = pagerExclusiveLock(pPager); + if( rc==SQLITE_OK ){ + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); + pPager->pWal = 0; + pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + } + return rc; +} + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** If pager pPager is a wal-mode database not in exclusive locking mode, +** invoke the sqlite3WalWriteLock() function on the associated Wal object +** with the same db and bLock parameters as were passed to this function. +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ + int rc = SQLITE_OK; + if( pagerUseWal(pPager) && pPager->exclusiveMode==0 ){ + rc = sqlite3WalWriteLock(pPager->pWal, bLock); + } + return rc; +} + +/* +** Set the database handle used by the wal layer to determine if +** blocking locks are required. +*/ +void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + if( pagerUseWal(pPager) ){ + sqlite3WalDb(pPager->pWal, db); + } +} +#endif + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** If this is a WAL database, obtain a snapshot handle for the snapshot +** currently open. Otherwise, return an error. +*/ +int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_ERROR; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotGet(pPager->pWal, ppSnapshot); + } + return rc; +} + +/* +** If this is a WAL database, store a pointer to pSnapshot. Next time a +** read transaction is opened, attempt to read from the snapshot it +** identifies. If this is not a WAL database, return an error. +*/ +int sqlite3PagerSnapshotOpen( + Pager *pPager, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** is not a WAL database, return an error. +*/ +int sqlite3PagerSnapshotRecover(Pager *pPager){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotRecover(pPager->pWal); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** The caller currently has a read transaction open on the database. +** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, +** this function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotCheck(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3PagerSnapshotCheck(). +*/ +void sqlite3PagerSnapshotUnlock(Pager *pPager){ + assert( pPager->pWal ); + sqlite3WalSnapshotUnlock(pPager->pWal); +} + +#endif /* SQLITE_ENABLE_SNAPSHOT */ +#endif /* !SQLITE_OMIT_WAL */ + +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. +*/ +int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState>=PAGER_READER ); + return sqlite3WalFramesize(pPager->pWal); +} +#endif + +#endif /* SQLITE_OMIT_DISKIO */ diff --git a/third_party/sqlite3/pager.inc b/third_party/sqlite3/pager.inc new file mode 100644 index 000000000..68a66cc96 --- /dev/null +++ b/third_party/sqlite3/pager.inc @@ -0,0 +1,243 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. The page cache subsystem reads and writes a file a page +** at a time and provides a journal for rollback. +*/ +/* clang-format off */ + +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H + +/* +** Default maximum size for persistent journal files. A negative +** value means no limit. This value may be overridden using the +** sqlite3PagerJournalSizeLimit() API. See also "PRAGMA journal_size_limit". +*/ +#ifndef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + #define SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT -1 +#endif + +/* +** The type used to represent a page number. The first page in a file +** is called page 1. 0 is used to represent "not a page". +*/ +typedef u32 Pgno; + +/* +** Each open file is managed by a separate instance of the "Pager" structure. +*/ +typedef struct Pager Pager; + +/* +** Handle type for pages. +*/ +typedef struct PgHdr DbPage; + +/* +** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is +** reserved for working around a windows/posix incompatibility). It is +** used in the journal to signify that the remainder of the journal file +** is devoted to storing a super-journal name - there are no more pages to +** roll back. See comments for function writeSuperJournal() in pager.c +** for details. +*/ +#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) + +/* +** Allowed values for the flags parameter to sqlite3PagerOpen(). +** +** NOTE: These values must match the corresponding BTREE_ values in btree.h. +*/ +#define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ +#define PAGER_MEMORY 0x0002 /* In-memory database */ + +/* +** Valid values for the second argument to sqlite3PagerLockingMode(). +*/ +#define PAGER_LOCKINGMODE_QUERY -1 +#define PAGER_LOCKINGMODE_NORMAL 0 +#define PAGER_LOCKINGMODE_EXCLUSIVE 1 + +/* +** Numeric constants that encode the journalmode. +** +** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) +** are exposed in the API via the "PRAGMA journal_mode" command and +** therefore cannot be changed without a compatibility break. +*/ +#define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ +#define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ +#define PAGER_JOURNALMODE_PERSIST 1 /* Commit by zeroing journal header */ +#define PAGER_JOURNALMODE_OFF 2 /* Journal omitted. */ +#define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ +#define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ +#define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ + +/* +** Flags that make up the mask passed to sqlite3PagerGet(). +*/ +#define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ + +/* +** Flags for sqlite3PagerSetFlags() +** +** Value constraints (enforced via assert()): +** PAGER_FULLFSYNC == SQLITE_FullFSync +** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync +** PAGER_CACHE_SPILL == SQLITE_CacheSpill +*/ +#define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ +#define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ +#define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ +#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ +#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ +#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ + +/* +** The remainder of this file contains the declarations of the functions +** that make up the Pager sub-system API. See source code comments for +** a detailed description of each routine. +*/ + +/* Open and close a Pager connection. */ +int sqlite3PagerOpen( + sqlite3_vfs*, + Pager **ppPager, + const char*, + int, + int, + int, + void(*)(DbPage*) +); +int sqlite3PagerClose(Pager *pPager, sqlite3*); +int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); + +/* Functions used to configure a Pager object. */ +void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); +int sqlite3PagerSetPagesize(Pager*, u32*, int); +Pgno sqlite3PagerMaxPageCount(Pager*, Pgno); +void sqlite3PagerSetCachesize(Pager*, int); +int sqlite3PagerSetSpillsize(Pager*, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); +void sqlite3PagerShrink(Pager*); +void sqlite3PagerSetFlags(Pager*,unsigned); +int sqlite3PagerLockingMode(Pager *, int); +int sqlite3PagerSetJournalMode(Pager *, int); +int sqlite3PagerGetJournalMode(Pager*); +int sqlite3PagerOkToChangeJournalMode(Pager*); +i64 sqlite3PagerJournalSizeLimit(Pager *, i64); +sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +int sqlite3PagerFlush(Pager*); + +/* Functions used to obtain and release page references. */ +int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); +DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); +void sqlite3PagerRef(DbPage*); +void sqlite3PagerUnref(DbPage*); +void sqlite3PagerUnrefNotNull(DbPage*); +void sqlite3PagerUnrefPageOne(DbPage*); + +/* Operations on page references. */ +int sqlite3PagerWrite(DbPage*); +void sqlite3PagerDontWrite(DbPage*); +int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); +int sqlite3PagerPageRefcount(DbPage*); +void *sqlite3PagerGetData(DbPage *); +void *sqlite3PagerGetExtra(DbPage *); + +/* Functions used to manage pager transactions and savepoints. */ +void sqlite3PagerPagecount(Pager*, int*); +int sqlite3PagerBegin(Pager*, int exFlag, int); +int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int); +int sqlite3PagerExclusiveLock(Pager*); +int sqlite3PagerSync(Pager *pPager, const char *zSuper); +int sqlite3PagerCommitPhaseTwo(Pager*); +int sqlite3PagerRollback(Pager*); +int sqlite3PagerOpenSavepoint(Pager *pPager, int n); +int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); +int sqlite3PagerSharedLock(Pager *pPager); + +#ifndef SQLITE_OMIT_WAL + int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); + int sqlite3PagerWalSupported(Pager *pPager); + int sqlite3PagerWalCallback(Pager *pPager); + int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); + int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); +# ifdef SQLITE_ENABLE_SNAPSHOT + int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot); + int sqlite3PagerSnapshotOpen(Pager*, sqlite3_snapshot *pSnapshot); + int sqlite3PagerSnapshotRecover(Pager *pPager); + int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); + void sqlite3PagerSnapshotUnlock(Pager *pPager); +# endif +#endif + +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_ENABLE_SETLK_TIMEOUT) + int sqlite3PagerWalWriteLock(Pager*, int); + void sqlite3PagerWalDb(Pager*, sqlite3*); +#else +# define sqlite3PagerWalWriteLock(y,z) SQLITE_OK +# define sqlite3PagerWalDb(x,y) +#endif + +#ifdef SQLITE_DIRECT_OVERFLOW_READ + int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); +#endif + +#ifdef SQLITE_ENABLE_ZIPVFS + int sqlite3PagerWalFramesize(Pager *pPager); +#endif + +/* Functions used to query pager state and configuration. */ +u8 sqlite3PagerIsreadonly(Pager*); +u32 sqlite3PagerDataVersion(Pager*); +#ifdef SQLITE_DEBUG + int sqlite3PagerRefcount(Pager*); +#endif +int sqlite3PagerMemUsed(Pager*); +const char *sqlite3PagerFilename(const Pager*, int); +sqlite3_vfs *sqlite3PagerVfs(Pager*); +sqlite3_file *sqlite3PagerFile(Pager*); +sqlite3_file *sqlite3PagerJrnlFile(Pager*); +const char *sqlite3PagerJournalname(Pager*); +void *sqlite3PagerTempSpace(Pager*); +int sqlite3PagerIsMemdb(Pager*); +void sqlite3PagerCacheStat(Pager *, int, int, int *); +void sqlite3PagerClearCache(Pager*); +int sqlite3SectorSize(sqlite3_file *); + +/* Functions used to truncate the database file. */ +void sqlite3PagerTruncateImage(Pager*,Pgno); + +void sqlite3PagerRekey(DbPage*, Pgno, u16); + +/* Functions to support testing and debugging. */ +#if !defined(NDEBUG) || defined(SQLITE_TEST) + Pgno sqlite3PagerPagenumber(DbPage*); + int sqlite3PagerIswriteable(DbPage*); +#endif +#ifdef SQLITE_TEST + int *sqlite3PagerStats(Pager*); + void sqlite3PagerRefdump(Pager*); + void disable_simulated_io_errors(void); + void enable_simulated_io_errors(void); +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +#endif /* SQLITE_PAGER_H */ diff --git a/third_party/sqlite3/parse.c b/third_party/sqlite3/parse.c new file mode 100644 index 000000000..40981994b --- /dev/null +++ b/third_party/sqlite3/parse.c @@ -0,0 +1,5425 @@ +/* This file is automatically generated by Lemon from input grammar +** source file "third_party/sqlite3/parse.y". */ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains SQLite's SQL parser. +** +** The canonical source code to this file ("third_party/sqlite3/parse.y") is a +*Lemon grammar +** file that specifies the input grammar and actions to take while parsing. +** That input file is processed by Lemon to generate a C-language +** implementation of a parser for the given grammer. You might be reading +** this comment as part of the translated C-code. Edits should be made +** to the original parse.y sources. +*/ +/* clang-format off */ + +#include "third_party/sqlite3/sqliteInt.inc" + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) + +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + +/* +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form +** +** UPDATE ON (a,b,c) +** +** Then the "b" IdList records the list "a,b,c". +*/ +struct TrigEvent { int a; IdList * b; }; + +struct FrameBound { int eType; Expr *pExpr; }; + +/* +** Disable lookaside memory allocation for objects that might be +** shared across database connections. +*/ +static void disableLookaside(Parse *pParse){ + sqlite3 *db = pParse->db; + pParse->disableLookaside++; + DisableLookaside; +} + +#if !defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) \ + && defined(SQLITE_UDL_CAPABLE_PARSER) +/* +** Issue an error message if an ORDER BY or LIMIT clause occurs on an +** UPDATE or DELETE statement. +*/ +static void updateDeleteLimitError( + Parse *pParse, + ExprList *pOrderBy, + Expr *pLimit +){ + if( pOrderBy ){ + sqlite3ErrorMsg(pParse, "syntax error near \"ORDER BY\""); + }else{ + sqlite3ErrorMsg(pParse, "syntax error near \"LIMIT\""); + } + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); +} +#endif /* SQLITE_ENABLE_UPDATE_DELETE_LIMIT */ + + + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + assert( p!=0 ); + if( p->pPrior ){ + Select *pNext = 0, *pLoop = p; + int mxSelect, cnt = 1; + while(1){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + pNext = pLoop; + pLoop = pLoop->pPrior; + if( pLoop==0 ) break; + cnt++; + if( pLoop->pOrderBy || pLoop->pLimit ){ + sqlite3ErrorMsg(pParse,"%s clause should come after %s not before", + pLoop->pOrderBy!=0 ? "ORDER BY" : "LIMIT", + sqlite3SelectOpName(pNext->op)); + break; + } + } + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } + } + + /* Attach a With object describing the WITH clause to a Select + ** object describing the query for which the WITH clause is a prefix. + */ + static Select *attachWithToSelect(Parse *pParse, Select *pSelect, With *pWith){ + if( pSelect ){ + pSelect->pWith = pWith; + parserDoubleLinkSelect(pParse, pSelect); + }else{ + sqlite3WithDelete(pParse->db, pWith); + } + return pSelect; + } + + + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static Expr *tokenExpr(Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + /* memset(p, 0, sizeof(Expr)); */ + p->op = (u8)op; + p->affExpr = 0; + p->flags = EP_Leaf; + ExprClearVVAProperties(p); + p->iAgg = -1; + p->pLeft = p->pRight = 0; + p->x.pList = 0; + p->pAggInfo = 0; + p->y.pTab = 0; + p->op2 = 0; + p->iTable = 0; + p->iColumn = 0; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + sqlite3DequoteExpr(p); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + if( IN_RENAME_OBJECT ){ + return (Expr*)sqlite3RenameTokenMap(pParse, (void*)p, &t); + } + } + return p; + } + + + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } + +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef TK_SEMI +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_WITHOUT 25 +#define TK_COMMA 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_MATCH 46 +#define TK_LIKE_KW 47 +#define TK_BETWEEN 48 +#define TK_IN 49 +#define TK_ISNULL 50 +#define TK_NOTNULL 51 +#define TK_NE 52 +#define TK_EQ 53 +#define TK_GT 54 +#define TK_LE 55 +#define TK_LT 56 +#define TK_GE 57 +#define TK_ESCAPE 58 +#define TK_ID 59 +#define TK_COLUMNKW 60 +#define TK_DO 61 +#define TK_FOR 62 +#define TK_IGNORE 63 +#define TK_INITIALLY 64 +#define TK_INSTEAD 65 +#define TK_NO 66 +#define TK_KEY 67 +#define TK_OF 68 +#define TK_OFFSET 69 +#define TK_PRAGMA 70 +#define TK_RAISE 71 +#define TK_RECURSIVE 72 +#define TK_REPLACE 73 +#define TK_RESTRICT 74 +#define TK_ROW 75 +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_GENERATED 95 +#define TK_ALWAYS 96 +#define TK_MATERIALIZED 97 +#define TK_REINDEX 98 +#define TK_RENAME 99 +#define TK_CTIME_KW 100 +#define TK_ANY 101 +#define TK_BITAND 102 +#define TK_BITOR 103 +#define TK_LSHIFT 104 +#define TK_RSHIFT 105 +#define TK_PLUS 106 +#define TK_MINUS 107 +#define TK_STAR 108 +#define TK_SLASH 109 +#define TK_REM 110 +#define TK_CONCAT 111 +#define TK_COLLATE 112 +#define TK_BITNOT 113 +#define TK_ON 114 +#define TK_INDEXED 115 +#define TK_STRING 116 +#define TK_JOIN_KW 117 +#define TK_CONSTRAINT 118 +#define TK_DEFAULT 119 +#define TK_NULL 120 +#define TK_PRIMARY 121 +#define TK_UNIQUE 122 +#define TK_CHECK 123 +#define TK_REFERENCES 124 +#define TK_AUTOINCR 125 +#define TK_INSERT 126 +#define TK_DELETE 127 +#define TK_UPDATE 128 +#define TK_SET 129 +#define TK_DEFERRABLE 130 +#define TK_FOREIGN 131 +#define TK_DROP 132 +#define TK_UNION 133 +#define TK_ALL 134 +#define TK_EXCEPT 135 +#define TK_INTERSECT 136 +#define TK_SELECT 137 +#define TK_VALUES 138 +#define TK_DISTINCT 139 +#define TK_DOT 140 +#define TK_FROM 141 +#define TK_JOIN 142 +#define TK_USING 143 +#define TK_ORDER 144 +#define TK_GROUP 145 +#define TK_HAVING 146 +#define TK_LIMIT 147 +#define TK_WHERE 148 +#define TK_RETURNING 149 +#define TK_INTO 150 +#define TK_NOTHING 151 +#define TK_FLOAT 152 +#define TK_BLOB 153 +#define TK_INTEGER 154 +#define TK_VARIABLE 155 +#define TK_CASE 156 +#define TK_WHEN 157 +#define TK_THEN 158 +#define TK_ELSE 159 +#define TK_INDEX 160 +#define TK_ALTER 161 +#define TK_ADD 162 +#define TK_WINDOW 163 +#define TK_OVER 164 +#define TK_FILTER 165 +#define TK_COLUMN 166 +#define TK_AGG_FUNCTION 167 +#define TK_AGG_COLUMN 168 +#define TK_TRUEFALSE 169 +#define TK_ISNOT 170 +#define TK_FUNCTION 171 +#define TK_UMINUS 172 +#define TK_UPLUS 173 +#define TK_TRUTH 174 +#define TK_REGISTER 175 +#define TK_VECTOR 176 +#define TK_SELECT_COLUMN 177 +#define TK_IF_NULL_ROW 178 +#define TK_ASTERISK 179 +#define TK_SPAN 180 +#define TK_SPACE 181 +#define TK_ILLEGAL 182 +#endif +/**************** End token definitions ***************************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define YYCODETYPE unsigned short int +#define YYNOCODE 316 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 101 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + Window* yy19; + struct TrigEvent yy50; + int yy60; + struct FrameBound yy113; + Upsert* yy178; + With* yy195; + IdList* yy288; + SrcList* yy291; + Select* yy307; + ExprList* yy338; + TriggerStep* yy483; + const char* yy528; + u8 yy570; + Expr* yy602; + Cte* yy607; + struct {int value; int mask;} yy615; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL +#define sqlite3ParserARG_PDECL +#define sqlite3ParserARG_PARAM +#define sqlite3ParserARG_FETCH +#define sqlite3ParserARG_STORE +#define sqlite3ParserCTX_SDECL Parse *pParse; +#define sqlite3ParserCTX_PDECL ,Parse *pParse +#define sqlite3ParserCTX_PARAM ,pParse +#define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; +#define sqlite3ParserCTX_STORE yypParser->pParse=pParse; +#define YYFALLBACK 1 +#define YYNSTATE 570 +#define YYNRULE 398 +#define YYNRULE_WITH_ACTION 337 +#define YYNTOKEN 183 +#define YY_MAX_SHIFT 569 +#define YY_MIN_SHIFTREDUCE 825 +#define YY_MAX_SHIFTREDUCE 1222 +#define YY_ERROR_ACTION 1223 +#define YY_ACCEPT_ACTION 1224 +#define YY_NO_ACTION 1225 +#define YY_MIN_REDUCE 1226 +#define YY_MAX_REDUCE 1623 +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (2020) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 563, 1295, 563, 1274, 168, 361, 115, 112, 218, 373, + /* 10 */ 563, 1295, 374, 563, 488, 563, 115, 112, 218, 406, + /* 20 */ 1300, 1300, 41, 41, 41, 41, 514, 1504, 520, 1298, + /* 30 */ 1298, 959, 41, 41, 1257, 71, 71, 51, 51, 960, + /* 40 */ 557, 557, 557, 122, 123, 113, 1200, 1200, 1035, 1038, + /* 50 */ 1028, 1028, 120, 120, 121, 121, 121, 121, 414, 406, + /* 60 */ 273, 273, 273, 273, 115, 112, 218, 115, 112, 218, + /* 70 */ 197, 268, 545, 560, 515, 560, 1260, 563, 385, 248, + /* 80 */ 215, 521, 399, 122, 123, 113, 1200, 1200, 1035, 1038, + /* 90 */ 1028, 1028, 120, 120, 121, 121, 121, 121, 540, 13, + /* 100 */ 13, 1259, 119, 119, 119, 119, 118, 118, 117, 117, + /* 110 */ 117, 116, 441, 1176, 419, 1531, 446, 137, 512, 1539, + /* 120 */ 1545, 372, 1547, 6, 371, 1176, 1148, 1584, 1148, 406, + /* 130 */ 1545, 534, 115, 112, 218, 1267, 99, 441, 121, 121, + /* 140 */ 121, 121, 119, 119, 119, 119, 118, 118, 117, 117, + /* 150 */ 117, 116, 441, 122, 123, 113, 1200, 1200, 1035, 1038, + /* 160 */ 1028, 1028, 120, 120, 121, 121, 121, 121, 197, 1176, + /* 170 */ 1177, 1178, 241, 304, 554, 501, 498, 497, 473, 124, + /* 180 */ 394, 1176, 1177, 1178, 1176, 496, 119, 119, 119, 119, + /* 190 */ 118, 118, 117, 117, 117, 116, 441, 139, 540, 406, + /* 200 */ 121, 121, 121, 121, 114, 117, 117, 117, 116, 441, + /* 210 */ 541, 1532, 119, 119, 119, 119, 118, 118, 117, 117, + /* 220 */ 117, 116, 441, 122, 123, 113, 1200, 1200, 1035, 1038, + /* 230 */ 1028, 1028, 120, 120, 121, 121, 121, 121, 406, 320, + /* 240 */ 1176, 1177, 1178, 81, 342, 1590, 396, 80, 119, 119, + /* 250 */ 119, 119, 118, 118, 117, 117, 117, 116, 441, 1176, + /* 260 */ 211, 450, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, + /* 270 */ 1028, 120, 120, 121, 121, 121, 121, 251, 450, 449, + /* 280 */ 273, 273, 119, 119, 119, 119, 118, 118, 117, 117, + /* 290 */ 117, 116, 441, 560, 1224, 1, 1, 569, 2, 1228, + /* 300 */ 317, 1176, 319, 1561, 305, 337, 140, 340, 406, 430, + /* 310 */ 469, 1533, 1197, 1308, 348, 1176, 1177, 1178, 168, 462, + /* 320 */ 330, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 330 */ 116, 441, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, + /* 340 */ 1028, 120, 120, 121, 121, 121, 121, 273, 273, 563, + /* 350 */ 83, 450, 416, 1564, 569, 2, 1228, 1176, 1177, 1178, + /* 360 */ 560, 305, 471, 140, 944, 995, 860, 563, 467, 1197, + /* 370 */ 1308, 13, 13, 137, 229, 118, 118, 117, 117, 117, + /* 380 */ 116, 441, 96, 318, 946, 504, 424, 361, 562, 71, + /* 390 */ 71, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 400 */ 116, 441, 427, 205, 273, 273, 445, 1015, 259, 276, + /* 410 */ 356, 507, 351, 506, 246, 406, 959, 560, 328, 344, + /* 420 */ 347, 315, 860, 1006, 960, 126, 545, 1005, 313, 304, + /* 430 */ 554, 229, 538, 1539, 148, 544, 281, 6, 203, 122, + /* 440 */ 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, + /* 450 */ 121, 121, 121, 121, 563, 217, 563, 12, 406, 1005, + /* 460 */ 1005, 1007, 502, 445, 119, 119, 119, 119, 118, 118, + /* 470 */ 117, 117, 117, 116, 441, 452, 71, 71, 70, 70, + /* 480 */ 944, 137, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, + /* 490 */ 1028, 120, 120, 121, 121, 121, 121, 1530, 119, 119, + /* 500 */ 119, 119, 118, 118, 117, 117, 117, 116, 441, 403, + /* 510 */ 402, 241, 1176, 545, 501, 498, 497, 1468, 1143, 451, + /* 520 */ 267, 267, 513, 1540, 496, 142, 1176, 6, 406, 530, + /* 530 */ 194, 1143, 864, 560, 1143, 461, 182, 304, 554, 32, + /* 540 */ 379, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 550 */ 116, 441, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, + /* 560 */ 1028, 120, 120, 121, 121, 121, 121, 406, 1176, 1177, + /* 570 */ 1178, 857, 568, 1176, 1228, 925, 1176, 454, 361, 305, + /* 580 */ 189, 140, 1176, 1177, 1178, 519, 529, 404, 1308, 183, + /* 590 */ 1015, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, + /* 600 */ 120, 120, 121, 121, 121, 121, 1006, 16, 16, 370, + /* 610 */ 1005, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 620 */ 116, 441, 273, 273, 1537, 150, 1176, 98, 6, 1176, + /* 630 */ 1177, 1178, 1176, 1177, 1178, 560, 380, 406, 376, 438, + /* 640 */ 437, 1161, 1005, 1005, 1007, 1025, 1025, 1036, 1039, 229, + /* 650 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 660 */ 441, 122, 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, + /* 670 */ 120, 120, 121, 121, 121, 121, 406, 1143, 1619, 392, + /* 680 */ 1016, 445, 1176, 1177, 1178, 1207, 525, 1207, 1530, 995, + /* 690 */ 1143, 304, 554, 1143, 5, 563, 543, 3, 361, 216, + /* 700 */ 122, 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, + /* 710 */ 120, 121, 121, 121, 121, 143, 563, 13, 13, 1029, + /* 720 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 730 */ 441, 1176, 426, 563, 1176, 563, 274, 274, 13, 13, + /* 740 */ 1078, 1176, 328, 457, 316, 147, 406, 211, 361, 560, + /* 750 */ 1000, 213, 511, 293, 477, 55, 55, 71, 71, 119, + /* 760 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 441, + /* 770 */ 122, 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, + /* 780 */ 120, 121, 121, 121, 121, 406, 455, 1176, 1177, 1178, + /* 790 */ 1176, 1177, 1178, 471, 526, 149, 404, 1176, 1177, 1178, + /* 800 */ 105, 270, 103, 563, 944, 563, 116, 441, 1530, 122, + /* 810 */ 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, + /* 820 */ 121, 121, 121, 121, 945, 13, 13, 13, 13, 119, + /* 830 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 441, + /* 840 */ 191, 563, 192, 563, 416, 439, 439, 439, 1083, 1083, + /* 850 */ 485, 561, 285, 914, 914, 406, 462, 330, 1530, 830, + /* 860 */ 831, 832, 206, 71, 71, 71, 71, 286, 119, 119, + /* 870 */ 119, 119, 118, 118, 117, 117, 117, 116, 441, 122, + /* 880 */ 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, + /* 890 */ 121, 121, 121, 121, 563, 217, 563, 1122, 1617, 406, + /* 900 */ 300, 1617, 301, 416, 1278, 1473, 244, 243, 242, 1249, + /* 910 */ 412, 556, 412, 282, 842, 279, 71, 71, 71, 71, + /* 920 */ 944, 1415, 1473, 1475, 101, 113, 1200, 1200, 1035, 1038, + /* 930 */ 1028, 1028, 120, 120, 121, 121, 121, 121, 119, 119, + /* 940 */ 119, 119, 118, 118, 117, 117, 117, 116, 441, 273, + /* 950 */ 273, 1099, 563, 436, 1143, 440, 563, 1122, 1618, 357, + /* 960 */ 1558, 1618, 560, 546, 488, 197, 1100, 1143, 378, 290, + /* 970 */ 1143, 1306, 284, 460, 71, 71, 1120, 405, 13, 13, + /* 980 */ 145, 1101, 119, 119, 119, 119, 118, 118, 117, 117, + /* 990 */ 117, 116, 441, 542, 104, 1473, 509, 273, 273, 294, + /* 1000 */ 1514, 294, 900, 273, 273, 273, 273, 563, 1503, 563, + /* 1010 */ 560, 545, 901, 464, 406, 1058, 560, 852, 560, 198, + /* 1020 */ 547, 1080, 920, 404, 1400, 1080, 146, 919, 38, 56, + /* 1030 */ 56, 15, 15, 563, 406, 12, 1120, 471, 122, 123, + /* 1040 */ 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, 121, + /* 1050 */ 121, 121, 121, 1460, 406, 43, 43, 483, 122, 123, + /* 1060 */ 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, 121, + /* 1070 */ 121, 121, 121, 563, 852, 9, 471, 251, 122, 111, + /* 1080 */ 113, 1200, 1200, 1035, 1038, 1028, 1028, 120, 120, 121, + /* 1090 */ 121, 121, 121, 563, 421, 57, 57, 119, 119, 119, + /* 1100 */ 119, 118, 118, 117, 117, 117, 116, 441, 1176, 493, + /* 1110 */ 563, 289, 1197, 478, 1516, 44, 44, 119, 119, 119, + /* 1120 */ 119, 118, 118, 117, 117, 117, 116, 441, 880, 563, + /* 1130 */ 536, 563, 58, 58, 488, 1414, 245, 119, 119, 119, + /* 1140 */ 119, 118, 118, 117, 117, 117, 116, 441, 563, 535, + /* 1150 */ 291, 59, 59, 60, 60, 438, 437, 406, 1154, 505, + /* 1160 */ 304, 554, 477, 1204, 1176, 1177, 1178, 881, 1206, 1197, + /* 1170 */ 61, 61, 1246, 357, 1558, 1538, 1205, 563, 1467, 6, + /* 1180 */ 1176, 488, 123, 113, 1200, 1200, 1035, 1038, 1028, 1028, + /* 1190 */ 120, 120, 121, 121, 121, 121, 1400, 1143, 410, 62, + /* 1200 */ 62, 1207, 1099, 1207, 411, 447, 273, 273, 537, 1154, + /* 1210 */ 1143, 108, 555, 1143, 4, 391, 1220, 1100, 1512, 560, + /* 1220 */ 347, 516, 428, 548, 308, 1307, 1536, 1077, 558, 1077, + /* 1230 */ 6, 488, 1101, 1400, 488, 309, 1176, 1177, 1178, 563, + /* 1240 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 1250 */ 441, 442, 278, 551, 563, 273, 273, 273, 273, 563, + /* 1260 */ 327, 45, 45, 552, 563, 528, 422, 563, 560, 1400, + /* 1270 */ 560, 108, 555, 137, 4, 1303, 46, 46, 335, 563, + /* 1280 */ 482, 47, 47, 477, 479, 307, 49, 49, 558, 50, + /* 1290 */ 50, 563, 1015, 563, 1221, 563, 1400, 563, 106, 106, + /* 1300 */ 8, 63, 63, 423, 563, 107, 312, 442, 565, 564, + /* 1310 */ 563, 442, 1005, 64, 64, 65, 65, 14, 14, 66, + /* 1320 */ 66, 391, 1121, 552, 1312, 1180, 128, 128, 563, 304, + /* 1330 */ 554, 563, 67, 67, 563, 359, 560, 532, 563, 484, + /* 1340 */ 563, 1196, 531, 222, 1005, 1005, 1007, 1008, 27, 522, + /* 1350 */ 52, 52, 1015, 68, 68, 563, 69, 69, 106, 106, + /* 1360 */ 53, 53, 156, 156, 563, 107, 434, 442, 565, 564, + /* 1370 */ 272, 215, 1005, 425, 563, 359, 563, 157, 157, 563, + /* 1380 */ 1535, 292, 1180, 98, 6, 1344, 76, 76, 1215, 475, + /* 1390 */ 413, 169, 226, 563, 245, 563, 54, 54, 72, 72, + /* 1400 */ 1221, 129, 129, 1343, 1005, 1005, 1007, 1008, 27, 1563, + /* 1410 */ 1165, 444, 456, 433, 277, 73, 73, 130, 130, 389, + /* 1420 */ 389, 388, 262, 386, 1165, 444, 839, 1519, 277, 108, + /* 1430 */ 555, 321, 4, 389, 389, 388, 262, 386, 563, 223, + /* 1440 */ 839, 311, 468, 84, 202, 523, 558, 1492, 303, 310, + /* 1450 */ 563, 110, 404, 223, 563, 311, 206, 30, 404, 277, + /* 1460 */ 131, 131, 411, 310, 389, 389, 388, 262, 386, 442, + /* 1470 */ 920, 839, 127, 127, 563, 919, 155, 155, 1491, 225, + /* 1480 */ 563, 552, 871, 563, 223, 476, 311, 161, 31, 563, + /* 1490 */ 135, 563, 480, 225, 310, 532, 154, 154, 332, 17, + /* 1500 */ 533, 161, 136, 136, 135, 134, 134, 224, 228, 355, + /* 1510 */ 1015, 132, 132, 133, 133, 1589, 106, 106, 889, 354, + /* 1520 */ 563, 224, 563, 107, 225, 442, 565, 564, 1117, 275, + /* 1530 */ 1005, 393, 161, 518, 563, 135, 108, 555, 417, 4, + /* 1540 */ 1340, 407, 75, 75, 77, 77, 304, 554, 867, 563, + /* 1550 */ 336, 563, 224, 558, 463, 407, 74, 74, 465, 1065, + /* 1560 */ 304, 554, 1005, 1005, 1007, 1008, 27, 962, 963, 543, + /* 1570 */ 448, 42, 42, 48, 48, 326, 442, 325, 98, 997, + /* 1580 */ 470, 287, 250, 250, 448, 1009, 407, 472, 552, 339, + /* 1590 */ 250, 304, 554, 879, 878, 331, 108, 555, 98, 4, + /* 1600 */ 1277, 494, 532, 345, 247, 867, 98, 531, 341, 886, + /* 1610 */ 887, 1126, 1076, 558, 1076, 448, 1065, 1015, 1061, 953, + /* 1620 */ 343, 247, 250, 106, 106, 1291, 917, 1276, 850, 110, + /* 1630 */ 107, 144, 442, 565, 564, 918, 442, 1005, 110, 1275, + /* 1640 */ 350, 360, 1009, 1331, 1352, 299, 1399, 1577, 552, 1327, + /* 1650 */ 1552, 550, 1338, 549, 1405, 1256, 1248, 1237, 1236, 1238, + /* 1660 */ 1571, 489, 265, 200, 1324, 363, 365, 367, 11, 1005, + /* 1670 */ 1005, 1007, 1008, 27, 390, 221, 1386, 1015, 280, 1391, + /* 1680 */ 1381, 208, 323, 106, 106, 924, 1374, 453, 283, 324, + /* 1690 */ 107, 474, 442, 565, 564, 1390, 499, 1005, 212, 288, + /* 1700 */ 1274, 397, 353, 108, 555, 195, 4, 1464, 369, 1463, + /* 1710 */ 1574, 1215, 1212, 329, 553, 171, 207, 383, 1511, 196, + /* 1720 */ 558, 254, 1509, 415, 100, 555, 83, 4, 204, 1005, + /* 1730 */ 1005, 1007, 1008, 27, 219, 79, 82, 1469, 180, 166, + /* 1740 */ 173, 558, 458, 442, 175, 176, 177, 178, 35, 1387, + /* 1750 */ 492, 459, 231, 1395, 96, 552, 1393, 1392, 395, 184, + /* 1760 */ 481, 466, 36, 235, 442, 89, 398, 266, 487, 1480, + /* 1770 */ 1458, 237, 188, 338, 508, 429, 552, 490, 400, 238, + /* 1780 */ 334, 1239, 239, 1294, 1015, 1293, 1292, 1285, 91, 871, + /* 1790 */ 106, 106, 213, 431, 1588, 432, 524, 107, 517, 442, + /* 1800 */ 565, 564, 401, 1264, 1005, 1015, 1263, 1587, 352, 1262, + /* 1810 */ 1557, 106, 106, 1586, 1284, 297, 298, 358, 107, 1335, + /* 1820 */ 442, 565, 564, 95, 362, 1005, 253, 252, 435, 125, + /* 1830 */ 543, 10, 1444, 1543, 377, 1542, 1005, 1005, 1007, 1008, + /* 1840 */ 27, 302, 102, 97, 527, 1336, 260, 1317, 364, 1245, + /* 1850 */ 1334, 34, 566, 1171, 366, 381, 375, 1005, 1005, 1007, + /* 1860 */ 1008, 27, 1333, 1359, 368, 1316, 199, 382, 261, 263, + /* 1870 */ 264, 1358, 158, 1496, 141, 1497, 1495, 567, 1234, 1229, + /* 1880 */ 1494, 295, 159, 209, 210, 78, 826, 443, 201, 306, + /* 1890 */ 220, 1075, 138, 1073, 160, 314, 162, 172, 1196, 174, + /* 1900 */ 903, 227, 230, 322, 1089, 179, 163, 164, 418, 85, + /* 1910 */ 420, 181, 170, 408, 409, 86, 87, 165, 88, 1092, + /* 1920 */ 232, 233, 1088, 151, 18, 234, 1081, 250, 333, 185, + /* 1930 */ 1209, 486, 236, 186, 37, 841, 491, 354, 240, 346, + /* 1940 */ 503, 187, 90, 167, 19, 495, 20, 869, 500, 349, + /* 1950 */ 92, 882, 296, 152, 93, 510, 1127, 1159, 153, 1041, + /* 1960 */ 214, 1128, 39, 94, 269, 271, 952, 190, 947, 110, + /* 1970 */ 1149, 1145, 1153, 249, 1133, 1147, 7, 33, 21, 193, + /* 1980 */ 22, 23, 24, 25, 1152, 539, 98, 1056, 26, 1042, + /* 1990 */ 1040, 1044, 1098, 1045, 1097, 256, 255, 28, 40, 387, + /* 2000 */ 1010, 851, 109, 29, 1167, 559, 384, 257, 913, 258, + /* 2010 */ 1166, 1579, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1578, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 191, 220, 191, 222, 191, 191, 271, 272, 273, 216, + /* 10 */ 191, 230, 216, 191, 191, 191, 271, 272, 273, 19, + /* 20 */ 232, 233, 213, 214, 213, 214, 202, 292, 202, 232, + /* 30 */ 233, 31, 213, 214, 213, 213, 214, 213, 214, 39, + /* 40 */ 207, 208, 209, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 235, 19, + /* 60 */ 236, 237, 236, 237, 271, 272, 273, 271, 272, 273, + /* 70 */ 191, 210, 250, 249, 250, 249, 213, 191, 199, 253, + /* 80 */ 254, 259, 203, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 191, 213, + /* 100 */ 214, 213, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 59, 228, 301, 293, 81, 305, 306, + /* 120 */ 311, 312, 311, 310, 313, 59, 86, 212, 88, 19, + /* 130 */ 311, 312, 271, 272, 273, 220, 26, 112, 54, 55, + /* 140 */ 56, 57, 102, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, + /* 160 */ 50, 51, 52, 53, 54, 55, 56, 57, 191, 115, + /* 170 */ 116, 117, 118, 137, 138, 121, 122, 123, 191, 69, + /* 180 */ 203, 115, 116, 117, 59, 131, 102, 103, 104, 105, + /* 190 */ 106, 107, 108, 109, 110, 111, 112, 72, 191, 19, + /* 200 */ 54, 55, 56, 57, 58, 108, 109, 110, 111, 112, + /* 210 */ 303, 304, 102, 103, 104, 105, 106, 107, 108, 109, + /* 220 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, + /* 230 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 16, + /* 240 */ 115, 116, 117, 24, 16, 227, 202, 67, 102, 103, + /* 250 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 59, + /* 260 */ 26, 191, 43, 44, 45, 46, 47, 48, 49, 50, + /* 270 */ 51, 52, 53, 54, 55, 56, 57, 24, 208, 209, + /* 280 */ 236, 237, 102, 103, 104, 105, 106, 107, 108, 109, + /* 290 */ 110, 111, 112, 249, 183, 184, 185, 186, 187, 188, + /* 300 */ 77, 59, 79, 191, 193, 77, 195, 79, 19, 19, + /* 310 */ 266, 304, 59, 202, 24, 115, 116, 117, 191, 127, + /* 320 */ 128, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 330 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, + /* 340 */ 51, 52, 53, 54, 55, 56, 57, 236, 237, 191, + /* 350 */ 150, 281, 191, 185, 186, 187, 188, 115, 116, 117, + /* 360 */ 249, 193, 191, 195, 26, 73, 59, 191, 114, 116, + /* 370 */ 202, 213, 214, 81, 263, 106, 107, 108, 109, 110, + /* 380 */ 111, 112, 148, 160, 142, 95, 228, 191, 191, 213, + /* 390 */ 214, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 400 */ 111, 112, 112, 149, 236, 237, 295, 100, 118, 119, + /* 410 */ 120, 121, 122, 123, 124, 19, 31, 249, 126, 23, + /* 420 */ 130, 260, 115, 116, 39, 22, 250, 120, 191, 137, + /* 430 */ 138, 263, 305, 306, 238, 259, 265, 310, 149, 43, + /* 440 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 450 */ 54, 55, 56, 57, 191, 117, 191, 210, 19, 152, + /* 460 */ 153, 154, 23, 295, 102, 103, 104, 105, 106, 107, + /* 470 */ 108, 109, 110, 111, 112, 266, 213, 214, 213, 214, + /* 480 */ 142, 81, 43, 44, 45, 46, 47, 48, 49, 50, + /* 490 */ 51, 52, 53, 54, 55, 56, 57, 301, 102, 103, + /* 500 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 106, + /* 510 */ 107, 118, 59, 250, 121, 122, 123, 280, 76, 119, + /* 520 */ 236, 237, 259, 306, 131, 72, 59, 310, 19, 87, + /* 530 */ 283, 89, 23, 249, 92, 288, 22, 137, 138, 22, + /* 540 */ 275, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 550 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, + /* 560 */ 51, 52, 53, 54, 55, 56, 57, 19, 115, 116, + /* 570 */ 117, 23, 186, 59, 188, 108, 59, 241, 191, 193, + /* 580 */ 26, 195, 115, 116, 117, 191, 144, 251, 202, 22, + /* 590 */ 100, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 600 */ 52, 53, 54, 55, 56, 57, 116, 213, 214, 191, + /* 610 */ 120, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 620 */ 111, 112, 236, 237, 306, 238, 59, 26, 310, 115, + /* 630 */ 116, 117, 115, 116, 117, 249, 246, 19, 248, 106, + /* 640 */ 107, 23, 152, 153, 154, 46, 47, 48, 49, 263, + /* 650 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 660 */ 112, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 670 */ 52, 53, 54, 55, 56, 57, 19, 76, 298, 299, + /* 680 */ 23, 295, 115, 116, 117, 152, 191, 154, 301, 73, + /* 690 */ 89, 137, 138, 92, 22, 191, 144, 22, 191, 191, + /* 700 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 710 */ 53, 54, 55, 56, 57, 163, 191, 213, 214, 120, + /* 720 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 730 */ 112, 59, 228, 191, 59, 191, 236, 237, 213, 214, + /* 740 */ 11, 59, 126, 127, 128, 238, 19, 26, 191, 249, + /* 750 */ 23, 164, 165, 228, 191, 213, 214, 213, 214, 102, + /* 760 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 770 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 780 */ 53, 54, 55, 56, 57, 19, 241, 115, 116, 117, + /* 790 */ 115, 116, 117, 191, 250, 238, 251, 115, 116, 117, + /* 800 */ 157, 23, 159, 191, 26, 191, 111, 112, 301, 43, + /* 810 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 820 */ 54, 55, 56, 57, 142, 213, 214, 213, 214, 102, + /* 830 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 840 */ 228, 191, 228, 191, 191, 207, 208, 209, 126, 127, + /* 850 */ 128, 133, 289, 135, 136, 19, 127, 128, 301, 7, + /* 860 */ 8, 9, 141, 213, 214, 213, 214, 265, 102, 103, + /* 870 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 43, + /* 880 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 890 */ 54, 55, 56, 57, 191, 117, 191, 22, 23, 19, + /* 900 */ 250, 26, 250, 191, 223, 191, 126, 127, 128, 205, + /* 910 */ 206, 205, 206, 260, 21, 202, 213, 214, 213, 214, + /* 920 */ 142, 270, 208, 209, 158, 45, 46, 47, 48, 49, + /* 930 */ 50, 51, 52, 53, 54, 55, 56, 57, 102, 103, + /* 940 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 236, + /* 950 */ 237, 12, 191, 250, 76, 250, 191, 22, 23, 308, + /* 960 */ 309, 26, 249, 202, 191, 191, 27, 89, 191, 202, + /* 970 */ 92, 202, 260, 80, 213, 214, 101, 203, 213, 214, + /* 980 */ 22, 42, 102, 103, 104, 105, 106, 107, 108, 109, + /* 990 */ 110, 111, 112, 228, 158, 281, 108, 236, 237, 225, + /* 1000 */ 191, 227, 63, 236, 237, 236, 237, 191, 235, 191, + /* 1010 */ 249, 250, 73, 241, 19, 122, 249, 59, 249, 24, + /* 1020 */ 259, 29, 134, 251, 191, 33, 22, 139, 24, 213, + /* 1030 */ 214, 213, 214, 191, 19, 210, 101, 191, 43, 44, + /* 1040 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1050 */ 55, 56, 57, 160, 19, 213, 214, 65, 43, 44, + /* 1060 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1070 */ 55, 56, 57, 191, 116, 22, 191, 24, 43, 44, + /* 1080 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1090 */ 55, 56, 57, 191, 261, 213, 214, 102, 103, 104, + /* 1100 */ 105, 106, 107, 108, 109, 110, 111, 112, 59, 19, + /* 1110 */ 191, 265, 59, 288, 191, 213, 214, 102, 103, 104, + /* 1120 */ 105, 106, 107, 108, 109, 110, 111, 112, 35, 191, + /* 1130 */ 66, 191, 213, 214, 191, 270, 46, 102, 103, 104, + /* 1140 */ 105, 106, 107, 108, 109, 110, 111, 112, 191, 85, + /* 1150 */ 265, 213, 214, 213, 214, 106, 107, 19, 94, 66, + /* 1160 */ 137, 138, 191, 114, 115, 116, 117, 74, 119, 116, + /* 1170 */ 213, 214, 202, 308, 309, 306, 127, 191, 235, 310, + /* 1180 */ 59, 191, 44, 45, 46, 47, 48, 49, 50, 51, + /* 1190 */ 52, 53, 54, 55, 56, 57, 191, 76, 196, 213, + /* 1200 */ 214, 152, 12, 154, 114, 191, 236, 237, 87, 145, + /* 1210 */ 89, 19, 20, 92, 22, 22, 23, 27, 191, 249, + /* 1220 */ 130, 202, 129, 202, 191, 235, 306, 152, 36, 154, + /* 1230 */ 310, 191, 42, 191, 191, 191, 115, 116, 117, 191, + /* 1240 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 1250 */ 112, 59, 99, 63, 191, 236, 237, 236, 237, 191, + /* 1260 */ 289, 213, 214, 71, 191, 144, 261, 191, 249, 191, + /* 1270 */ 249, 19, 20, 81, 22, 235, 213, 214, 235, 191, + /* 1280 */ 278, 213, 214, 191, 282, 132, 213, 214, 36, 213, + /* 1290 */ 214, 191, 100, 191, 101, 191, 191, 191, 106, 107, + /* 1300 */ 48, 213, 214, 261, 191, 113, 191, 115, 116, 117, + /* 1310 */ 191, 59, 120, 213, 214, 213, 214, 213, 214, 213, + /* 1320 */ 214, 22, 23, 71, 237, 59, 213, 214, 191, 137, + /* 1330 */ 138, 191, 213, 214, 191, 191, 249, 85, 191, 261, + /* 1340 */ 191, 26, 90, 15, 152, 153, 154, 155, 156, 19, + /* 1350 */ 213, 214, 100, 213, 214, 191, 213, 214, 106, 107, + /* 1360 */ 213, 214, 213, 214, 191, 113, 261, 115, 116, 117, + /* 1370 */ 253, 254, 120, 229, 191, 191, 191, 213, 214, 191, + /* 1380 */ 306, 289, 116, 26, 310, 191, 213, 214, 60, 19, + /* 1390 */ 296, 297, 24, 191, 46, 191, 213, 214, 213, 214, + /* 1400 */ 101, 213, 214, 191, 152, 153, 154, 155, 156, 0, + /* 1410 */ 1, 2, 191, 229, 5, 213, 214, 213, 214, 10, + /* 1420 */ 11, 12, 13, 14, 1, 2, 17, 191, 5, 19, + /* 1430 */ 20, 191, 22, 10, 11, 12, 13, 14, 191, 30, + /* 1440 */ 17, 32, 241, 148, 149, 115, 36, 191, 241, 40, + /* 1450 */ 191, 26, 251, 30, 191, 32, 141, 22, 251, 5, + /* 1460 */ 213, 214, 114, 40, 10, 11, 12, 13, 14, 59, + /* 1470 */ 134, 17, 213, 214, 191, 139, 213, 214, 191, 70, + /* 1480 */ 191, 71, 125, 191, 30, 115, 32, 78, 53, 191, + /* 1490 */ 81, 191, 191, 70, 40, 85, 213, 214, 191, 22, + /* 1500 */ 90, 78, 213, 214, 81, 213, 214, 98, 140, 120, + /* 1510 */ 100, 213, 214, 213, 214, 23, 106, 107, 26, 130, + /* 1520 */ 191, 98, 191, 113, 70, 115, 116, 117, 23, 22, + /* 1530 */ 120, 26, 78, 19, 191, 81, 19, 20, 61, 22, + /* 1540 */ 191, 132, 213, 214, 213, 214, 137, 138, 59, 191, + /* 1550 */ 191, 191, 98, 36, 128, 132, 213, 214, 128, 59, + /* 1560 */ 137, 138, 152, 153, 154, 155, 156, 83, 84, 144, + /* 1570 */ 161, 213, 214, 213, 214, 23, 59, 151, 26, 23, + /* 1580 */ 23, 151, 26, 26, 161, 59, 132, 23, 71, 191, + /* 1590 */ 26, 137, 138, 119, 120, 23, 19, 20, 26, 22, + /* 1600 */ 223, 23, 85, 23, 26, 116, 26, 90, 191, 7, + /* 1610 */ 8, 97, 152, 36, 154, 161, 116, 100, 23, 23, + /* 1620 */ 191, 26, 26, 106, 107, 191, 23, 223, 23, 26, + /* 1630 */ 113, 26, 115, 116, 117, 23, 59, 120, 26, 191, + /* 1640 */ 191, 191, 116, 255, 191, 252, 191, 140, 71, 191, + /* 1650 */ 315, 233, 191, 191, 191, 191, 191, 191, 191, 191, + /* 1660 */ 191, 285, 284, 239, 252, 252, 252, 252, 240, 152, + /* 1670 */ 153, 154, 155, 156, 189, 294, 268, 100, 242, 268, + /* 1680 */ 264, 211, 290, 106, 107, 108, 264, 256, 256, 243, + /* 1690 */ 113, 290, 115, 116, 117, 268, 217, 120, 226, 243, + /* 1700 */ 222, 268, 216, 19, 20, 246, 22, 216, 256, 216, + /* 1710 */ 194, 60, 38, 242, 277, 294, 240, 242, 198, 246, + /* 1720 */ 36, 140, 198, 198, 19, 20, 150, 22, 149, 152, + /* 1730 */ 153, 154, 155, 156, 294, 291, 291, 280, 22, 43, + /* 1740 */ 231, 36, 18, 59, 234, 234, 234, 234, 267, 269, + /* 1750 */ 18, 198, 197, 231, 148, 71, 269, 269, 243, 231, + /* 1760 */ 198, 243, 267, 197, 59, 157, 243, 198, 62, 287, + /* 1770 */ 243, 197, 22, 198, 114, 64, 71, 218, 218, 197, + /* 1780 */ 286, 198, 197, 215, 100, 215, 215, 224, 22, 125, + /* 1790 */ 106, 107, 164, 24, 221, 112, 143, 113, 302, 115, + /* 1800 */ 116, 117, 218, 215, 120, 100, 217, 221, 215, 215, + /* 1810 */ 309, 106, 107, 215, 224, 279, 279, 218, 113, 258, + /* 1820 */ 115, 116, 117, 114, 257, 120, 91, 198, 82, 147, + /* 1830 */ 144, 22, 274, 314, 198, 314, 152, 153, 154, 155, + /* 1840 */ 156, 276, 157, 146, 145, 258, 25, 247, 257, 201, + /* 1850 */ 258, 26, 200, 13, 257, 244, 246, 152, 153, 154, + /* 1860 */ 155, 156, 258, 262, 257, 247, 245, 243, 192, 192, + /* 1870 */ 6, 262, 204, 210, 219, 210, 210, 190, 190, 190, + /* 1880 */ 210, 219, 204, 211, 211, 210, 4, 3, 22, 162, + /* 1890 */ 15, 23, 16, 23, 204, 138, 129, 150, 26, 141, + /* 1900 */ 20, 24, 143, 16, 1, 141, 129, 129, 61, 53, + /* 1910 */ 37, 150, 297, 300, 300, 53, 53, 129, 53, 115, + /* 1920 */ 34, 140, 1, 5, 22, 114, 68, 26, 160, 68, + /* 1930 */ 75, 41, 140, 114, 24, 20, 19, 130, 124, 23, + /* 1940 */ 96, 22, 22, 37, 22, 67, 22, 59, 67, 24, + /* 1950 */ 22, 28, 67, 23, 148, 22, 97, 23, 23, 23, + /* 1960 */ 140, 23, 22, 26, 23, 23, 115, 22, 142, 26, + /* 1970 */ 75, 88, 75, 34, 23, 86, 44, 22, 34, 26, + /* 1980 */ 34, 34, 34, 34, 93, 24, 26, 23, 34, 23, + /* 1990 */ 23, 23, 23, 11, 23, 22, 26, 22, 22, 15, + /* 2000 */ 23, 23, 22, 22, 1, 26, 23, 140, 134, 140, + /* 2010 */ 1, 140, 316, 316, 316, 316, 316, 316, 316, 140, + /* 2020 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2030 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2040 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2050 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2060 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2070 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2080 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2090 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2100 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2110 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2120 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2130 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2140 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2150 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2160 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2170 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2180 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2190 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, + /* 2200 */ 316, 316, 316, +}; +#define YY_SHIFT_COUNT (569) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (2009) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 1423, 1409, 1454, 1192, 1192, 36, 1252, 1410, 1517, 1684, + /* 10 */ 1684, 1684, 292, 0, 0, 180, 1015, 1684, 1684, 1684, + /* 20 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 30 */ 1049, 1049, 1121, 1121, 54, 400, 36, 36, 36, 36, + /* 40 */ 36, 40, 110, 219, 289, 396, 439, 509, 548, 618, + /* 50 */ 657, 727, 766, 836, 995, 1015, 1015, 1015, 1015, 1015, + /* 60 */ 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, + /* 70 */ 1015, 1015, 1015, 1035, 1015, 1138, 880, 880, 1577, 1684, + /* 80 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 90 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 100 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 110 */ 1684, 1684, 1684, 1705, 1684, 1684, 1684, 1684, 1684, 1684, + /* 120 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 146, 84, 84, + /* 130 */ 84, 84, 84, 362, 269, 125, 97, 453, 66, 66, + /* 140 */ 893, 1090, 66, 66, 533, 533, 66, 554, 554, 554, + /* 150 */ 554, 192, 587, 587, 695, 25, 2020, 2020, 290, 290, + /* 160 */ 290, 200, 514, 514, 514, 514, 939, 939, 442, 875, + /* 170 */ 935, 66, 66, 66, 66, 66, 66, 66, 66, 66, + /* 180 */ 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + /* 190 */ 66, 601, 601, 66, 729, 878, 878, 1266, 1266, 552, + /* 200 */ 1023, 2020, 2020, 2020, 2020, 2020, 2020, 2020, 307, 490, + /* 210 */ 490, 567, 393, 517, 467, 672, 242, 682, 675, 66, + /* 220 */ 66, 66, 66, 66, 66, 66, 66, 66, 66, 616, + /* 230 */ 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + /* 240 */ 66, 66, 1093, 1093, 1093, 66, 66, 66, 778, 66, + /* 250 */ 66, 66, 1053, 1064, 66, 66, 1190, 66, 66, 66, + /* 260 */ 66, 66, 66, 66, 66, 722, 992, 718, 253, 253, + /* 270 */ 253, 253, 338, 718, 718, 888, 403, 852, 1328, 254, + /* 280 */ 1295, 721, 1330, 1295, 1330, 1370, 234, 254, 254, 234, + /* 290 */ 254, 721, 1370, 1357, 1492, 1348, 385, 385, 385, 1330, + /* 300 */ 1425, 1425, 643, 1315, 1336, 1004, 1651, 1651, 1581, 1581, + /* 310 */ 1674, 1674, 1581, 1576, 1579, 1716, 1696, 1724, 1724, 1724, + /* 320 */ 1724, 1581, 1732, 1606, 1579, 1579, 1606, 1716, 1696, 1606, + /* 330 */ 1696, 1606, 1581, 1732, 1608, 1706, 1581, 1732, 1750, 1581, + /* 340 */ 1732, 1581, 1732, 1750, 1660, 1660, 1660, 1711, 1766, 1766, + /* 350 */ 1750, 1660, 1664, 1660, 1711, 1660, 1660, 1628, 1769, 1683, + /* 360 */ 1683, 1750, 1653, 1709, 1653, 1709, 1653, 1709, 1653, 1709, + /* 370 */ 1581, 1735, 1735, 1746, 1746, 1682, 1686, 1809, 1581, 1685, + /* 380 */ 1682, 1697, 1699, 1606, 1821, 1825, 1840, 1840, 1864, 1864, + /* 390 */ 1864, 2020, 2020, 2020, 2020, 2020, 2020, 2020, 2020, 2020, + /* 400 */ 2020, 2020, 2020, 2020, 2020, 2020, 599, 223, 1193, 1299, + /* 410 */ 228, 780, 958, 1505, 1153, 1435, 1368, 1426, 1430, 1552, + /* 420 */ 1477, 1556, 1557, 1564, 1572, 1578, 1580, 1489, 1474, 1602, + /* 430 */ 1389, 1514, 1500, 1595, 1596, 1484, 1603, 1075, 1460, 1605, + /* 440 */ 1612, 1526, 1507, 1882, 1884, 1866, 1727, 1875, 1876, 1868, + /* 450 */ 1870, 1757, 1747, 1767, 1872, 1872, 1877, 1758, 1880, 1759, + /* 460 */ 1887, 1903, 1764, 1777, 1872, 1778, 1847, 1873, 1872, 1761, + /* 470 */ 1856, 1862, 1863, 1865, 1788, 1804, 1886, 1781, 1921, 1918, + /* 480 */ 1902, 1811, 1768, 1858, 1901, 1861, 1855, 1890, 1792, 1819, + /* 490 */ 1910, 1915, 1917, 1807, 1814, 1919, 1878, 1920, 1922, 1916, + /* 500 */ 1924, 1881, 1888, 1925, 1844, 1923, 1928, 1885, 1906, 1930, + /* 510 */ 1806, 1933, 1934, 1935, 1936, 1937, 1938, 1940, 1859, 1820, + /* 520 */ 1941, 1942, 1851, 1939, 1945, 1826, 1943, 1944, 1946, 1947, + /* 530 */ 1948, 1883, 1895, 1889, 1932, 1897, 1891, 1949, 1951, 1955, + /* 540 */ 1961, 1953, 1960, 1954, 1964, 1943, 1966, 1967, 1968, 1969, + /* 550 */ 1970, 1971, 1973, 1982, 1975, 1976, 1977, 1978, 1980, 1981, + /* 560 */ 1979, 1874, 1867, 1869, 1871, 1879, 1983, 1984, 2003, 2009, +}; +#define YY_REDUCE_COUNT (405) +#define YY_REDUCE_MIN (-265) +#define YY_REDUCE_MAX (1690) +static const short yy_reduce_ofst[] = { + /* 0 */ 111, 168, 386, 761, -176, -174, -191, -189, -181, -178, + /* 10 */ 176, 263, 44, -207, -204, -265, -139, -114, 158, 504, + /* 20 */ 525, 544, 612, 614, 650, 652, 765, 265, 703, 705, + /* 30 */ 70, 714, -187, 127, 774, 713, 767, 769, 970, 1019, + /* 40 */ 1021, -255, -255, -255, -255, -255, -255, -255, -255, -255, + /* 50 */ -255, -255, -255, -255, -255, -255, -255, -255, -255, -255, + /* 60 */ -255, -255, -255, -255, -255, -255, -255, -255, -255, -255, + /* 70 */ -255, -255, -255, -255, -255, -255, -255, -255, 394, 542, + /* 80 */ 816, 818, 842, 882, 902, 919, 938, 940, 957, 986, + /* 90 */ 1048, 1063, 1068, 1073, 1076, 1088, 1100, 1102, 1104, 1106, + /* 100 */ 1113, 1119, 1137, 1140, 1143, 1147, 1149, 1164, 1173, 1183, + /* 110 */ 1185, 1188, 1202, 1204, 1247, 1259, 1263, 1283, 1289, 1292, + /* 120 */ 1298, 1300, 1329, 1331, 1343, 1358, 1360, -255, -255, -255, + /* 130 */ -255, -255, -255, -255, -255, 196, -255, 387, -177, 507, + /* 140 */ 1002, -219, 557, -93, -167, 638, -121, 284, 500, 284, + /* 150 */ 500, 247, 651, 865, -255, -255, -255, -255, -85, -85, + /* 160 */ -85, 237, 171, 602, 846, 885, -212, -203, 217, 380, + /* 170 */ 380, -23, 161, 653, 712, 773, 943, 990, 1040, 563, + /* 180 */ 833, 971, 1005, 1042, 1092, 1078, 1043, 1144, 1184, -186, + /* 190 */ 1105, 318, 869, 7, 825, 920, 1074, 704, 706, 390, + /* 200 */ 1087, 1094, 336, 545, 772, 1201, 1117, 1207, -179, -137, + /* 210 */ -112, -13, 18, 112, 197, 418, 495, 508, 777, 809, + /* 220 */ 923, 1014, 1027, 1033, 1044, 1115, 1194, 1212, 1221, 209, + /* 230 */ 1236, 1240, 1256, 1287, 1301, 1307, 1349, 1359, 1398, 1417, + /* 240 */ 1429, 1434, 681, 1377, 1404, 1448, 1449, 1450, 1388, 1453, + /* 250 */ 1455, 1458, 1393, 1335, 1461, 1462, 1418, 1463, 197, 1464, + /* 260 */ 1465, 1466, 1467, 1468, 1469, 1376, 1378, 1424, 1412, 1413, + /* 270 */ 1414, 1415, 1388, 1424, 1424, 1428, 1470, 1485, 1381, 1408, + /* 280 */ 1416, 1436, 1431, 1422, 1432, 1392, 1446, 1411, 1427, 1456, + /* 290 */ 1433, 1471, 1401, 1479, 1472, 1478, 1486, 1491, 1493, 1452, + /* 300 */ 1459, 1473, 1437, 1475, 1476, 1516, 1421, 1440, 1520, 1524, + /* 310 */ 1444, 1445, 1525, 1457, 1480, 1481, 1509, 1510, 1511, 1512, + /* 320 */ 1513, 1553, 1555, 1515, 1487, 1488, 1518, 1495, 1522, 1523, + /* 330 */ 1528, 1527, 1562, 1566, 1482, 1494, 1569, 1574, 1559, 1575, + /* 340 */ 1582, 1583, 1585, 1560, 1568, 1570, 1571, 1563, 1573, 1586, + /* 350 */ 1584, 1588, 1589, 1593, 1590, 1594, 1598, 1501, 1496, 1536, + /* 360 */ 1537, 1599, 1561, 1567, 1587, 1591, 1592, 1597, 1604, 1607, + /* 370 */ 1629, 1519, 1521, 1601, 1609, 1600, 1610, 1558, 1636, 1565, + /* 380 */ 1618, 1621, 1611, 1624, 1648, 1652, 1676, 1677, 1687, 1688, + /* 390 */ 1689, 1613, 1614, 1615, 1668, 1663, 1665, 1666, 1670, 1678, + /* 400 */ 1655, 1662, 1672, 1673, 1675, 1690, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 1623, 1623, 1623, 1453, 1223, 1332, 1223, 1223, 1223, 1453, + /* 10 */ 1453, 1453, 1223, 1362, 1362, 1506, 1254, 1223, 1223, 1223, + /* 20 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1452, 1223, 1223, + /* 30 */ 1223, 1223, 1541, 1541, 1223, 1223, 1223, 1223, 1223, 1223, + /* 40 */ 1223, 1223, 1371, 1223, 1378, 1223, 1223, 1223, 1223, 1223, + /* 50 */ 1454, 1455, 1223, 1223, 1223, 1505, 1507, 1470, 1385, 1384, + /* 60 */ 1383, 1382, 1488, 1349, 1376, 1369, 1373, 1448, 1449, 1447, + /* 70 */ 1451, 1455, 1454, 1223, 1372, 1419, 1433, 1418, 1223, 1223, + /* 80 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 90 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 100 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 110 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 120 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1427, 1432, 1438, + /* 130 */ 1431, 1428, 1421, 1420, 1422, 1223, 1423, 1223, 1223, 1223, + /* 140 */ 1244, 1296, 1223, 1223, 1223, 1223, 1223, 1525, 1524, 1223, + /* 150 */ 1223, 1254, 1413, 1412, 1424, 1425, 1435, 1434, 1513, 1576, + /* 160 */ 1575, 1471, 1223, 1223, 1223, 1223, 1223, 1223, 1541, 1223, + /* 170 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 180 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 190 */ 1223, 1541, 1541, 1223, 1254, 1541, 1541, 1250, 1250, 1356, + /* 200 */ 1223, 1520, 1323, 1323, 1323, 1323, 1332, 1323, 1223, 1223, + /* 210 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 220 */ 1223, 1223, 1223, 1510, 1508, 1223, 1223, 1223, 1223, 1223, + /* 230 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 240 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 250 */ 1223, 1223, 1328, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 260 */ 1223, 1223, 1223, 1223, 1570, 1223, 1483, 1310, 1328, 1328, + /* 270 */ 1328, 1328, 1330, 1311, 1309, 1322, 1255, 1230, 1615, 1388, + /* 280 */ 1377, 1329, 1351, 1377, 1351, 1612, 1375, 1388, 1388, 1375, + /* 290 */ 1388, 1329, 1612, 1271, 1592, 1266, 1362, 1362, 1362, 1351, + /* 300 */ 1356, 1356, 1450, 1329, 1322, 1223, 1615, 1615, 1337, 1337, + /* 310 */ 1614, 1614, 1337, 1471, 1599, 1397, 1299, 1305, 1305, 1305, + /* 320 */ 1305, 1337, 1241, 1375, 1599, 1599, 1375, 1397, 1299, 1375, + /* 330 */ 1299, 1375, 1337, 1241, 1487, 1609, 1337, 1241, 1461, 1337, + /* 340 */ 1241, 1337, 1241, 1461, 1297, 1297, 1297, 1286, 1223, 1223, + /* 350 */ 1461, 1297, 1271, 1297, 1286, 1297, 1297, 1559, 1223, 1465, + /* 360 */ 1465, 1461, 1355, 1350, 1355, 1350, 1355, 1350, 1355, 1350, + /* 370 */ 1337, 1551, 1551, 1365, 1365, 1370, 1356, 1456, 1337, 1223, + /* 380 */ 1370, 1368, 1366, 1375, 1247, 1289, 1573, 1573, 1569, 1569, + /* 390 */ 1569, 1620, 1620, 1520, 1585, 1254, 1254, 1254, 1254, 1585, + /* 400 */ 1273, 1273, 1255, 1255, 1254, 1585, 1223, 1223, 1223, 1223, + /* 410 */ 1223, 1223, 1580, 1223, 1515, 1472, 1341, 1223, 1223, 1223, + /* 420 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 430 */ 1223, 1526, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 440 */ 1223, 1223, 1402, 1223, 1226, 1517, 1223, 1223, 1223, 1223, + /* 450 */ 1223, 1223, 1223, 1223, 1379, 1380, 1342, 1223, 1223, 1223, + /* 460 */ 1223, 1223, 1223, 1223, 1394, 1223, 1223, 1223, 1389, 1223, + /* 470 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1611, 1223, 1223, + /* 480 */ 1223, 1223, 1223, 1223, 1486, 1485, 1223, 1223, 1339, 1223, + /* 490 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 500 */ 1223, 1223, 1269, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 510 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 520 */ 1223, 1223, 1223, 1223, 1223, 1223, 1367, 1223, 1223, 1223, + /* 530 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 540 */ 1223, 1556, 1357, 1223, 1223, 1602, 1223, 1223, 1223, 1223, + /* 550 */ 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, 1223, + /* 560 */ 1596, 1313, 1404, 1223, 1403, 1407, 1223, 1235, 1223, 1223, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* SEMI => nothing */ + 59, /* EXPLAIN => ID */ + 59, /* QUERY => ID */ + 59, /* PLAN => ID */ + 59, /* BEGIN => ID */ + 0, /* TRANSACTION => nothing */ + 59, /* DEFERRED => ID */ + 59, /* IMMEDIATE => ID */ + 59, /* EXCLUSIVE => ID */ + 0, /* COMMIT => nothing */ + 59, /* END => ID */ + 59, /* ROLLBACK => ID */ + 59, /* SAVEPOINT => ID */ + 59, /* RELEASE => ID */ + 0, /* TO => nothing */ + 0, /* TABLE => nothing */ + 0, /* CREATE => nothing */ + 59, /* IF => ID */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 59, /* TEMP => ID */ + 0, /* LP => nothing */ + 0, /* RP => nothing */ + 0, /* AS => nothing */ + 59, /* WITHOUT => ID */ + 0, /* COMMA => nothing */ + 59, /* ABORT => ID */ + 59, /* ACTION => ID */ + 59, /* AFTER => ID */ + 59, /* ANALYZE => ID */ + 59, /* ASC => ID */ + 59, /* ATTACH => ID */ + 59, /* BEFORE => ID */ + 59, /* BY => ID */ + 59, /* CASCADE => ID */ + 59, /* CAST => ID */ + 59, /* CONFLICT => ID */ + 59, /* DATABASE => ID */ + 59, /* DESC => ID */ + 59, /* DETACH => ID */ + 59, /* EACH => ID */ + 59, /* FAIL => ID */ + 0, /* OR => nothing */ + 0, /* AND => nothing */ + 0, /* IS => nothing */ + 59, /* MATCH => ID */ + 59, /* LIKE_KW => ID */ + 0, /* BETWEEN => nothing */ + 0, /* IN => nothing */ + 0, /* ISNULL => nothing */ + 0, /* NOTNULL => nothing */ + 0, /* NE => nothing */ + 0, /* EQ => nothing */ + 0, /* GT => nothing */ + 0, /* LE => nothing */ + 0, /* LT => nothing */ + 0, /* GE => nothing */ + 0, /* ESCAPE => nothing */ + 0, /* ID => nothing */ + 59, /* COLUMNKW => ID */ + 59, /* DO => ID */ + 59, /* FOR => ID */ + 59, /* IGNORE => ID */ + 59, /* INITIALLY => ID */ + 59, /* INSTEAD => ID */ + 59, /* NO => ID */ + 59, /* KEY => ID */ + 59, /* OF => ID */ + 59, /* OFFSET => ID */ + 59, /* PRAGMA => ID */ + 59, /* RAISE => ID */ + 59, /* RECURSIVE => ID */ + 59, /* REPLACE => ID */ + 59, /* RESTRICT => ID */ + 59, /* ROW => ID */ + 59, /* ROWS => ID */ + 59, /* TRIGGER => ID */ + 59, /* VACUUM => ID */ + 59, /* VIEW => ID */ + 59, /* VIRTUAL => ID */ + 59, /* WITH => ID */ + 59, /* NULLS => ID */ + 59, /* FIRST => ID */ + 59, /* LAST => ID */ + 59, /* CURRENT => ID */ + 59, /* FOLLOWING => ID */ + 59, /* PARTITION => ID */ + 59, /* PRECEDING => ID */ + 59, /* RANGE => ID */ + 59, /* UNBOUNDED => ID */ + 59, /* EXCLUDE => ID */ + 59, /* GROUPS => ID */ + 59, /* OTHERS => ID */ + 59, /* TIES => ID */ + 59, /* GENERATED => ID */ + 59, /* ALWAYS => ID */ + 59, /* MATERIALIZED => ID */ + 59, /* REINDEX => ID */ + 59, /* RENAME => ID */ + 59, /* CTIME_KW => ID */ + 0, /* ANY => nothing */ + 0, /* BITAND => nothing */ + 0, /* BITOR => nothing */ + 0, /* LSHIFT => nothing */ + 0, /* RSHIFT => nothing */ + 0, /* PLUS => nothing */ + 0, /* MINUS => nothing */ + 0, /* STAR => nothing */ + 0, /* SLASH => nothing */ + 0, /* REM => nothing */ + 0, /* CONCAT => nothing */ + 0, /* COLLATE => nothing */ + 0, /* BITNOT => nothing */ + 0, /* ON => nothing */ + 0, /* INDEXED => nothing */ + 0, /* STRING => nothing */ + 0, /* JOIN_KW => nothing */ + 0, /* CONSTRAINT => nothing */ + 0, /* DEFAULT => nothing */ + 0, /* NULL => nothing */ + 0, /* PRIMARY => nothing */ + 0, /* UNIQUE => nothing */ + 0, /* CHECK => nothing */ + 0, /* REFERENCES => nothing */ + 0, /* AUTOINCR => nothing */ + 0, /* INSERT => nothing */ + 0, /* DELETE => nothing */ + 0, /* UPDATE => nothing */ + 0, /* SET => nothing */ + 0, /* DEFERRABLE => nothing */ + 0, /* FOREIGN => nothing */ + 0, /* DROP => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* SELECT => nothing */ + 0, /* VALUES => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* DOT => nothing */ + 0, /* FROM => nothing */ + 0, /* JOIN => nothing */ + 0, /* USING => nothing */ + 0, /* ORDER => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* LIMIT => nothing */ + 0, /* WHERE => nothing */ + 0, /* RETURNING => nothing */ + 0, /* INTO => nothing */ + 0, /* NOTHING => nothing */ + 0, /* FLOAT => nothing */ + 0, /* BLOB => nothing */ + 0, /* INTEGER => nothing */ + 0, /* VARIABLE => nothing */ + 0, /* CASE => nothing */ + 0, /* WHEN => nothing */ + 0, /* THEN => nothing */ + 0, /* ELSE => nothing */ + 0, /* INDEX => nothing */ + 0, /* ALTER => nothing */ + 0, /* ADD => nothing */ + 0, /* WINDOW => nothing */ + 0, /* OVER => nothing */ + 0, /* FILTER => nothing */ + 0, /* COLUMN => nothing */ + 0, /* AGG_FUNCTION => nothing */ + 0, /* AGG_COLUMN => nothing */ + 0, /* TRUEFALSE => nothing */ + 0, /* ISNOT => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* UMINUS => nothing */ + 0, /* UPLUS => nothing */ + 0, /* TRUTH => nothing */ + 0, /* REGISTER => nothing */ + 0, /* VECTOR => nothing */ + 0, /* SELECT_COLUMN => nothing */ + 0, /* IF_NULL_ROW => nothing */ + 0, /* ASTERISK => nothing */ + 0, /* SPAN => nothing */ + 0, /* SPACE => nothing */ + 0, /* ILLEGAL => nothing */ +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ + sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +#include "libc/assert.h" +#include "libc/stdio/stdio.h" +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "SEMI", + /* 2 */ "EXPLAIN", + /* 3 */ "QUERY", + /* 4 */ "PLAN", + /* 5 */ "BEGIN", + /* 6 */ "TRANSACTION", + /* 7 */ "DEFERRED", + /* 8 */ "IMMEDIATE", + /* 9 */ "EXCLUSIVE", + /* 10 */ "COMMIT", + /* 11 */ "END", + /* 12 */ "ROLLBACK", + /* 13 */ "SAVEPOINT", + /* 14 */ "RELEASE", + /* 15 */ "TO", + /* 16 */ "TABLE", + /* 17 */ "CREATE", + /* 18 */ "IF", + /* 19 */ "NOT", + /* 20 */ "EXISTS", + /* 21 */ "TEMP", + /* 22 */ "LP", + /* 23 */ "RP", + /* 24 */ "AS", + /* 25 */ "WITHOUT", + /* 26 */ "COMMA", + /* 27 */ "ABORT", + /* 28 */ "ACTION", + /* 29 */ "AFTER", + /* 30 */ "ANALYZE", + /* 31 */ "ASC", + /* 32 */ "ATTACH", + /* 33 */ "BEFORE", + /* 34 */ "BY", + /* 35 */ "CASCADE", + /* 36 */ "CAST", + /* 37 */ "CONFLICT", + /* 38 */ "DATABASE", + /* 39 */ "DESC", + /* 40 */ "DETACH", + /* 41 */ "EACH", + /* 42 */ "FAIL", + /* 43 */ "OR", + /* 44 */ "AND", + /* 45 */ "IS", + /* 46 */ "MATCH", + /* 47 */ "LIKE_KW", + /* 48 */ "BETWEEN", + /* 49 */ "IN", + /* 50 */ "ISNULL", + /* 51 */ "NOTNULL", + /* 52 */ "NE", + /* 53 */ "EQ", + /* 54 */ "GT", + /* 55 */ "LE", + /* 56 */ "LT", + /* 57 */ "GE", + /* 58 */ "ESCAPE", + /* 59 */ "ID", + /* 60 */ "COLUMNKW", + /* 61 */ "DO", + /* 62 */ "FOR", + /* 63 */ "IGNORE", + /* 64 */ "INITIALLY", + /* 65 */ "INSTEAD", + /* 66 */ "NO", + /* 67 */ "KEY", + /* 68 */ "OF", + /* 69 */ "OFFSET", + /* 70 */ "PRAGMA", + /* 71 */ "RAISE", + /* 72 */ "RECURSIVE", + /* 73 */ "REPLACE", + /* 74 */ "RESTRICT", + /* 75 */ "ROW", + /* 76 */ "ROWS", + /* 77 */ "TRIGGER", + /* 78 */ "VACUUM", + /* 79 */ "VIEW", + /* 80 */ "VIRTUAL", + /* 81 */ "WITH", + /* 82 */ "NULLS", + /* 83 */ "FIRST", + /* 84 */ "LAST", + /* 85 */ "CURRENT", + /* 86 */ "FOLLOWING", + /* 87 */ "PARTITION", + /* 88 */ "PRECEDING", + /* 89 */ "RANGE", + /* 90 */ "UNBOUNDED", + /* 91 */ "EXCLUDE", + /* 92 */ "GROUPS", + /* 93 */ "OTHERS", + /* 94 */ "TIES", + /* 95 */ "GENERATED", + /* 96 */ "ALWAYS", + /* 97 */ "MATERIALIZED", + /* 98 */ "REINDEX", + /* 99 */ "RENAME", + /* 100 */ "CTIME_KW", + /* 101 */ "ANY", + /* 102 */ "BITAND", + /* 103 */ "BITOR", + /* 104 */ "LSHIFT", + /* 105 */ "RSHIFT", + /* 106 */ "PLUS", + /* 107 */ "MINUS", + /* 108 */ "STAR", + /* 109 */ "SLASH", + /* 110 */ "REM", + /* 111 */ "CONCAT", + /* 112 */ "COLLATE", + /* 113 */ "BITNOT", + /* 114 */ "ON", + /* 115 */ "INDEXED", + /* 116 */ "STRING", + /* 117 */ "JOIN_KW", + /* 118 */ "CONSTRAINT", + /* 119 */ "DEFAULT", + /* 120 */ "NULL", + /* 121 */ "PRIMARY", + /* 122 */ "UNIQUE", + /* 123 */ "CHECK", + /* 124 */ "REFERENCES", + /* 125 */ "AUTOINCR", + /* 126 */ "INSERT", + /* 127 */ "DELETE", + /* 128 */ "UPDATE", + /* 129 */ "SET", + /* 130 */ "DEFERRABLE", + /* 131 */ "FOREIGN", + /* 132 */ "DROP", + /* 133 */ "UNION", + /* 134 */ "ALL", + /* 135 */ "EXCEPT", + /* 136 */ "INTERSECT", + /* 137 */ "SELECT", + /* 138 */ "VALUES", + /* 139 */ "DISTINCT", + /* 140 */ "DOT", + /* 141 */ "FROM", + /* 142 */ "JOIN", + /* 143 */ "USING", + /* 144 */ "ORDER", + /* 145 */ "GROUP", + /* 146 */ "HAVING", + /* 147 */ "LIMIT", + /* 148 */ "WHERE", + /* 149 */ "RETURNING", + /* 150 */ "INTO", + /* 151 */ "NOTHING", + /* 152 */ "FLOAT", + /* 153 */ "BLOB", + /* 154 */ "INTEGER", + /* 155 */ "VARIABLE", + /* 156 */ "CASE", + /* 157 */ "WHEN", + /* 158 */ "THEN", + /* 159 */ "ELSE", + /* 160 */ "INDEX", + /* 161 */ "ALTER", + /* 162 */ "ADD", + /* 163 */ "WINDOW", + /* 164 */ "OVER", + /* 165 */ "FILTER", + /* 166 */ "COLUMN", + /* 167 */ "AGG_FUNCTION", + /* 168 */ "AGG_COLUMN", + /* 169 */ "TRUEFALSE", + /* 170 */ "ISNOT", + /* 171 */ "FUNCTION", + /* 172 */ "UMINUS", + /* 173 */ "UPLUS", + /* 174 */ "TRUTH", + /* 175 */ "REGISTER", + /* 176 */ "VECTOR", + /* 177 */ "SELECT_COLUMN", + /* 178 */ "IF_NULL_ROW", + /* 179 */ "ASTERISK", + /* 180 */ "SPAN", + /* 181 */ "SPACE", + /* 182 */ "ILLEGAL", + /* 183 */ "input", + /* 184 */ "cmdlist", + /* 185 */ "ecmd", + /* 186 */ "cmdx", + /* 187 */ "explain", + /* 188 */ "cmd", + /* 189 */ "transtype", + /* 190 */ "trans_opt", + /* 191 */ "nm", + /* 192 */ "savepoint_opt", + /* 193 */ "create_table", + /* 194 */ "create_table_args", + /* 195 */ "createkw", + /* 196 */ "temp", + /* 197 */ "ifnotexists", + /* 198 */ "dbnm", + /* 199 */ "columnlist", + /* 200 */ "conslist_opt", + /* 201 */ "table_options", + /* 202 */ "select", + /* 203 */ "columnname", + /* 204 */ "carglist", + /* 205 */ "typetoken", + /* 206 */ "typename", + /* 207 */ "signed", + /* 208 */ "plus_num", + /* 209 */ "minus_num", + /* 210 */ "scanpt", + /* 211 */ "scantok", + /* 212 */ "ccons", + /* 213 */ "term", + /* 214 */ "expr", + /* 215 */ "onconf", + /* 216 */ "sortorder", + /* 217 */ "autoinc", + /* 218 */ "eidlist_opt", + /* 219 */ "refargs", + /* 220 */ "defer_subclause", + /* 221 */ "generated", + /* 222 */ "refarg", + /* 223 */ "refact", + /* 224 */ "init_deferred_pred_opt", + /* 225 */ "conslist", + /* 226 */ "tconscomma", + /* 227 */ "tcons", + /* 228 */ "sortlist", + /* 229 */ "eidlist", + /* 230 */ "defer_subclause_opt", + /* 231 */ "orconf", + /* 232 */ "resolvetype", + /* 233 */ "raisetype", + /* 234 */ "ifexists", + /* 235 */ "fullname", + /* 236 */ "selectnowith", + /* 237 */ "oneselect", + /* 238 */ "wqlist", + /* 239 */ "multiselect_op", + /* 240 */ "distinct", + /* 241 */ "selcollist", + /* 242 */ "from", + /* 243 */ "where_opt", + /* 244 */ "groupby_opt", + /* 245 */ "having_opt", + /* 246 */ "orderby_opt", + /* 247 */ "limit_opt", + /* 248 */ "window_clause", + /* 249 */ "values", + /* 250 */ "nexprlist", + /* 251 */ "sclp", + /* 252 */ "as", + /* 253 */ "seltablist", + /* 254 */ "stl_prefix", + /* 255 */ "joinop", + /* 256 */ "indexed_opt", + /* 257 */ "on_opt", + /* 258 */ "using_opt", + /* 259 */ "exprlist", + /* 260 */ "xfullname", + /* 261 */ "idlist", + /* 262 */ "nulls", + /* 263 */ "with", + /* 264 */ "where_opt_ret", + /* 265 */ "setlist", + /* 266 */ "insert_cmd", + /* 267 */ "idlist_opt", + /* 268 */ "upsert", + /* 269 */ "returning", + /* 270 */ "filter_over", + /* 271 */ "likeop", + /* 272 */ "between_op", + /* 273 */ "in_op", + /* 274 */ "paren_exprlist", + /* 275 */ "case_operand", + /* 276 */ "case_exprlist", + /* 277 */ "case_else", + /* 278 */ "uniqueflag", + /* 279 */ "collate", + /* 280 */ "vinto", + /* 281 */ "nmnum", + /* 282 */ "trigger_decl", + /* 283 */ "trigger_cmd_list", + /* 284 */ "trigger_time", + /* 285 */ "trigger_event", + /* 286 */ "foreach_clause", + /* 287 */ "when_clause", + /* 288 */ "trigger_cmd", + /* 289 */ "trnm", + /* 290 */ "tridxby", + /* 291 */ "database_kw_opt", + /* 292 */ "key_opt", + /* 293 */ "add_column_fullname", + /* 294 */ "kwcolumn_opt", + /* 295 */ "create_vtab", + /* 296 */ "vtabarglist", + /* 297 */ "vtabarg", + /* 298 */ "vtabargtoken", + /* 299 */ "lp", + /* 300 */ "anylist", + /* 301 */ "wqitem", + /* 302 */ "wqas", + /* 303 */ "windowdefn_list", + /* 304 */ "windowdefn", + /* 305 */ "window", + /* 306 */ "frame_opt", + /* 307 */ "part_opt", + /* 308 */ "filter_clause", + /* 309 */ "over_clause", + /* 310 */ "range_or_rows", + /* 311 */ "frame_bound", + /* 312 */ "frame_bound_s", + /* 313 */ "frame_bound_e", + /* 314 */ "frame_exclude_opt", + /* 315 */ "frame_exclude", +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "explain ::= EXPLAIN", + /* 1 */ "explain ::= EXPLAIN QUERY PLAN", + /* 2 */ "cmdx ::= cmd", + /* 3 */ "cmd ::= BEGIN transtype trans_opt", + /* 4 */ "transtype ::=", + /* 5 */ "transtype ::= DEFERRED", + /* 6 */ "transtype ::= IMMEDIATE", + /* 7 */ "transtype ::= EXCLUSIVE", + /* 8 */ "cmd ::= COMMIT|END trans_opt", + /* 9 */ "cmd ::= ROLLBACK trans_opt", + /* 10 */ "cmd ::= SAVEPOINT nm", + /* 11 */ "cmd ::= RELEASE savepoint_opt nm", + /* 12 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 13 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 14 */ "createkw ::= CREATE", + /* 15 */ "ifnotexists ::=", + /* 16 */ "ifnotexists ::= IF NOT EXISTS", + /* 17 */ "temp ::= TEMP", + /* 18 */ "temp ::=", + /* 19 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 20 */ "create_table_args ::= AS select", + /* 21 */ "table_options ::=", + /* 22 */ "table_options ::= WITHOUT nm", + /* 23 */ "columnname ::= nm typetoken", + /* 24 */ "typetoken ::=", + /* 25 */ "typetoken ::= typename LP signed RP", + /* 26 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 27 */ "typename ::= typename ID|STRING", + /* 28 */ "scanpt ::=", + /* 29 */ "scantok ::=", + /* 30 */ "ccons ::= CONSTRAINT nm", + /* 31 */ "ccons ::= DEFAULT scantok term", + /* 32 */ "ccons ::= DEFAULT LP expr RP", + /* 33 */ "ccons ::= DEFAULT PLUS scantok term", + /* 34 */ "ccons ::= DEFAULT MINUS scantok term", + /* 35 */ "ccons ::= DEFAULT scantok ID|INDEXED", + /* 36 */ "ccons ::= NOT NULL onconf", + /* 37 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 38 */ "ccons ::= UNIQUE onconf", + /* 39 */ "ccons ::= CHECK LP expr RP", + /* 40 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 41 */ "ccons ::= defer_subclause", + /* 42 */ "ccons ::= COLLATE ID|STRING", + /* 43 */ "generated ::= LP expr RP", + /* 44 */ "generated ::= LP expr RP ID", + /* 45 */ "autoinc ::=", + /* 46 */ "autoinc ::= AUTOINCR", + /* 47 */ "refargs ::=", + /* 48 */ "refargs ::= refargs refarg", + /* 49 */ "refarg ::= MATCH nm", + /* 50 */ "refarg ::= ON INSERT refact", + /* 51 */ "refarg ::= ON DELETE refact", + /* 52 */ "refarg ::= ON UPDATE refact", + /* 53 */ "refact ::= SET NULL", + /* 54 */ "refact ::= SET DEFAULT", + /* 55 */ "refact ::= CASCADE", + /* 56 */ "refact ::= RESTRICT", + /* 57 */ "refact ::= NO ACTION", + /* 58 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 59 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 60 */ "init_deferred_pred_opt ::=", + /* 61 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 62 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 63 */ "conslist_opt ::=", + /* 64 */ "tconscomma ::= COMMA", + /* 65 */ "tcons ::= CONSTRAINT nm", + /* 66 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 67 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 68 */ "tcons ::= CHECK LP expr RP onconf", + /* 69 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 70 */ "defer_subclause_opt ::=", + /* 71 */ "onconf ::=", + /* 72 */ "onconf ::= ON CONFLICT resolvetype", + /* 73 */ "orconf ::=", + /* 74 */ "orconf ::= OR resolvetype", + /* 75 */ "resolvetype ::= IGNORE", + /* 76 */ "resolvetype ::= REPLACE", + /* 77 */ "cmd ::= DROP TABLE ifexists fullname", + /* 78 */ "ifexists ::= IF EXISTS", + /* 79 */ "ifexists ::=", + /* 80 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 81 */ "cmd ::= DROP VIEW ifexists fullname", + /* 82 */ "cmd ::= select", + /* 83 */ "select ::= WITH wqlist selectnowith", + /* 84 */ "select ::= WITH RECURSIVE wqlist selectnowith", + /* 85 */ "select ::= selectnowith", + /* 86 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 87 */ "multiselect_op ::= UNION", + /* 88 */ "multiselect_op ::= UNION ALL", + /* 89 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 90 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 91 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 92 */ "values ::= VALUES LP nexprlist RP", + /* 93 */ "values ::= values COMMA LP nexprlist RP", + /* 94 */ "distinct ::= DISTINCT", + /* 95 */ "distinct ::= ALL", + /* 96 */ "distinct ::=", + /* 97 */ "sclp ::=", + /* 98 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 99 */ "selcollist ::= sclp scanpt STAR", + /* 100 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 101 */ "as ::= AS nm", + /* 102 */ "as ::=", + /* 103 */ "from ::=", + /* 104 */ "from ::= FROM seltablist", + /* 105 */ "stl_prefix ::= seltablist joinop", + /* 106 */ "stl_prefix ::=", + /* 107 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 108 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 109 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 110 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 111 */ "dbnm ::=", + /* 112 */ "dbnm ::= DOT nm", + /* 113 */ "fullname ::= nm", + /* 114 */ "fullname ::= nm DOT nm", + /* 115 */ "xfullname ::= nm", + /* 116 */ "xfullname ::= nm DOT nm", + /* 117 */ "xfullname ::= nm DOT nm AS nm", + /* 118 */ "xfullname ::= nm AS nm", + /* 119 */ "joinop ::= COMMA|JOIN", + /* 120 */ "joinop ::= JOIN_KW JOIN", + /* 121 */ "joinop ::= JOIN_KW nm JOIN", + /* 122 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 123 */ "on_opt ::= ON expr", + /* 124 */ "on_opt ::=", + /* 125 */ "indexed_opt ::=", + /* 126 */ "indexed_opt ::= INDEXED BY nm", + /* 127 */ "indexed_opt ::= NOT INDEXED", + /* 128 */ "using_opt ::= USING LP idlist RP", + /* 129 */ "using_opt ::=", + /* 130 */ "orderby_opt ::=", + /* 131 */ "orderby_opt ::= ORDER BY sortlist", + /* 132 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 133 */ "sortlist ::= expr sortorder nulls", + /* 134 */ "sortorder ::= ASC", + /* 135 */ "sortorder ::= DESC", + /* 136 */ "sortorder ::=", + /* 137 */ "nulls ::= NULLS FIRST", + /* 138 */ "nulls ::= NULLS LAST", + /* 139 */ "nulls ::=", + /* 140 */ "groupby_opt ::=", + /* 141 */ "groupby_opt ::= GROUP BY nexprlist", + /* 142 */ "having_opt ::=", + /* 143 */ "having_opt ::= HAVING expr", + /* 144 */ "limit_opt ::=", + /* 145 */ "limit_opt ::= LIMIT expr", + /* 146 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 147 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 148 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 149 */ "where_opt ::=", + /* 150 */ "where_opt ::= WHERE expr", + /* 151 */ "where_opt_ret ::=", + /* 152 */ "where_opt_ret ::= WHERE expr", + /* 153 */ "where_opt_ret ::= RETURNING selcollist", + /* 154 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 155 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 156 */ "setlist ::= setlist COMMA nm EQ expr", + /* 157 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 158 */ "setlist ::= nm EQ expr", + /* 159 */ "setlist ::= LP idlist RP EQ expr", + /* 160 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 161 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 162 */ "upsert ::=", + /* 163 */ "upsert ::= RETURNING selcollist", + /* 164 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 165 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 166 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 167 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 168 */ "returning ::= RETURNING selcollist", + /* 169 */ "insert_cmd ::= INSERT orconf", + /* 170 */ "insert_cmd ::= REPLACE", + /* 171 */ "idlist_opt ::=", + /* 172 */ "idlist_opt ::= LP idlist RP", + /* 173 */ "idlist ::= idlist COMMA nm", + /* 174 */ "idlist ::= nm", + /* 175 */ "expr ::= LP expr RP", + /* 176 */ "expr ::= ID|INDEXED", + /* 177 */ "expr ::= JOIN_KW", + /* 178 */ "expr ::= nm DOT nm", + /* 179 */ "expr ::= nm DOT nm DOT nm", + /* 180 */ "term ::= NULL|FLOAT|BLOB", + /* 181 */ "term ::= STRING", + /* 182 */ "term ::= INTEGER", + /* 183 */ "expr ::= VARIABLE", + /* 184 */ "expr ::= expr COLLATE ID|STRING", + /* 185 */ "expr ::= CAST LP expr AS typetoken RP", + /* 186 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 187 */ "expr ::= ID|INDEXED LP STAR RP", + /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", + /* 189 */ "expr ::= ID|INDEXED LP STAR RP filter_over", + /* 190 */ "term ::= CTIME_KW", + /* 191 */ "expr ::= LP nexprlist COMMA expr RP", + /* 192 */ "expr ::= expr AND expr", + /* 193 */ "expr ::= expr OR expr", + /* 194 */ "expr ::= expr LT|GT|GE|LE expr", + /* 195 */ "expr ::= expr EQ|NE expr", + /* 196 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 197 */ "expr ::= expr PLUS|MINUS expr", + /* 198 */ "expr ::= expr STAR|SLASH|REM expr", + /* 199 */ "expr ::= expr CONCAT expr", + /* 200 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 201 */ "expr ::= expr likeop expr", + /* 202 */ "expr ::= expr likeop expr ESCAPE expr", + /* 203 */ "expr ::= expr ISNULL|NOTNULL", + /* 204 */ "expr ::= expr NOT NULL", + /* 205 */ "expr ::= expr IS expr", + /* 206 */ "expr ::= expr IS NOT expr", + /* 207 */ "expr ::= NOT expr", + /* 208 */ "expr ::= BITNOT expr", + /* 209 */ "expr ::= PLUS|MINUS expr", + /* 210 */ "between_op ::= BETWEEN", + /* 211 */ "between_op ::= NOT BETWEEN", + /* 212 */ "expr ::= expr between_op expr AND expr", + /* 213 */ "in_op ::= IN", + /* 214 */ "in_op ::= NOT IN", + /* 215 */ "expr ::= expr in_op LP exprlist RP", + /* 216 */ "expr ::= LP select RP", + /* 217 */ "expr ::= expr in_op LP select RP", + /* 218 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 219 */ "expr ::= EXISTS LP select RP", + /* 220 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 221 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 222 */ "case_exprlist ::= WHEN expr THEN expr", + /* 223 */ "case_else ::= ELSE expr", + /* 224 */ "case_else ::=", + /* 225 */ "case_operand ::= expr", + /* 226 */ "case_operand ::=", + /* 227 */ "exprlist ::=", + /* 228 */ "nexprlist ::= nexprlist COMMA expr", + /* 229 */ "nexprlist ::= expr", + /* 230 */ "paren_exprlist ::=", + /* 231 */ "paren_exprlist ::= LP exprlist RP", + /* 232 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 233 */ "uniqueflag ::= UNIQUE", + /* 234 */ "uniqueflag ::=", + /* 235 */ "eidlist_opt ::=", + /* 236 */ "eidlist_opt ::= LP eidlist RP", + /* 237 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 238 */ "eidlist ::= nm collate sortorder", + /* 239 */ "collate ::=", + /* 240 */ "collate ::= COLLATE ID|STRING", + /* 241 */ "cmd ::= DROP INDEX ifexists fullname", + /* 242 */ "cmd ::= VACUUM vinto", + /* 243 */ "cmd ::= VACUUM nm vinto", + /* 244 */ "vinto ::= INTO expr", + /* 245 */ "vinto ::=", + /* 246 */ "cmd ::= PRAGMA nm dbnm", + /* 247 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 248 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 249 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 250 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 251 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 252 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 253 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 254 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 255 */ "trigger_time ::= BEFORE|AFTER", + /* 256 */ "trigger_time ::= INSTEAD OF", + /* 257 */ "trigger_time ::=", + /* 258 */ "trigger_event ::= DELETE|INSERT", + /* 259 */ "trigger_event ::= UPDATE", + /* 260 */ "trigger_event ::= UPDATE OF idlist", + /* 261 */ "when_clause ::=", + /* 262 */ "when_clause ::= WHEN expr", + /* 263 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 264 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 265 */ "trnm ::= nm DOT nm", + /* 266 */ "tridxby ::= INDEXED BY nm", + /* 267 */ "tridxby ::= NOT INDEXED", + /* 268 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 269 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 270 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 271 */ "trigger_cmd ::= scanpt select scanpt", + /* 272 */ "expr ::= RAISE LP IGNORE RP", + /* 273 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 274 */ "raisetype ::= ROLLBACK", + /* 275 */ "raisetype ::= ABORT", + /* 276 */ "raisetype ::= FAIL", + /* 277 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 278 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 279 */ "cmd ::= DETACH database_kw_opt expr", + /* 280 */ "key_opt ::=", + /* 281 */ "key_opt ::= KEY expr", + /* 282 */ "cmd ::= REINDEX", + /* 283 */ "cmd ::= REINDEX nm dbnm", + /* 284 */ "cmd ::= ANALYZE", + /* 285 */ "cmd ::= ANALYZE nm dbnm", + /* 286 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 287 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 288 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 289 */ "add_column_fullname ::= fullname", + /* 290 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 291 */ "cmd ::= create_vtab", + /* 292 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 293 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 294 */ "vtabarg ::=", + /* 295 */ "vtabargtoken ::= ANY", + /* 296 */ "vtabargtoken ::= lp anylist RP", + /* 297 */ "lp ::= LP", + /* 298 */ "with ::= WITH wqlist", + /* 299 */ "with ::= WITH RECURSIVE wqlist", + /* 300 */ "wqas ::= AS", + /* 301 */ "wqas ::= AS MATERIALIZED", + /* 302 */ "wqas ::= AS NOT MATERIALIZED", + /* 303 */ "wqitem ::= nm eidlist_opt wqas LP select RP", + /* 304 */ "wqlist ::= wqitem", + /* 305 */ "wqlist ::= wqlist COMMA wqitem", + /* 306 */ "windowdefn_list ::= windowdefn", + /* 307 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 308 */ "windowdefn ::= nm AS LP window RP", + /* 309 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 310 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 311 */ "window ::= ORDER BY sortlist frame_opt", + /* 312 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 313 */ "window ::= frame_opt", + /* 314 */ "window ::= nm frame_opt", + /* 315 */ "frame_opt ::=", + /* 316 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 317 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 318 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 319 */ "frame_bound_s ::= frame_bound", + /* 320 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 321 */ "frame_bound_e ::= frame_bound", + /* 322 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 323 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 324 */ "frame_bound ::= CURRENT ROW", + /* 325 */ "frame_exclude_opt ::=", + /* 326 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 327 */ "frame_exclude ::= NO OTHERS", + /* 328 */ "frame_exclude ::= CURRENT ROW", + /* 329 */ "frame_exclude ::= GROUP|TIES", + /* 330 */ "window_clause ::= WINDOW windowdefn_list", + /* 331 */ "filter_over ::= filter_clause over_clause", + /* 332 */ "filter_over ::= over_clause", + /* 333 */ "filter_over ::= filter_clause", + /* 334 */ "over_clause ::= OVER LP window RP", + /* 335 */ "over_clause ::= OVER nm", + /* 336 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 337 */ "input ::= cmdlist", + /* 338 */ "cmdlist ::= cmdlist ecmd", + /* 339 */ "cmdlist ::= ecmd", + /* 340 */ "ecmd ::= SEMI", + /* 341 */ "ecmd ::= cmdx SEMI", + /* 342 */ "ecmd ::= explain cmdx SEMI", + /* 343 */ "trans_opt ::=", + /* 344 */ "trans_opt ::= TRANSACTION", + /* 345 */ "trans_opt ::= TRANSACTION nm", + /* 346 */ "savepoint_opt ::= SAVEPOINT", + /* 347 */ "savepoint_opt ::=", + /* 348 */ "cmd ::= create_table create_table_args", + /* 349 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 350 */ "columnlist ::= columnname carglist", + /* 351 */ "nm ::= ID|INDEXED", + /* 352 */ "nm ::= STRING", + /* 353 */ "nm ::= JOIN_KW", + /* 354 */ "typetoken ::= typename", + /* 355 */ "typename ::= ID|STRING", + /* 356 */ "signed ::= plus_num", + /* 357 */ "signed ::= minus_num", + /* 358 */ "carglist ::= carglist ccons", + /* 359 */ "carglist ::=", + /* 360 */ "ccons ::= NULL onconf", + /* 361 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 362 */ "ccons ::= AS generated", + /* 363 */ "conslist_opt ::= COMMA conslist", + /* 364 */ "conslist ::= conslist tconscomma tcons", + /* 365 */ "conslist ::= tcons", + /* 366 */ "tconscomma ::=", + /* 367 */ "defer_subclause_opt ::= defer_subclause", + /* 368 */ "resolvetype ::= raisetype", + /* 369 */ "selectnowith ::= oneselect", + /* 370 */ "oneselect ::= values", + /* 371 */ "sclp ::= selcollist COMMA", + /* 372 */ "as ::= ID|STRING", + /* 373 */ "returning ::=", + /* 374 */ "expr ::= term", + /* 375 */ "likeop ::= LIKE_KW|MATCH", + /* 376 */ "exprlist ::= nexprlist", + /* 377 */ "nmnum ::= plus_num", + /* 378 */ "nmnum ::= nm", + /* 379 */ "nmnum ::= ON", + /* 380 */ "nmnum ::= DELETE", + /* 381 */ "nmnum ::= DEFAULT", + /* 382 */ "plus_num ::= INTEGER|FLOAT", + /* 383 */ "foreach_clause ::=", + /* 384 */ "foreach_clause ::= FOR EACH ROW", + /* 385 */ "trnm ::= nm", + /* 386 */ "tridxby ::=", + /* 387 */ "database_kw_opt ::= DATABASE", + /* 388 */ "database_kw_opt ::=", + /* 389 */ "kwcolumn_opt ::=", + /* 390 */ "kwcolumn_opt ::= COLUMNKW", + /* 391 */ "vtabarglist ::= vtabarg", + /* 392 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 393 */ "vtabarg ::= vtabarg vtabargtoken", + /* 394 */ "anylist ::=", + /* 395 */ "anylist ::= anylist LP anylist RP", + /* 396 */ "anylist ::= anylist ANY", + /* 397 */ "with ::=", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + sqlite3ParserCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Parser and sqlite3ParserFree. +*/ +void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) sqlite3ParserCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + sqlite3ParserCTX_STORE + sqlite3ParserInit(yypParser sqlite3ParserCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + case 202: /* select */ + case 236: /* selectnowith */ + case 237: /* oneselect */ + case 249: /* values */ +{ +sqlite3SelectDelete(pParse->db, (yypminor->yy307)); +} + break; + case 213: /* term */ + case 214: /* expr */ + case 243: /* where_opt */ + case 245: /* having_opt */ + case 257: /* on_opt */ + case 264: /* where_opt_ret */ + case 275: /* case_operand */ + case 277: /* case_else */ + case 280: /* vinto */ + case 287: /* when_clause */ + case 292: /* key_opt */ + case 308: /* filter_clause */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy602)); +} + break; + case 218: /* eidlist_opt */ + case 228: /* sortlist */ + case 229: /* eidlist */ + case 241: /* selcollist */ + case 244: /* groupby_opt */ + case 246: /* orderby_opt */ + case 250: /* nexprlist */ + case 251: /* sclp */ + case 259: /* exprlist */ + case 265: /* setlist */ + case 274: /* paren_exprlist */ + case 276: /* case_exprlist */ + case 307: /* part_opt */ +{ +sqlite3ExprListDelete(pParse->db, (yypminor->yy338)); +} + break; + case 235: /* fullname */ + case 242: /* from */ + case 253: /* seltablist */ + case 254: /* stl_prefix */ + case 260: /* xfullname */ +{ +sqlite3SrcListDelete(pParse->db, (yypminor->yy291)); +} + break; + case 238: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy195)); +} + break; + case 248: /* window_clause */ + case 303: /* windowdefn_list */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ +sqlite3WindowListDelete(pParse->db, (yypminor->yy19)); +} +#endif + break; + case 258: /* using_opt */ + case 261: /* idlist */ + case 267: /* idlist_opt */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy288)); +} + break; + case 270: /* filter_over */ + case 304: /* windowdefn */ + case 305: /* window */ + case 306: /* frame_opt */ + case 309: /* over_clause */ +{ +sqlite3WindowDelete(pParse->db, (yypminor->yy19)); +} + break; + case 283: /* trigger_cmd_list */ + case 288: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy483)); +} + break; + case 285: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy50).b); +} + break; + case 311: /* frame_bound */ + case 312: /* frame_bound_s */ + case 313: /* frame_bound_e */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy113).pExpr); +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ +#ifdef SQLITE_OMIT_WINDOWFUNC + SqliteOmitWindowfunc: + sqlite3ErrorMsg(pParse, "window functions not included in production builds"); + break; +#endif + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +void sqlite3ParserFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void sqlite3ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + sqlite3ParserFinalize(p); + (*freeProc)(p); +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int sqlite3ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ + + sqlite3ErrorMsg(pParse, "parser stack overflow"); +/******** End %stack_overflow code ********************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3ParserCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 187, /* (0) explain ::= EXPLAIN */ + 187, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 186, /* (2) cmdx ::= cmd */ + 188, /* (3) cmd ::= BEGIN transtype trans_opt */ + 189, /* (4) transtype ::= */ + 189, /* (5) transtype ::= DEFERRED */ + 189, /* (6) transtype ::= IMMEDIATE */ + 189, /* (7) transtype ::= EXCLUSIVE */ + 188, /* (8) cmd ::= COMMIT|END trans_opt */ + 188, /* (9) cmd ::= ROLLBACK trans_opt */ + 188, /* (10) cmd ::= SAVEPOINT nm */ + 188, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 188, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 193, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 195, /* (14) createkw ::= CREATE */ + 197, /* (15) ifnotexists ::= */ + 197, /* (16) ifnotexists ::= IF NOT EXISTS */ + 196, /* (17) temp ::= TEMP */ + 196, /* (18) temp ::= */ + 194, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ + 194, /* (20) create_table_args ::= AS select */ + 201, /* (21) table_options ::= */ + 201, /* (22) table_options ::= WITHOUT nm */ + 203, /* (23) columnname ::= nm typetoken */ + 205, /* (24) typetoken ::= */ + 205, /* (25) typetoken ::= typename LP signed RP */ + 205, /* (26) typetoken ::= typename LP signed COMMA signed RP */ + 206, /* (27) typename ::= typename ID|STRING */ + 210, /* (28) scanpt ::= */ + 211, /* (29) scantok ::= */ + 212, /* (30) ccons ::= CONSTRAINT nm */ + 212, /* (31) ccons ::= DEFAULT scantok term */ + 212, /* (32) ccons ::= DEFAULT LP expr RP */ + 212, /* (33) ccons ::= DEFAULT PLUS scantok term */ + 212, /* (34) ccons ::= DEFAULT MINUS scantok term */ + 212, /* (35) ccons ::= DEFAULT scantok ID|INDEXED */ + 212, /* (36) ccons ::= NOT NULL onconf */ + 212, /* (37) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 212, /* (38) ccons ::= UNIQUE onconf */ + 212, /* (39) ccons ::= CHECK LP expr RP */ + 212, /* (40) ccons ::= REFERENCES nm eidlist_opt refargs */ + 212, /* (41) ccons ::= defer_subclause */ + 212, /* (42) ccons ::= COLLATE ID|STRING */ + 221, /* (43) generated ::= LP expr RP */ + 221, /* (44) generated ::= LP expr RP ID */ + 217, /* (45) autoinc ::= */ + 217, /* (46) autoinc ::= AUTOINCR */ + 219, /* (47) refargs ::= */ + 219, /* (48) refargs ::= refargs refarg */ + 222, /* (49) refarg ::= MATCH nm */ + 222, /* (50) refarg ::= ON INSERT refact */ + 222, /* (51) refarg ::= ON DELETE refact */ + 222, /* (52) refarg ::= ON UPDATE refact */ + 223, /* (53) refact ::= SET NULL */ + 223, /* (54) refact ::= SET DEFAULT */ + 223, /* (55) refact ::= CASCADE */ + 223, /* (56) refact ::= RESTRICT */ + 223, /* (57) refact ::= NO ACTION */ + 220, /* (58) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 220, /* (59) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 224, /* (60) init_deferred_pred_opt ::= */ + 224, /* (61) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 224, /* (62) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 200, /* (63) conslist_opt ::= */ + 226, /* (64) tconscomma ::= COMMA */ + 227, /* (65) tcons ::= CONSTRAINT nm */ + 227, /* (66) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 227, /* (67) tcons ::= UNIQUE LP sortlist RP onconf */ + 227, /* (68) tcons ::= CHECK LP expr RP onconf */ + 227, /* (69) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 230, /* (70) defer_subclause_opt ::= */ + 215, /* (71) onconf ::= */ + 215, /* (72) onconf ::= ON CONFLICT resolvetype */ + 231, /* (73) orconf ::= */ + 231, /* (74) orconf ::= OR resolvetype */ + 232, /* (75) resolvetype ::= IGNORE */ + 232, /* (76) resolvetype ::= REPLACE */ + 188, /* (77) cmd ::= DROP TABLE ifexists fullname */ + 234, /* (78) ifexists ::= IF EXISTS */ + 234, /* (79) ifexists ::= */ + 188, /* (80) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 188, /* (81) cmd ::= DROP VIEW ifexists fullname */ + 188, /* (82) cmd ::= select */ + 202, /* (83) select ::= WITH wqlist selectnowith */ + 202, /* (84) select ::= WITH RECURSIVE wqlist selectnowith */ + 202, /* (85) select ::= selectnowith */ + 236, /* (86) selectnowith ::= selectnowith multiselect_op oneselect */ + 239, /* (87) multiselect_op ::= UNION */ + 239, /* (88) multiselect_op ::= UNION ALL */ + 239, /* (89) multiselect_op ::= EXCEPT|INTERSECT */ + 237, /* (90) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 237, /* (91) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 249, /* (92) values ::= VALUES LP nexprlist RP */ + 249, /* (93) values ::= values COMMA LP nexprlist RP */ + 240, /* (94) distinct ::= DISTINCT */ + 240, /* (95) distinct ::= ALL */ + 240, /* (96) distinct ::= */ + 251, /* (97) sclp ::= */ + 241, /* (98) selcollist ::= sclp scanpt expr scanpt as */ + 241, /* (99) selcollist ::= sclp scanpt STAR */ + 241, /* (100) selcollist ::= sclp scanpt nm DOT STAR */ + 252, /* (101) as ::= AS nm */ + 252, /* (102) as ::= */ + 242, /* (103) from ::= */ + 242, /* (104) from ::= FROM seltablist */ + 254, /* (105) stl_prefix ::= seltablist joinop */ + 254, /* (106) stl_prefix ::= */ + 253, /* (107) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + 253, /* (108) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + 253, /* (109) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + 253, /* (110) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 198, /* (111) dbnm ::= */ + 198, /* (112) dbnm ::= DOT nm */ + 235, /* (113) fullname ::= nm */ + 235, /* (114) fullname ::= nm DOT nm */ + 260, /* (115) xfullname ::= nm */ + 260, /* (116) xfullname ::= nm DOT nm */ + 260, /* (117) xfullname ::= nm DOT nm AS nm */ + 260, /* (118) xfullname ::= nm AS nm */ + 255, /* (119) joinop ::= COMMA|JOIN */ + 255, /* (120) joinop ::= JOIN_KW JOIN */ + 255, /* (121) joinop ::= JOIN_KW nm JOIN */ + 255, /* (122) joinop ::= JOIN_KW nm nm JOIN */ + 257, /* (123) on_opt ::= ON expr */ + 257, /* (124) on_opt ::= */ + 256, /* (125) indexed_opt ::= */ + 256, /* (126) indexed_opt ::= INDEXED BY nm */ + 256, /* (127) indexed_opt ::= NOT INDEXED */ + 258, /* (128) using_opt ::= USING LP idlist RP */ + 258, /* (129) using_opt ::= */ + 246, /* (130) orderby_opt ::= */ + 246, /* (131) orderby_opt ::= ORDER BY sortlist */ + 228, /* (132) sortlist ::= sortlist COMMA expr sortorder nulls */ + 228, /* (133) sortlist ::= expr sortorder nulls */ + 216, /* (134) sortorder ::= ASC */ + 216, /* (135) sortorder ::= DESC */ + 216, /* (136) sortorder ::= */ + 262, /* (137) nulls ::= NULLS FIRST */ + 262, /* (138) nulls ::= NULLS LAST */ + 262, /* (139) nulls ::= */ + 244, /* (140) groupby_opt ::= */ + 244, /* (141) groupby_opt ::= GROUP BY nexprlist */ + 245, /* (142) having_opt ::= */ + 245, /* (143) having_opt ::= HAVING expr */ + 247, /* (144) limit_opt ::= */ + 247, /* (145) limit_opt ::= LIMIT expr */ + 247, /* (146) limit_opt ::= LIMIT expr OFFSET expr */ + 247, /* (147) limit_opt ::= LIMIT expr COMMA expr */ + 188, /* (148) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 243, /* (149) where_opt ::= */ + 243, /* (150) where_opt ::= WHERE expr */ + 264, /* (151) where_opt_ret ::= */ + 264, /* (152) where_opt_ret ::= WHERE expr */ + 264, /* (153) where_opt_ret ::= RETURNING selcollist */ + 264, /* (154) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 188, /* (155) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 265, /* (156) setlist ::= setlist COMMA nm EQ expr */ + 265, /* (157) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 265, /* (158) setlist ::= nm EQ expr */ + 265, /* (159) setlist ::= LP idlist RP EQ expr */ + 188, /* (160) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 188, /* (161) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 268, /* (162) upsert ::= */ + 268, /* (163) upsert ::= RETURNING selcollist */ + 268, /* (164) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 268, /* (165) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 268, /* (166) upsert ::= ON CONFLICT DO NOTHING returning */ + 268, /* (167) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 269, /* (168) returning ::= RETURNING selcollist */ + 266, /* (169) insert_cmd ::= INSERT orconf */ + 266, /* (170) insert_cmd ::= REPLACE */ + 267, /* (171) idlist_opt ::= */ + 267, /* (172) idlist_opt ::= LP idlist RP */ + 261, /* (173) idlist ::= idlist COMMA nm */ + 261, /* (174) idlist ::= nm */ + 214, /* (175) expr ::= LP expr RP */ + 214, /* (176) expr ::= ID|INDEXED */ + 214, /* (177) expr ::= JOIN_KW */ + 214, /* (178) expr ::= nm DOT nm */ + 214, /* (179) expr ::= nm DOT nm DOT nm */ + 213, /* (180) term ::= NULL|FLOAT|BLOB */ + 213, /* (181) term ::= STRING */ + 213, /* (182) term ::= INTEGER */ + 214, /* (183) expr ::= VARIABLE */ + 214, /* (184) expr ::= expr COLLATE ID|STRING */ + 214, /* (185) expr ::= CAST LP expr AS typetoken RP */ + 214, /* (186) expr ::= ID|INDEXED LP distinct exprlist RP */ + 214, /* (187) expr ::= ID|INDEXED LP STAR RP */ + 214, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + 214, /* (189) expr ::= ID|INDEXED LP STAR RP filter_over */ + 213, /* (190) term ::= CTIME_KW */ + 214, /* (191) expr ::= LP nexprlist COMMA expr RP */ + 214, /* (192) expr ::= expr AND expr */ + 214, /* (193) expr ::= expr OR expr */ + 214, /* (194) expr ::= expr LT|GT|GE|LE expr */ + 214, /* (195) expr ::= expr EQ|NE expr */ + 214, /* (196) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 214, /* (197) expr ::= expr PLUS|MINUS expr */ + 214, /* (198) expr ::= expr STAR|SLASH|REM expr */ + 214, /* (199) expr ::= expr CONCAT expr */ + 271, /* (200) likeop ::= NOT LIKE_KW|MATCH */ + 214, /* (201) expr ::= expr likeop expr */ + 214, /* (202) expr ::= expr likeop expr ESCAPE expr */ + 214, /* (203) expr ::= expr ISNULL|NOTNULL */ + 214, /* (204) expr ::= expr NOT NULL */ + 214, /* (205) expr ::= expr IS expr */ + 214, /* (206) expr ::= expr IS NOT expr */ + 214, /* (207) expr ::= NOT expr */ + 214, /* (208) expr ::= BITNOT expr */ + 214, /* (209) expr ::= PLUS|MINUS expr */ + 272, /* (210) between_op ::= BETWEEN */ + 272, /* (211) between_op ::= NOT BETWEEN */ + 214, /* (212) expr ::= expr between_op expr AND expr */ + 273, /* (213) in_op ::= IN */ + 273, /* (214) in_op ::= NOT IN */ + 214, /* (215) expr ::= expr in_op LP exprlist RP */ + 214, /* (216) expr ::= LP select RP */ + 214, /* (217) expr ::= expr in_op LP select RP */ + 214, /* (218) expr ::= expr in_op nm dbnm paren_exprlist */ + 214, /* (219) expr ::= EXISTS LP select RP */ + 214, /* (220) expr ::= CASE case_operand case_exprlist case_else END */ + 276, /* (221) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 276, /* (222) case_exprlist ::= WHEN expr THEN expr */ + 277, /* (223) case_else ::= ELSE expr */ + 277, /* (224) case_else ::= */ + 275, /* (225) case_operand ::= expr */ + 275, /* (226) case_operand ::= */ + 259, /* (227) exprlist ::= */ + 250, /* (228) nexprlist ::= nexprlist COMMA expr */ + 250, /* (229) nexprlist ::= expr */ + 274, /* (230) paren_exprlist ::= */ + 274, /* (231) paren_exprlist ::= LP exprlist RP */ + 188, /* (232) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 278, /* (233) uniqueflag ::= UNIQUE */ + 278, /* (234) uniqueflag ::= */ + 218, /* (235) eidlist_opt ::= */ + 218, /* (236) eidlist_opt ::= LP eidlist RP */ + 229, /* (237) eidlist ::= eidlist COMMA nm collate sortorder */ + 229, /* (238) eidlist ::= nm collate sortorder */ + 279, /* (239) collate ::= */ + 279, /* (240) collate ::= COLLATE ID|STRING */ + 188, /* (241) cmd ::= DROP INDEX ifexists fullname */ + 188, /* (242) cmd ::= VACUUM vinto */ + 188, /* (243) cmd ::= VACUUM nm vinto */ + 280, /* (244) vinto ::= INTO expr */ + 280, /* (245) vinto ::= */ + 188, /* (246) cmd ::= PRAGMA nm dbnm */ + 188, /* (247) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 188, /* (248) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 188, /* (249) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 188, /* (250) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 208, /* (251) plus_num ::= PLUS INTEGER|FLOAT */ + 209, /* (252) minus_num ::= MINUS INTEGER|FLOAT */ + 188, /* (253) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 282, /* (254) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 284, /* (255) trigger_time ::= BEFORE|AFTER */ + 284, /* (256) trigger_time ::= INSTEAD OF */ + 284, /* (257) trigger_time ::= */ + 285, /* (258) trigger_event ::= DELETE|INSERT */ + 285, /* (259) trigger_event ::= UPDATE */ + 285, /* (260) trigger_event ::= UPDATE OF idlist */ + 287, /* (261) when_clause ::= */ + 287, /* (262) when_clause ::= WHEN expr */ + 283, /* (263) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 283, /* (264) trigger_cmd_list ::= trigger_cmd SEMI */ + 289, /* (265) trnm ::= nm DOT nm */ + 290, /* (266) tridxby ::= INDEXED BY nm */ + 290, /* (267) tridxby ::= NOT INDEXED */ + 288, /* (268) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 288, /* (269) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 288, /* (270) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 288, /* (271) trigger_cmd ::= scanpt select scanpt */ + 214, /* (272) expr ::= RAISE LP IGNORE RP */ + 214, /* (273) expr ::= RAISE LP raisetype COMMA nm RP */ + 233, /* (274) raisetype ::= ROLLBACK */ + 233, /* (275) raisetype ::= ABORT */ + 233, /* (276) raisetype ::= FAIL */ + 188, /* (277) cmd ::= DROP TRIGGER ifexists fullname */ + 188, /* (278) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 188, /* (279) cmd ::= DETACH database_kw_opt expr */ + 292, /* (280) key_opt ::= */ + 292, /* (281) key_opt ::= KEY expr */ + 188, /* (282) cmd ::= REINDEX */ + 188, /* (283) cmd ::= REINDEX nm dbnm */ + 188, /* (284) cmd ::= ANALYZE */ + 188, /* (285) cmd ::= ANALYZE nm dbnm */ + 188, /* (286) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 188, /* (287) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 188, /* (288) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 293, /* (289) add_column_fullname ::= fullname */ + 188, /* (290) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 188, /* (291) cmd ::= create_vtab */ + 188, /* (292) cmd ::= create_vtab LP vtabarglist RP */ + 295, /* (293) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 297, /* (294) vtabarg ::= */ + 298, /* (295) vtabargtoken ::= ANY */ + 298, /* (296) vtabargtoken ::= lp anylist RP */ + 299, /* (297) lp ::= LP */ + 263, /* (298) with ::= WITH wqlist */ + 263, /* (299) with ::= WITH RECURSIVE wqlist */ + 302, /* (300) wqas ::= AS */ + 302, /* (301) wqas ::= AS MATERIALIZED */ + 302, /* (302) wqas ::= AS NOT MATERIALIZED */ + 301, /* (303) wqitem ::= nm eidlist_opt wqas LP select RP */ + 238, /* (304) wqlist ::= wqitem */ + 238, /* (305) wqlist ::= wqlist COMMA wqitem */ + 303, /* (306) windowdefn_list ::= windowdefn */ + 303, /* (307) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 304, /* (308) windowdefn ::= nm AS LP window RP */ + 305, /* (309) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 305, /* (310) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 305, /* (311) window ::= ORDER BY sortlist frame_opt */ + 305, /* (312) window ::= nm ORDER BY sortlist frame_opt */ + 305, /* (313) window ::= frame_opt */ + 305, /* (314) window ::= nm frame_opt */ + 306, /* (315) frame_opt ::= */ + 306, /* (316) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 306, /* (317) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 310, /* (318) range_or_rows ::= RANGE|ROWS|GROUPS */ + 312, /* (319) frame_bound_s ::= frame_bound */ + 312, /* (320) frame_bound_s ::= UNBOUNDED PRECEDING */ + 313, /* (321) frame_bound_e ::= frame_bound */ + 313, /* (322) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 311, /* (323) frame_bound ::= expr PRECEDING|FOLLOWING */ + 311, /* (324) frame_bound ::= CURRENT ROW */ + 314, /* (325) frame_exclude_opt ::= */ + 314, /* (326) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 315, /* (327) frame_exclude ::= NO OTHERS */ + 315, /* (328) frame_exclude ::= CURRENT ROW */ + 315, /* (329) frame_exclude ::= GROUP|TIES */ + 248, /* (330) window_clause ::= WINDOW windowdefn_list */ + 270, /* (331) filter_over ::= filter_clause over_clause */ + 270, /* (332) filter_over ::= over_clause */ + 270, /* (333) filter_over ::= filter_clause */ + 309, /* (334) over_clause ::= OVER LP window RP */ + 309, /* (335) over_clause ::= OVER nm */ + 308, /* (336) filter_clause ::= FILTER LP WHERE expr RP */ + 183, /* (337) input ::= cmdlist */ + 184, /* (338) cmdlist ::= cmdlist ecmd */ + 184, /* (339) cmdlist ::= ecmd */ + 185, /* (340) ecmd ::= SEMI */ + 185, /* (341) ecmd ::= cmdx SEMI */ + 185, /* (342) ecmd ::= explain cmdx SEMI */ + 190, /* (343) trans_opt ::= */ + 190, /* (344) trans_opt ::= TRANSACTION */ + 190, /* (345) trans_opt ::= TRANSACTION nm */ + 192, /* (346) savepoint_opt ::= SAVEPOINT */ + 192, /* (347) savepoint_opt ::= */ + 188, /* (348) cmd ::= create_table create_table_args */ + 199, /* (349) columnlist ::= columnlist COMMA columnname carglist */ + 199, /* (350) columnlist ::= columnname carglist */ + 191, /* (351) nm ::= ID|INDEXED */ + 191, /* (352) nm ::= STRING */ + 191, /* (353) nm ::= JOIN_KW */ + 205, /* (354) typetoken ::= typename */ + 206, /* (355) typename ::= ID|STRING */ + 207, /* (356) signed ::= plus_num */ + 207, /* (357) signed ::= minus_num */ + 204, /* (358) carglist ::= carglist ccons */ + 204, /* (359) carglist ::= */ + 212, /* (360) ccons ::= NULL onconf */ + 212, /* (361) ccons ::= GENERATED ALWAYS AS generated */ + 212, /* (362) ccons ::= AS generated */ + 200, /* (363) conslist_opt ::= COMMA conslist */ + 225, /* (364) conslist ::= conslist tconscomma tcons */ + 225, /* (365) conslist ::= tcons */ + 226, /* (366) tconscomma ::= */ + 230, /* (367) defer_subclause_opt ::= defer_subclause */ + 232, /* (368) resolvetype ::= raisetype */ + 236, /* (369) selectnowith ::= oneselect */ + 237, /* (370) oneselect ::= values */ + 251, /* (371) sclp ::= selcollist COMMA */ + 252, /* (372) as ::= ID|STRING */ + 269, /* (373) returning ::= */ + 214, /* (374) expr ::= term */ + 271, /* (375) likeop ::= LIKE_KW|MATCH */ + 259, /* (376) exprlist ::= nexprlist */ + 281, /* (377) nmnum ::= plus_num */ + 281, /* (378) nmnum ::= nm */ + 281, /* (379) nmnum ::= ON */ + 281, /* (380) nmnum ::= DELETE */ + 281, /* (381) nmnum ::= DEFAULT */ + 208, /* (382) plus_num ::= INTEGER|FLOAT */ + 286, /* (383) foreach_clause ::= */ + 286, /* (384) foreach_clause ::= FOR EACH ROW */ + 289, /* (385) trnm ::= nm */ + 290, /* (386) tridxby ::= */ + 291, /* (387) database_kw_opt ::= DATABASE */ + 291, /* (388) database_kw_opt ::= */ + 294, /* (389) kwcolumn_opt ::= */ + 294, /* (390) kwcolumn_opt ::= COLUMNKW */ + 296, /* (391) vtabarglist ::= vtabarg */ + 296, /* (392) vtabarglist ::= vtabarglist COMMA vtabarg */ + 297, /* (393) vtabarg ::= vtabarg vtabargtoken */ + 300, /* (394) anylist ::= */ + 300, /* (395) anylist ::= anylist LP anylist RP */ + 300, /* (396) anylist ::= anylist ANY */ + 263, /* (397) with ::= */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) explain ::= EXPLAIN */ + -3, /* (1) explain ::= EXPLAIN QUERY PLAN */ + -1, /* (2) cmdx ::= cmd */ + -3, /* (3) cmd ::= BEGIN transtype trans_opt */ + 0, /* (4) transtype ::= */ + -1, /* (5) transtype ::= DEFERRED */ + -1, /* (6) transtype ::= IMMEDIATE */ + -1, /* (7) transtype ::= EXCLUSIVE */ + -2, /* (8) cmd ::= COMMIT|END trans_opt */ + -2, /* (9) cmd ::= ROLLBACK trans_opt */ + -2, /* (10) cmd ::= SAVEPOINT nm */ + -3, /* (11) cmd ::= RELEASE savepoint_opt nm */ + -5, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + -6, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + -1, /* (14) createkw ::= CREATE */ + 0, /* (15) ifnotexists ::= */ + -3, /* (16) ifnotexists ::= IF NOT EXISTS */ + -1, /* (17) temp ::= TEMP */ + 0, /* (18) temp ::= */ + -5, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ + -2, /* (20) create_table_args ::= AS select */ + 0, /* (21) table_options ::= */ + -2, /* (22) table_options ::= WITHOUT nm */ + -2, /* (23) columnname ::= nm typetoken */ + 0, /* (24) typetoken ::= */ + -4, /* (25) typetoken ::= typename LP signed RP */ + -6, /* (26) typetoken ::= typename LP signed COMMA signed RP */ + -2, /* (27) typename ::= typename ID|STRING */ + 0, /* (28) scanpt ::= */ + 0, /* (29) scantok ::= */ + -2, /* (30) ccons ::= CONSTRAINT nm */ + -3, /* (31) ccons ::= DEFAULT scantok term */ + -4, /* (32) ccons ::= DEFAULT LP expr RP */ + -4, /* (33) ccons ::= DEFAULT PLUS scantok term */ + -4, /* (34) ccons ::= DEFAULT MINUS scantok term */ + -3, /* (35) ccons ::= DEFAULT scantok ID|INDEXED */ + -3, /* (36) ccons ::= NOT NULL onconf */ + -5, /* (37) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + -2, /* (38) ccons ::= UNIQUE onconf */ + -4, /* (39) ccons ::= CHECK LP expr RP */ + -4, /* (40) ccons ::= REFERENCES nm eidlist_opt refargs */ + -1, /* (41) ccons ::= defer_subclause */ + -2, /* (42) ccons ::= COLLATE ID|STRING */ + -3, /* (43) generated ::= LP expr RP */ + -4, /* (44) generated ::= LP expr RP ID */ + 0, /* (45) autoinc ::= */ + -1, /* (46) autoinc ::= AUTOINCR */ + 0, /* (47) refargs ::= */ + -2, /* (48) refargs ::= refargs refarg */ + -2, /* (49) refarg ::= MATCH nm */ + -3, /* (50) refarg ::= ON INSERT refact */ + -3, /* (51) refarg ::= ON DELETE refact */ + -3, /* (52) refarg ::= ON UPDATE refact */ + -2, /* (53) refact ::= SET NULL */ + -2, /* (54) refact ::= SET DEFAULT */ + -1, /* (55) refact ::= CASCADE */ + -1, /* (56) refact ::= RESTRICT */ + -2, /* (57) refact ::= NO ACTION */ + -3, /* (58) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + -2, /* (59) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 0, /* (60) init_deferred_pred_opt ::= */ + -2, /* (61) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + -2, /* (62) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 0, /* (63) conslist_opt ::= */ + -1, /* (64) tconscomma ::= COMMA */ + -2, /* (65) tcons ::= CONSTRAINT nm */ + -7, /* (66) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + -5, /* (67) tcons ::= UNIQUE LP sortlist RP onconf */ + -5, /* (68) tcons ::= CHECK LP expr RP onconf */ + -10, /* (69) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 0, /* (70) defer_subclause_opt ::= */ + 0, /* (71) onconf ::= */ + -3, /* (72) onconf ::= ON CONFLICT resolvetype */ + 0, /* (73) orconf ::= */ + -2, /* (74) orconf ::= OR resolvetype */ + -1, /* (75) resolvetype ::= IGNORE */ + -1, /* (76) resolvetype ::= REPLACE */ + -4, /* (77) cmd ::= DROP TABLE ifexists fullname */ + -2, /* (78) ifexists ::= IF EXISTS */ + 0, /* (79) ifexists ::= */ + -9, /* (80) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + -4, /* (81) cmd ::= DROP VIEW ifexists fullname */ + -1, /* (82) cmd ::= select */ + -3, /* (83) select ::= WITH wqlist selectnowith */ + -4, /* (84) select ::= WITH RECURSIVE wqlist selectnowith */ + -1, /* (85) select ::= selectnowith */ + -3, /* (86) selectnowith ::= selectnowith multiselect_op oneselect */ + -1, /* (87) multiselect_op ::= UNION */ + -2, /* (88) multiselect_op ::= UNION ALL */ + -1, /* (89) multiselect_op ::= EXCEPT|INTERSECT */ + -9, /* (90) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + -10, /* (91) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + -4, /* (92) values ::= VALUES LP nexprlist RP */ + -5, /* (93) values ::= values COMMA LP nexprlist RP */ + -1, /* (94) distinct ::= DISTINCT */ + -1, /* (95) distinct ::= ALL */ + 0, /* (96) distinct ::= */ + 0, /* (97) sclp ::= */ + -5, /* (98) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (99) selcollist ::= sclp scanpt STAR */ + -5, /* (100) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (101) as ::= AS nm */ + 0, /* (102) as ::= */ + 0, /* (103) from ::= */ + -2, /* (104) from ::= FROM seltablist */ + -2, /* (105) stl_prefix ::= seltablist joinop */ + 0, /* (106) stl_prefix ::= */ + -7, /* (107) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + -9, /* (108) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + -7, /* (109) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + -7, /* (110) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 0, /* (111) dbnm ::= */ + -2, /* (112) dbnm ::= DOT nm */ + -1, /* (113) fullname ::= nm */ + -3, /* (114) fullname ::= nm DOT nm */ + -1, /* (115) xfullname ::= nm */ + -3, /* (116) xfullname ::= nm DOT nm */ + -5, /* (117) xfullname ::= nm DOT nm AS nm */ + -3, /* (118) xfullname ::= nm AS nm */ + -1, /* (119) joinop ::= COMMA|JOIN */ + -2, /* (120) joinop ::= JOIN_KW JOIN */ + -3, /* (121) joinop ::= JOIN_KW nm JOIN */ + -4, /* (122) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (123) on_opt ::= ON expr */ + 0, /* (124) on_opt ::= */ + 0, /* (125) indexed_opt ::= */ + -3, /* (126) indexed_opt ::= INDEXED BY nm */ + -2, /* (127) indexed_opt ::= NOT INDEXED */ + -4, /* (128) using_opt ::= USING LP idlist RP */ + 0, /* (129) using_opt ::= */ + 0, /* (130) orderby_opt ::= */ + -3, /* (131) orderby_opt ::= ORDER BY sortlist */ + -5, /* (132) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (133) sortlist ::= expr sortorder nulls */ + -1, /* (134) sortorder ::= ASC */ + -1, /* (135) sortorder ::= DESC */ + 0, /* (136) sortorder ::= */ + -2, /* (137) nulls ::= NULLS FIRST */ + -2, /* (138) nulls ::= NULLS LAST */ + 0, /* (139) nulls ::= */ + 0, /* (140) groupby_opt ::= */ + -3, /* (141) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (142) having_opt ::= */ + -2, /* (143) having_opt ::= HAVING expr */ + 0, /* (144) limit_opt ::= */ + -2, /* (145) limit_opt ::= LIMIT expr */ + -4, /* (146) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (147) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (148) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (149) where_opt ::= */ + -2, /* (150) where_opt ::= WHERE expr */ + 0, /* (151) where_opt_ret ::= */ + -2, /* (152) where_opt_ret ::= WHERE expr */ + -2, /* (153) where_opt_ret ::= RETURNING selcollist */ + -4, /* (154) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (155) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (156) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (157) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (158) setlist ::= nm EQ expr */ + -5, /* (159) setlist ::= LP idlist RP EQ expr */ + -7, /* (160) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (161) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (162) upsert ::= */ + -2, /* (163) upsert ::= RETURNING selcollist */ + -12, /* (164) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (165) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (166) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (167) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (168) returning ::= RETURNING selcollist */ + -2, /* (169) insert_cmd ::= INSERT orconf */ + -1, /* (170) insert_cmd ::= REPLACE */ + 0, /* (171) idlist_opt ::= */ + -3, /* (172) idlist_opt ::= LP idlist RP */ + -3, /* (173) idlist ::= idlist COMMA nm */ + -1, /* (174) idlist ::= nm */ + -3, /* (175) expr ::= LP expr RP */ + -1, /* (176) expr ::= ID|INDEXED */ + -1, /* (177) expr ::= JOIN_KW */ + -3, /* (178) expr ::= nm DOT nm */ + -5, /* (179) expr ::= nm DOT nm DOT nm */ + -1, /* (180) term ::= NULL|FLOAT|BLOB */ + -1, /* (181) term ::= STRING */ + -1, /* (182) term ::= INTEGER */ + -1, /* (183) expr ::= VARIABLE */ + -3, /* (184) expr ::= expr COLLATE ID|STRING */ + -6, /* (185) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (186) expr ::= ID|INDEXED LP distinct exprlist RP */ + -4, /* (187) expr ::= ID|INDEXED LP STAR RP */ + -6, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + -5, /* (189) expr ::= ID|INDEXED LP STAR RP filter_over */ + -1, /* (190) term ::= CTIME_KW */ + -5, /* (191) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (192) expr ::= expr AND expr */ + -3, /* (193) expr ::= expr OR expr */ + -3, /* (194) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (195) expr ::= expr EQ|NE expr */ + -3, /* (196) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (197) expr ::= expr PLUS|MINUS expr */ + -3, /* (198) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (199) expr ::= expr CONCAT expr */ + -2, /* (200) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (201) expr ::= expr likeop expr */ + -5, /* (202) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (203) expr ::= expr ISNULL|NOTNULL */ + -3, /* (204) expr ::= expr NOT NULL */ + -3, /* (205) expr ::= expr IS expr */ + -4, /* (206) expr ::= expr IS NOT expr */ + -2, /* (207) expr ::= NOT expr */ + -2, /* (208) expr ::= BITNOT expr */ + -2, /* (209) expr ::= PLUS|MINUS expr */ + -1, /* (210) between_op ::= BETWEEN */ + -2, /* (211) between_op ::= NOT BETWEEN */ + -5, /* (212) expr ::= expr between_op expr AND expr */ + -1, /* (213) in_op ::= IN */ + -2, /* (214) in_op ::= NOT IN */ + -5, /* (215) expr ::= expr in_op LP exprlist RP */ + -3, /* (216) expr ::= LP select RP */ + -5, /* (217) expr ::= expr in_op LP select RP */ + -5, /* (218) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (219) expr ::= EXISTS LP select RP */ + -5, /* (220) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (221) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (222) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (223) case_else ::= ELSE expr */ + 0, /* (224) case_else ::= */ + -1, /* (225) case_operand ::= expr */ + 0, /* (226) case_operand ::= */ + 0, /* (227) exprlist ::= */ + -3, /* (228) nexprlist ::= nexprlist COMMA expr */ + -1, /* (229) nexprlist ::= expr */ + 0, /* (230) paren_exprlist ::= */ + -3, /* (231) paren_exprlist ::= LP exprlist RP */ + -12, /* (232) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (233) uniqueflag ::= UNIQUE */ + 0, /* (234) uniqueflag ::= */ + 0, /* (235) eidlist_opt ::= */ + -3, /* (236) eidlist_opt ::= LP eidlist RP */ + -5, /* (237) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (238) eidlist ::= nm collate sortorder */ + 0, /* (239) collate ::= */ + -2, /* (240) collate ::= COLLATE ID|STRING */ + -4, /* (241) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (242) cmd ::= VACUUM vinto */ + -3, /* (243) cmd ::= VACUUM nm vinto */ + -2, /* (244) vinto ::= INTO expr */ + 0, /* (245) vinto ::= */ + -3, /* (246) cmd ::= PRAGMA nm dbnm */ + -5, /* (247) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (248) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (249) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (250) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (251) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (252) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (253) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (254) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (255) trigger_time ::= BEFORE|AFTER */ + -2, /* (256) trigger_time ::= INSTEAD OF */ + 0, /* (257) trigger_time ::= */ + -1, /* (258) trigger_event ::= DELETE|INSERT */ + -1, /* (259) trigger_event ::= UPDATE */ + -3, /* (260) trigger_event ::= UPDATE OF idlist */ + 0, /* (261) when_clause ::= */ + -2, /* (262) when_clause ::= WHEN expr */ + -3, /* (263) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (264) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (265) trnm ::= nm DOT nm */ + -3, /* (266) tridxby ::= INDEXED BY nm */ + -2, /* (267) tridxby ::= NOT INDEXED */ + -9, /* (268) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (269) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (270) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (271) trigger_cmd ::= scanpt select scanpt */ + -4, /* (272) expr ::= RAISE LP IGNORE RP */ + -6, /* (273) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (274) raisetype ::= ROLLBACK */ + -1, /* (275) raisetype ::= ABORT */ + -1, /* (276) raisetype ::= FAIL */ + -4, /* (277) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (278) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (279) cmd ::= DETACH database_kw_opt expr */ + 0, /* (280) key_opt ::= */ + -2, /* (281) key_opt ::= KEY expr */ + -1, /* (282) cmd ::= REINDEX */ + -3, /* (283) cmd ::= REINDEX nm dbnm */ + -1, /* (284) cmd ::= ANALYZE */ + -3, /* (285) cmd ::= ANALYZE nm dbnm */ + -6, /* (286) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (287) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (288) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (289) add_column_fullname ::= fullname */ + -8, /* (290) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (291) cmd ::= create_vtab */ + -4, /* (292) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (293) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (294) vtabarg ::= */ + -1, /* (295) vtabargtoken ::= ANY */ + -3, /* (296) vtabargtoken ::= lp anylist RP */ + -1, /* (297) lp ::= LP */ + -2, /* (298) with ::= WITH wqlist */ + -3, /* (299) with ::= WITH RECURSIVE wqlist */ + -1, /* (300) wqas ::= AS */ + -2, /* (301) wqas ::= AS MATERIALIZED */ + -3, /* (302) wqas ::= AS NOT MATERIALIZED */ + -6, /* (303) wqitem ::= nm eidlist_opt wqas LP select RP */ + -1, /* (304) wqlist ::= wqitem */ + -3, /* (305) wqlist ::= wqlist COMMA wqitem */ + -1, /* (306) windowdefn_list ::= windowdefn */ + -3, /* (307) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (308) windowdefn ::= nm AS LP window RP */ + -5, /* (309) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (310) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (311) window ::= ORDER BY sortlist frame_opt */ + -5, /* (312) window ::= nm ORDER BY sortlist frame_opt */ + -1, /* (313) window ::= frame_opt */ + -2, /* (314) window ::= nm frame_opt */ + 0, /* (315) frame_opt ::= */ + -3, /* (316) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (317) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (318) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (319) frame_bound_s ::= frame_bound */ + -2, /* (320) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (321) frame_bound_e ::= frame_bound */ + -2, /* (322) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (323) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (324) frame_bound ::= CURRENT ROW */ + 0, /* (325) frame_exclude_opt ::= */ + -2, /* (326) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (327) frame_exclude ::= NO OTHERS */ + -2, /* (328) frame_exclude ::= CURRENT ROW */ + -1, /* (329) frame_exclude ::= GROUP|TIES */ + -2, /* (330) window_clause ::= WINDOW windowdefn_list */ + -2, /* (331) filter_over ::= filter_clause over_clause */ + -1, /* (332) filter_over ::= over_clause */ + -1, /* (333) filter_over ::= filter_clause */ + -4, /* (334) over_clause ::= OVER LP window RP */ + -2, /* (335) over_clause ::= OVER nm */ + -5, /* (336) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (337) input ::= cmdlist */ + -2, /* (338) cmdlist ::= cmdlist ecmd */ + -1, /* (339) cmdlist ::= ecmd */ + -1, /* (340) ecmd ::= SEMI */ + -2, /* (341) ecmd ::= cmdx SEMI */ + -3, /* (342) ecmd ::= explain cmdx SEMI */ + 0, /* (343) trans_opt ::= */ + -1, /* (344) trans_opt ::= TRANSACTION */ + -2, /* (345) trans_opt ::= TRANSACTION nm */ + -1, /* (346) savepoint_opt ::= SAVEPOINT */ + 0, /* (347) savepoint_opt ::= */ + -2, /* (348) cmd ::= create_table create_table_args */ + -4, /* (349) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (350) columnlist ::= columnname carglist */ + -1, /* (351) nm ::= ID|INDEXED */ + -1, /* (352) nm ::= STRING */ + -1, /* (353) nm ::= JOIN_KW */ + -1, /* (354) typetoken ::= typename */ + -1, /* (355) typename ::= ID|STRING */ + -1, /* (356) signed ::= plus_num */ + -1, /* (357) signed ::= minus_num */ + -2, /* (358) carglist ::= carglist ccons */ + 0, /* (359) carglist ::= */ + -2, /* (360) ccons ::= NULL onconf */ + -4, /* (361) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (362) ccons ::= AS generated */ + -2, /* (363) conslist_opt ::= COMMA conslist */ + -3, /* (364) conslist ::= conslist tconscomma tcons */ + -1, /* (365) conslist ::= tcons */ + 0, /* (366) tconscomma ::= */ + -1, /* (367) defer_subclause_opt ::= defer_subclause */ + -1, /* (368) resolvetype ::= raisetype */ + -1, /* (369) selectnowith ::= oneselect */ + -1, /* (370) oneselect ::= values */ + -2, /* (371) sclp ::= selcollist COMMA */ + -1, /* (372) as ::= ID|STRING */ + 0, /* (373) returning ::= */ + -1, /* (374) expr ::= term */ + -1, /* (375) likeop ::= LIKE_KW|MATCH */ + -1, /* (376) exprlist ::= nexprlist */ + -1, /* (377) nmnum ::= plus_num */ + -1, /* (378) nmnum ::= nm */ + -1, /* (379) nmnum ::= ON */ + -1, /* (380) nmnum ::= DELETE */ + -1, /* (381) nmnum ::= DEFAULT */ + -1, /* (382) plus_num ::= INTEGER|FLOAT */ + 0, /* (383) foreach_clause ::= */ + -3, /* (384) foreach_clause ::= FOR EACH ROW */ + -1, /* (385) trnm ::= nm */ + 0, /* (386) tridxby ::= */ + -1, /* (387) database_kw_opt ::= DATABASE */ + 0, /* (388) database_kw_opt ::= */ + 0, /* (389) kwcolumn_opt ::= */ + -1, /* (390) kwcolumn_opt ::= COLUMNKW */ + -1, /* (391) vtabarglist ::= vtabarg */ + -3, /* (392) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (393) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (394) anylist ::= */ + -4, /* (395) anylist ::= anylist LP anylist RP */ + -2, /* (396) anylist ::= anylist ANY */ + 0, /* (397) with ::= */ +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + sqlite3ParserCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + sqlite3ParserARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* explain ::= EXPLAIN */ +{ pParse->explain = 1; } + break; + case 1: /* explain ::= EXPLAIN QUERY PLAN */ +{ pParse->explain = 2; } + break; + case 2: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 3: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy60);} + break; + case 4: /* transtype ::= */ +{yymsp[1].minor.yy60 = TK_DEFERRED;} + break; + case 5: /* transtype ::= DEFERRED */ + case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); + case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); + case 318: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==318); +{yymsp[0].minor.yy60 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 8: /* cmd ::= COMMIT|END trans_opt */ + case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); +{sqlite3EndTransaction(pParse,yymsp[-1].major);} + break; + case 10: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 11: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 12: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy60,0,0,yymsp[-2].minor.yy60); +} + break; + case 14: /* createkw ::= CREATE */ +{disableLookaside(pParse);} + break; + case 15: /* ifnotexists ::= */ + case 18: /* temp ::= */ yytestcase(yyruleno==18); + case 21: /* table_options ::= */ yytestcase(yyruleno==21); + case 45: /* autoinc ::= */ yytestcase(yyruleno==45); + case 60: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==60); + case 70: /* defer_subclause_opt ::= */ yytestcase(yyruleno==70); + case 79: /* ifexists ::= */ yytestcase(yyruleno==79); + case 96: /* distinct ::= */ yytestcase(yyruleno==96); + case 239: /* collate ::= */ yytestcase(yyruleno==239); +{yymsp[1].minor.yy60 = 0;} + break; + case 16: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy60 = 1;} + break; + case 17: /* temp ::= TEMP */ + case 46: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==46); +{yymsp[0].minor.yy60 = 1;} + break; + case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ +{ + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy60,0); +} + break; + case 20: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy307); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy307); +} + break; + case 22: /* table_options ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yymsp[-1].minor.yy60 = TF_WithoutRowid | TF_NoVisibleRowid; + }else{ + yymsp[-1].minor.yy60 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + break; + case 23: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 24: /* typetoken ::= */ + case 63: /* conslist_opt ::= */ yytestcase(yyruleno==63); + case 102: /* as ::= */ yytestcase(yyruleno==102); +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} + break; + case 25: /* typetoken ::= typename LP signed RP */ +{ + yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 26: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 27: /* typename ::= typename ID|STRING */ +{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 28: /* scanpt ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy528 = yyLookaheadToken.z; +} + break; + case 29: /* scantok ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy0 = yyLookaheadToken; +} + break; + case 30: /* ccons ::= CONSTRAINT nm */ + case 65: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==65); +{pParse->constraintName = yymsp[0].minor.yy0;} + break; + case 31: /* ccons ::= DEFAULT scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy602,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 32: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy602,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} + break; + case 33: /* ccons ::= DEFAULT PLUS scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy602,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 34: /* ccons ::= DEFAULT MINUS scantok term */ +{ + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy602, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); +} + break; + case 35: /* ccons ::= DEFAULT scantok ID|INDEXED */ +{ + Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } + sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); +} + break; + case 36: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy60);} + break; + case 37: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy60,yymsp[0].minor.yy60,yymsp[-2].minor.yy60);} + break; + case 38: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy60,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 39: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy602,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} + break; + case 40: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy338,yymsp[0].minor.yy60);} + break; + case 41: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy60);} + break; + case 42: /* ccons ::= COLLATE ID|STRING */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 43: /* generated ::= LP expr RP */ +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy602,0);} + break; + case 44: /* generated ::= LP expr RP ID */ +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy602,&yymsp[0].minor.yy0);} + break; + case 47: /* refargs ::= */ +{ yymsp[1].minor.yy60 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 48: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy60 = (yymsp[-1].minor.yy60 & ~yymsp[0].minor.yy615.mask) | yymsp[0].minor.yy615.value; } + break; + case 49: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy615.value = 0; yymsp[-1].minor.yy615.mask = 0x000000; } + break; + case 50: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy615.value = 0; yymsp[-2].minor.yy615.mask = 0x000000; } + break; + case 51: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy615.value = yymsp[0].minor.yy60; yymsp[-2].minor.yy615.mask = 0x0000ff; } + break; + case 52: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy615.value = yymsp[0].minor.yy60<<8; yymsp[-2].minor.yy615.mask = 0x00ff00; } + break; + case 53: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy60 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 54: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy60 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 55: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy60 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 56: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy60 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 57: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy60 = OE_None; /* EV: R-33326-45252 */} + break; + case 58: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy60 = 0;} + break; + case 59: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 74: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==74); + case 169: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==169); +{yymsp[-1].minor.yy60 = yymsp[0].minor.yy60;} + break; + case 61: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 78: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==78); + case 211: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==211); + case 214: /* in_op ::= NOT IN */ yytestcase(yyruleno==214); + case 240: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==240); +{yymsp[-1].minor.yy60 = 1;} + break; + case 62: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy60 = 0;} + break; + case 64: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} + break; + case 66: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy338,yymsp[0].minor.yy60,yymsp[-2].minor.yy60,0);} + break; + case 67: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy338,yymsp[0].minor.yy60,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 68: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy602,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} + break; + case 69: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy338, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy338, yymsp[-1].minor.yy60); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy60); +} + break; + case 71: /* onconf ::= */ + case 73: /* orconf ::= */ yytestcase(yyruleno==73); +{yymsp[1].minor.yy60 = OE_Default;} + break; + case 72: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy60 = yymsp[0].minor.yy60;} + break; + case 75: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy60 = OE_Ignore;} + break; + case 76: /* resolvetype ::= REPLACE */ + case 170: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==170); +{yymsp[0].minor.yy60 = OE_Replace;} + break; + case 77: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy291, 0, yymsp[-1].minor.yy60); +} + break; + case 80: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy338, yymsp[0].minor.yy307, yymsp[-7].minor.yy60, yymsp[-5].minor.yy60); +} + break; + case 81: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy291, 1, yymsp[-1].minor.yy60); +} + break; + case 82: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy307, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy307); +} + break; + case 83: /* select ::= WITH wqlist selectnowith */ +{yymsp[-2].minor.yy307 = attachWithToSelect(pParse,yymsp[0].minor.yy307,yymsp[-1].minor.yy195);} + break; + case 84: /* select ::= WITH RECURSIVE wqlist selectnowith */ +{yymsp[-3].minor.yy307 = attachWithToSelect(pParse,yymsp[0].minor.yy307,yymsp[-1].minor.yy195);} + break; + case 85: /* select ::= selectnowith */ +{ + Select *p = yymsp[0].minor.yy307; + if( p ){ + parserDoubleLinkSelect(pParse, p); + } + yymsp[0].minor.yy307 = p; /*A-overwrites-X*/ +} + break; + case 86: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy307; + Select *pLhs = yymsp[-2].minor.yy307; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy60; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; + pRhs->selFlags &= ~SF_MultiValue; + if( yymsp[-1].minor.yy60!=TK_ALL ) pParse->hasCompound = 1; + }else{ + sqlite3SelectDelete(pParse->db, pLhs); + } + yymsp[-2].minor.yy307 = pRhs; +} + break; + case 87: /* multiselect_op ::= UNION */ + case 89: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==89); +{yymsp[0].minor.yy60 = yymsp[0].major; /*A-overwrites-OP*/} + break; + case 88: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy60 = TK_ALL;} + break; + case 90: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yymsp[-8].minor.yy307 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy338,yymsp[-5].minor.yy291,yymsp[-4].minor.yy602,yymsp[-3].minor.yy338,yymsp[-2].minor.yy602,yymsp[-1].minor.yy338,yymsp[-7].minor.yy60,yymsp[0].minor.yy602); +} + break; + case 91: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yymsp[-9].minor.yy307 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy338,yymsp[-6].minor.yy291,yymsp[-5].minor.yy602,yymsp[-4].minor.yy338,yymsp[-3].minor.yy602,yymsp[-1].minor.yy338,yymsp[-8].minor.yy60,yymsp[0].minor.yy602); + if( yymsp[-9].minor.yy307 ){ + yymsp[-9].minor.yy307->pWinDefn = yymsp[-2].minor.yy19; + }else{ + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy19); + } +} +#endif + break; + case 92: /* values ::= VALUES LP nexprlist RP */ +{ + yymsp[-3].minor.yy307 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy338,0,0,0,0,0,SF_Values,0); +} + break; + case 93: /* values ::= values COMMA LP nexprlist RP */ +{ + Select *pRight, *pLeft = yymsp[-4].minor.yy307; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy338,0,0,0,0,0,SF_Values|SF_MultiValue,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = pLeft; + yymsp[-4].minor.yy307 = pRight; + }else{ + yymsp[-4].minor.yy307 = pLeft; + } +} + break; + case 94: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy60 = SF_Distinct;} + break; + case 95: /* distinct ::= ALL */ +{yymsp[0].minor.yy60 = SF_All;} + break; + case 97: /* sclp ::= */ + case 130: /* orderby_opt ::= */ yytestcase(yyruleno==130); + case 140: /* groupby_opt ::= */ yytestcase(yyruleno==140); + case 227: /* exprlist ::= */ yytestcase(yyruleno==227); + case 230: /* paren_exprlist ::= */ yytestcase(yyruleno==230); + case 235: /* eidlist_opt ::= */ yytestcase(yyruleno==235); +{yymsp[1].minor.yy338 = 0;} + break; + case 98: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy338, yymsp[-2].minor.yy602); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy338, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy338,yymsp[-3].minor.yy528,yymsp[-1].minor.yy528); +} + break; + case 99: /* selcollist ::= sclp scanpt STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + yymsp[-2].minor.yy338 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy338, p); +} + break; + case 100: /* selcollist ::= sclp scanpt nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy338, pDot); +} + break; + case 101: /* as ::= AS nm */ + case 112: /* dbnm ::= DOT nm */ yytestcase(yyruleno==112); + case 251: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==251); + case 252: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==252); +{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 103: /* from ::= */ + case 106: /* stl_prefix ::= */ yytestcase(yyruleno==106); +{yymsp[1].minor.yy291 = 0;} + break; + case 104: /* from ::= FROM seltablist */ +{ + yymsp[-1].minor.yy291 = yymsp[0].minor.yy291; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy291); +} + break; + case 105: /* stl_prefix ::= seltablist joinop */ +{ + if( ALWAYS(yymsp[-1].minor.yy291 && yymsp[-1].minor.yy291->nSrc>0) ) yymsp[-1].minor.yy291->a[yymsp[-1].minor.yy291->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy60; +} + break; + case 107: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yymsp[-6].minor.yy291 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy291,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy602,yymsp[0].minor.yy288); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy291, &yymsp[-2].minor.yy0); +} + break; + case 108: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ +{ + yymsp[-8].minor.yy291 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy291,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy602,yymsp[0].minor.yy288); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy291, yymsp[-4].minor.yy338); +} + break; + case 109: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yymsp[-6].minor.yy291 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy291,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy307,yymsp[-1].minor.yy602,yymsp[0].minor.yy288); + } + break; + case 110: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +{ + if( yymsp[-6].minor.yy291==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy602==0 && yymsp[0].minor.yy288==0 ){ + yymsp[-6].minor.yy291 = yymsp[-4].minor.yy291; + }else if( yymsp[-4].minor.yy291->nSrc==1 ){ + yymsp[-6].minor.yy291 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy291,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy602,yymsp[0].minor.yy288); + if( yymsp[-6].minor.yy291 ){ + SrcItem *pNew = &yymsp[-6].minor.yy291->a[yymsp[-6].minor.yy291->nSrc-1]; + SrcItem *pOld = yymsp[-4].minor.yy291->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy291); + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy291); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy291,0,0,0,0,SF_NestedFrom,0); + yymsp[-6].minor.yy291 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy291,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy602,yymsp[0].minor.yy288); + } + } + break; + case 111: /* dbnm ::= */ + case 125: /* indexed_opt ::= */ yytestcase(yyruleno==125); +{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} + break; + case 113: /* fullname ::= nm */ +{ + yylhsminor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy291 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy291->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[0].minor.yy291 = yylhsminor.yy291; + break; + case 114: /* fullname ::= nm DOT nm */ +{ + yylhsminor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy291 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy291->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[-2].minor.yy291 = yylhsminor.yy291; + break; + case 115: /* xfullname ::= nm */ +{yymsp[0].minor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + break; + case 116: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 117: /* xfullname ::= nm DOT nm AS nm */ +{ + yymsp[-4].minor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy291 ) yymsp[-4].minor.yy291->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 118: /* xfullname ::= nm AS nm */ +{ + yymsp[-2].minor.yy291 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy291 ) yymsp[-2].minor.yy291->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 119: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy60 = JT_INNER; } + break; + case 120: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy60 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + break; + case 121: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy60 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + break; + case 122: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy60 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + break; + case 123: /* on_opt ::= ON expr */ + case 143: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==143); + case 150: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==150); + case 152: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==152); + case 223: /* case_else ::= ELSE expr */ yytestcase(yyruleno==223); + case 244: /* vinto ::= INTO expr */ yytestcase(yyruleno==244); +{yymsp[-1].minor.yy602 = yymsp[0].minor.yy602;} + break; + case 124: /* on_opt ::= */ + case 142: /* having_opt ::= */ yytestcase(yyruleno==142); + case 144: /* limit_opt ::= */ yytestcase(yyruleno==144); + case 149: /* where_opt ::= */ yytestcase(yyruleno==149); + case 151: /* where_opt_ret ::= */ yytestcase(yyruleno==151); + case 224: /* case_else ::= */ yytestcase(yyruleno==224); + case 226: /* case_operand ::= */ yytestcase(yyruleno==226); + case 245: /* vinto ::= */ yytestcase(yyruleno==245); +{yymsp[1].minor.yy602 = 0;} + break; + case 126: /* indexed_opt ::= INDEXED BY nm */ +{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 127: /* indexed_opt ::= NOT INDEXED */ +{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} + break; + case 128: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy288 = yymsp[-1].minor.yy288;} + break; + case 129: /* using_opt ::= */ + case 171: /* idlist_opt ::= */ yytestcase(yyruleno==171); +{yymsp[1].minor.yy288 = 0;} + break; + case 131: /* orderby_opt ::= ORDER BY sortlist */ + case 141: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==141); +{yymsp[-2].minor.yy338 = yymsp[0].minor.yy338;} + break; + case 132: /* sortlist ::= sortlist COMMA expr sortorder nulls */ +{ + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy338,yymsp[-2].minor.yy602); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy338,yymsp[-1].minor.yy60,yymsp[0].minor.yy60); +} + break; + case 133: /* sortlist ::= expr sortorder nulls */ +{ + yymsp[-2].minor.yy338 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy602); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy338,yymsp[-1].minor.yy60,yymsp[0].minor.yy60); +} + break; + case 134: /* sortorder ::= ASC */ +{yymsp[0].minor.yy60 = SQLITE_SO_ASC;} + break; + case 135: /* sortorder ::= DESC */ +{yymsp[0].minor.yy60 = SQLITE_SO_DESC;} + break; + case 136: /* sortorder ::= */ + case 139: /* nulls ::= */ yytestcase(yyruleno==139); +{yymsp[1].minor.yy60 = SQLITE_SO_UNDEFINED;} + break; + case 137: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy60 = SQLITE_SO_ASC;} + break; + case 138: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy60 = SQLITE_SO_DESC;} + break; + case 145: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy602 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy602,0);} + break; + case 146: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy602 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy602,yymsp[0].minor.yy602);} + break; + case 147: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy602 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy602,yymsp[-2].minor.yy602);} + break; + case 148: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy291, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy291,yymsp[0].minor.yy602,0,0); +} + break; + case 153: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy338); yymsp[-1].minor.yy602 = 0;} + break; + case 154: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy338); yymsp[-3].minor.yy602 = yymsp[-2].minor.yy602;} + break; + case 155: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy291, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy338,"set list"); + yymsp[-5].minor.yy291 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy291, yymsp[-1].minor.yy291); + sqlite3Update(pParse,yymsp[-5].minor.yy291,yymsp[-2].minor.yy338,yymsp[0].minor.yy602,yymsp[-6].minor.yy60,0,0,0); +} + break; + case 156: /* setlist ::= setlist COMMA nm EQ expr */ +{ + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy338, yymsp[0].minor.yy602); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy338, &yymsp[-2].minor.yy0, 1); +} + break; + case 157: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ +{ + yymsp[-6].minor.yy338 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy338, yymsp[-3].minor.yy288, yymsp[0].minor.yy602); +} + break; + case 158: /* setlist ::= nm EQ expr */ +{ + yylhsminor.yy338 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy602); + sqlite3ExprListSetName(pParse, yylhsminor.yy338, &yymsp[-2].minor.yy0, 1); +} + yymsp[-2].minor.yy338 = yylhsminor.yy338; + break; + case 159: /* setlist ::= LP idlist RP EQ expr */ +{ + yymsp[-4].minor.yy338 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy288, yymsp[0].minor.yy602); +} + break; + case 160: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ +{ + sqlite3Insert(pParse, yymsp[-3].minor.yy291, yymsp[-1].minor.yy307, yymsp[-2].minor.yy288, yymsp[-5].minor.yy60, yymsp[0].minor.yy178); +} + break; + case 161: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ +{ + sqlite3Insert(pParse, yymsp[-4].minor.yy291, 0, yymsp[-3].minor.yy288, yymsp[-6].minor.yy60, 0); +} + break; + case 162: /* upsert ::= */ +{ yymsp[1].minor.yy178 = 0; } + break; + case 163: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy178 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy338); } + break; + case 164: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy178 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy338,yymsp[-6].minor.yy602,yymsp[-2].minor.yy338,yymsp[-1].minor.yy602,yymsp[0].minor.yy178);} + break; + case 165: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy178 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy338,yymsp[-3].minor.yy602,0,0,yymsp[0].minor.yy178); } + break; + case 166: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy178 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + break; + case 167: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy178 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy338,yymsp[-1].minor.yy602,0);} + break; + case 168: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy338);} + break; + case 172: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy288 = yymsp[-1].minor.yy288;} + break; + case 173: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy288 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy288,&yymsp[0].minor.yy0);} + break; + case 174: /* idlist ::= nm */ +{yymsp[0].minor.yy288 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + break; + case 175: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy602 = yymsp[-1].minor.yy602;} + break; + case 176: /* expr ::= ID|INDEXED */ + case 177: /* expr ::= JOIN_KW */ yytestcase(yyruleno==177); +{yymsp[0].minor.yy602=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 178: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); + } + yylhsminor.yy602 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); +} + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 179: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); + } + yylhsminor.yy602 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); +} + yymsp[-4].minor.yy602 = yylhsminor.yy602; + break; + case 180: /* term ::= NULL|FLOAT|BLOB */ + case 181: /* term ::= STRING */ yytestcase(yyruleno==181); +{yymsp[0].minor.yy602=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 182: /* term ::= INTEGER */ +{ + yylhsminor.yy602 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); +} + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 183: /* expr ::= VARIABLE */ +{ + if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ + u32 n = yymsp[0].minor.yy0.n; + yymsp[0].minor.yy602 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy602, n); + }else{ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ + assert( t.n>=2 ); + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + yymsp[0].minor.yy602 = 0; + }else{ + yymsp[0].minor.yy602 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy602 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy602->iTable); + } + } +} + break; + case 184: /* expr ::= expr COLLATE ID|STRING */ +{ + yymsp[-2].minor.yy602 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy602, &yymsp[0].minor.yy0, 1); +} + break; + case 185: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yymsp[-5].minor.yy602 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy602, yymsp[-3].minor.yy602, 0); +} + break; + case 186: /* expr ::= ID|INDEXED LP distinct exprlist RP */ +{ + yylhsminor.yy602 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy338, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy60); +} + yymsp[-4].minor.yy602 = yylhsminor.yy602; + break; + case 187: /* expr ::= ID|INDEXED LP STAR RP */ +{ + yylhsminor.yy602 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); +} + yymsp[-3].minor.yy602 = yylhsminor.yy602; + break; + case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ +{ + yylhsminor.yy602 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy338, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy60); + sqlite3WindowAttach(pParse, yylhsminor.yy602, yymsp[0].minor.yy19); +} + yymsp[-5].minor.yy602 = yylhsminor.yy602; + break; + case 189: /* expr ::= ID|INDEXED LP STAR RP filter_over */ +{ + yylhsminor.yy602 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy602, yymsp[0].minor.yy19); +} + yymsp[-4].minor.yy602 = yylhsminor.yy602; + break; + case 190: /* term ::= CTIME_KW */ +{ + yylhsminor.yy602 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); +} + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 191: /* expr ::= LP nexprlist COMMA expr RP */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy338, yymsp[-1].minor.yy602); + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy602 ){ + yymsp[-4].minor.yy602->x.pList = pList; + if( ALWAYS(pList->nExpr) ){ + yymsp[-4].minor.yy602->flags |= pList->a[0].pExpr->flags & EP_Propagate; + } + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + break; + case 192: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy602=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy602,yymsp[0].minor.yy602);} + break; + case 193: /* expr ::= expr OR expr */ + case 194: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==194); + case 195: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==195); + case 196: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==196); + case 197: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==197); + case 198: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==198); + case 199: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==199); +{yymsp[-2].minor.yy602=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy602,yymsp[0].minor.yy602);} + break; + case 200: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + break; + case 201: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + int bNot = yymsp[-1].minor.yy0.n & 0x80000000; + yymsp[-1].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy602); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy602); + yymsp[-2].minor.yy602 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy602, 0); + if( yymsp[-2].minor.yy602 ) yymsp[-2].minor.yy602->flags |= EP_InfixFunc; +} + break; + case 202: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + int bNot = yymsp[-3].minor.yy0.n & 0x80000000; + yymsp[-3].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy602); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy602); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy602); + yymsp[-4].minor.yy602 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); + if( yymsp[-4].minor.yy602 ) yymsp[-4].minor.yy602->flags |= EP_InfixFunc; +} + break; + case 203: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy602 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy602,0);} + break; + case 204: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy602 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy602,0);} + break; + case 205: /* expr ::= expr IS expr */ +{ + yymsp[-2].minor.yy602 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy602,yymsp[0].minor.yy602); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy602, yymsp[-2].minor.yy602, TK_ISNULL); +} + break; + case 206: /* expr ::= expr IS NOT expr */ +{ + yymsp[-3].minor.yy602 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy602,yymsp[0].minor.yy602); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy602, yymsp[-3].minor.yy602, TK_NOTNULL); +} + break; + case 207: /* expr ::= NOT expr */ + case 208: /* expr ::= BITNOT expr */ yytestcase(yyruleno==208); +{yymsp[-1].minor.yy602 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy602, 0);/*A-overwrites-B*/} + break; + case 209: /* expr ::= PLUS|MINUS expr */ +{ + yymsp[-1].minor.yy602 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy602, 0); + /*A-overwrites-B*/ +} + break; + case 210: /* between_op ::= BETWEEN */ + case 213: /* in_op ::= IN */ yytestcase(yyruleno==213); +{yymsp[0].minor.yy60 = 0;} + break; + case 212: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy602); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy602); + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy602, 0); + if( yymsp[-4].minor.yy602 ){ + yymsp[-4].minor.yy602->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy60 ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); +} + break; + case 215: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy338==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy602); + yymsp[-4].minor.yy602 = sqlite3Expr(pParse->db, TK_INTEGER, yymsp[-3].minor.yy60 ? "1" : "0"); + }else if( yymsp[-1].minor.yy338->nExpr==1 && sqlite3ExprIsConstant(yymsp[-1].minor.yy338->a[0].pExpr) ){ + Expr *pRHS = yymsp[-1].minor.yy338->a[0].pExpr; + yymsp[-1].minor.yy338->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy338); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy602, pRHS); + if( yymsp[-3].minor.yy60 ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); + }else{ + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy602, 0); + if( yymsp[-4].minor.yy602 ){ + yymsp[-4].minor.yy602->x.pList = yymsp[-1].minor.yy338; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy602); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy338); + } + if( yymsp[-3].minor.yy60 ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); + } + } + break; + case 216: /* expr ::= LP select RP */ +{ + yymsp[-2].minor.yy602 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy602, yymsp[-1].minor.yy307); + } + break; + case 217: /* expr ::= expr in_op LP select RP */ +{ + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy602, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy602, yymsp[-1].minor.yy307); + if( yymsp[-3].minor.yy60 ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); + } + break; + case 218: /* expr ::= expr in_op nm dbnm paren_exprlist */ +{ + SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); + if( yymsp[0].minor.yy338 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy338); + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy602, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy602, pSelect); + if( yymsp[-3].minor.yy60 ) yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy602, 0); + } + break; + case 219: /* expr ::= EXISTS LP select RP */ +{ + Expr *p; + p = yymsp[-3].minor.yy602 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy307); + } + break; + case 220: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yymsp[-4].minor.yy602 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy602, 0); + if( yymsp[-4].minor.yy602 ){ + yymsp[-4].minor.yy602->x.pList = yymsp[-1].minor.yy602 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy338,yymsp[-1].minor.yy602) : yymsp[-2].minor.yy338; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy602); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy338); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy602); + } +} + break; + case 221: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy338, yymsp[-2].minor.yy602); + yymsp[-4].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy338, yymsp[0].minor.yy602); +} + break; + case 222: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yymsp[-3].minor.yy338 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy602); + yymsp[-3].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy338, yymsp[0].minor.yy602); +} + break; + case 225: /* case_operand ::= expr */ +{yymsp[0].minor.yy602 = yymsp[0].minor.yy602; /*A-overwrites-X*/} + break; + case 228: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy338 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy338,yymsp[0].minor.yy602);} + break; + case 229: /* nexprlist ::= expr */ +{yymsp[0].minor.yy338 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy602); /*A-overwrites-Y*/} + break; + case 231: /* paren_exprlist ::= LP exprlist RP */ + case 236: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==236); +{yymsp[-2].minor.yy338 = yymsp[-1].minor.yy338;} + break; + case 232: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ +{ + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy338, yymsp[-10].minor.yy60, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy602, SQLITE_SO_ASC, yymsp[-8].minor.yy60, SQLITE_IDXTYPE_APPDEF); + if( IN_RENAME_OBJECT && pParse->pNewIndex ){ + sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); + } +} + break; + case 233: /* uniqueflag ::= UNIQUE */ + case 275: /* raisetype ::= ABORT */ yytestcase(yyruleno==275); +{yymsp[0].minor.yy60 = OE_Abort;} + break; + case 234: /* uniqueflag ::= */ +{yymsp[1].minor.yy60 = OE_None;} + break; + case 237: /* eidlist ::= eidlist COMMA nm collate sortorder */ +{ + yymsp[-4].minor.yy338 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy338, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy60, yymsp[0].minor.yy60); +} + break; + case 238: /* eidlist ::= nm collate sortorder */ +{ + yymsp[-2].minor.yy338 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy60, yymsp[0].minor.yy60); /*A-overwrites-Y*/ +} + break; + case 241: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy291, yymsp[-1].minor.yy60);} + break; + case 242: /* cmd ::= VACUUM vinto */ +#ifdef SQLITE_OMIT_VACUUM + goto SqliteOmitVacuum; +#else +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy602);} +#endif + break; + case 243: /* cmd ::= VACUUM nm vinto */ +#ifdef SQLITE_OMIT_VACUUM + goto SqliteOmitVacuum; +#else +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy602);} +#endif + break; + case 246: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} + break; + case 247: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} + break; + case 248: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} + break; + case 249: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} + break; + case 250: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} + break; + case 253: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy483, &all); +} + break; + case 254: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy60, yymsp[-4].minor.yy50.a, yymsp[-4].minor.yy50.b, yymsp[-2].minor.yy291, yymsp[0].minor.yy602, yymsp[-10].minor.yy60, yymsp[-8].minor.yy60); + yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ +} + break; + case 255: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy60 = yymsp[0].major; /*A-overwrites-X*/ } + break; + case 256: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy60 = TK_INSTEAD;} + break; + case 257: /* trigger_time ::= */ +{ yymsp[1].minor.yy60 = TK_BEFORE; } + break; + case 258: /* trigger_event ::= DELETE|INSERT */ + case 259: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==259); +{yymsp[0].minor.yy50.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy50.b = 0;} + break; + case 260: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy50.a = TK_UPDATE; yymsp[-2].minor.yy50.b = yymsp[0].minor.yy288;} + break; + case 261: /* when_clause ::= */ + case 280: /* key_opt ::= */ yytestcase(yyruleno==280); +{ yymsp[1].minor.yy602 = 0; } + break; + case 262: /* when_clause ::= WHEN expr */ + case 281: /* key_opt ::= KEY expr */ yytestcase(yyruleno==281); +{ yymsp[-1].minor.yy602 = yymsp[0].minor.yy602; } + break; + case 263: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy483!=0 ); + yymsp[-2].minor.yy483->pLast->pNext = yymsp[-1].minor.yy483; + yymsp[-2].minor.yy483->pLast = yymsp[-1].minor.yy483; +} + break; + case 264: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy483!=0 ); + yymsp[-1].minor.yy483->pLast = yymsp[-1].minor.yy483; +} + break; + case 265: /* trnm ::= nm DOT nm */ +{ + yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + break; + case 266: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 267: /* tridxby ::= NOT INDEXED */ +{ + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 268: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy483 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy291, yymsp[-3].minor.yy338, yymsp[-1].minor.yy602, yymsp[-7].minor.yy60, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy528);} + yymsp[-8].minor.yy483 = yylhsminor.yy483; + break; + case 269: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ +{ + yylhsminor.yy483 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy288,yymsp[-2].minor.yy307,yymsp[-6].minor.yy60,yymsp[-1].minor.yy178,yymsp[-7].minor.yy528,yymsp[0].minor.yy528);/*yylhsminor.yy483-overwrites-yymsp[-6].minor.yy60*/ +} + yymsp[-7].minor.yy483 = yylhsminor.yy483; + break; + case 270: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy483 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy602, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy528);} + yymsp[-5].minor.yy483 = yylhsminor.yy483; + break; + case 271: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy483 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy307, yymsp[-2].minor.yy528, yymsp[0].minor.yy528); /*yylhsminor.yy483-overwrites-yymsp[-1].minor.yy307*/} + yymsp[-2].minor.yy483 = yylhsminor.yy483; + break; + case 272: /* expr ::= RAISE LP IGNORE RP */ +{ + yymsp[-3].minor.yy602 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy602 ){ + yymsp[-3].minor.yy602->affExpr = OE_Ignore; + } +} + break; + case 273: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yymsp[-5].minor.yy602 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy602 ) { + yymsp[-5].minor.yy602->affExpr = (char)yymsp[-3].minor.yy60; + } +} + break; + case 274: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy60 = OE_Rollback;} + break; + case 276: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy60 = OE_Fail;} + break; + case 277: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy291,yymsp[-1].minor.yy60); +} + break; + case 278: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy602, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); +} + break; + case 279: /* cmd ::= DETACH database_kw_opt expr */ +{ + sqlite3Detach(pParse, yymsp[0].minor.yy602); +} + break; + case 282: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} + break; + case 283: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 284: /* cmd ::= ANALYZE */ +#ifdef SQLITE_OMIT_ANALYZE + goto SqliteOmitAnalyze; +#else +{sqlite3Analyze(pParse, 0, 0);} +#endif + break; + case 285: /* cmd ::= ANALYZE nm dbnm */ +#ifdef SQLITE_OMIT_ANALYZE + goto SqliteOmitAnalyze; +#else +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} +#endif + break; + case 286: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +#ifdef SQLITE_OMIT_ALTERTABLE + goto SqliteOmitAltertable; +#else + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy291,&yymsp[0].minor.yy0); +#endif + break; + case 287: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ +#ifdef SQLITE_OMIT_ALTERTABLE + goto SqliteOmitAltertable; +#else + yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; + sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); +#endif + break; + case 288: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ +#ifdef SQLITE_OMIT_ALTERTABLE + goto SqliteOmitAltertable; +#else + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy291, &yymsp[0].minor.yy0); +#endif + break; + case 289: /* add_column_fullname ::= fullname */ +#ifdef SQLITE_OMIT_ALTERTABLE + goto SqliteOmitAltertable; +#else + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy291); +#endif + break; + case 290: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ +#ifdef SQLITE_OMIT_ALTERTABLE + goto SqliteOmitAltertable; +#else + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy291, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +#endif + break; + case 291: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} + break; + case 292: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} + break; + case 293: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy60); +} + break; + case 294: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} + break; + case 295: /* vtabargtoken ::= ANY */ + case 296: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==296); + case 297: /* lp ::= LP */ yytestcase(yyruleno==297); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} + break; + case 298: /* with ::= WITH wqlist */ + case 299: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==299); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy195, 1); } + break; + case 300: /* wqas ::= AS */ +{yymsp[0].minor.yy570 = M10d_Any;} + break; + case 301: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy570 = M10d_Yes;} + break; + case 302: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy570 = M10d_No;} + break; + case 303: /* wqitem ::= nm eidlist_opt wqas LP select RP */ +{ + yymsp[-5].minor.yy607 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy338, yymsp[-1].minor.yy307, yymsp[-3].minor.yy570); /*A-overwrites-X*/ +} + break; + case 304: /* wqlist ::= wqitem */ +{ + yymsp[0].minor.yy195 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy607); /*A-overwrites-X*/ +} + break; + case 305: /* wqlist ::= wqlist COMMA wqitem */ +{ + yymsp[-2].minor.yy195 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy195, yymsp[0].minor.yy607); +} + break; + case 306: /* windowdefn_list ::= windowdefn */ +{ yylhsminor.yy19 = yymsp[0].minor.yy19; } + yymsp[0].minor.yy19 = yylhsminor.yy19; + break; + case 307: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + assert( yymsp[0].minor.yy19!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy19, yymsp[-2].minor.yy19); + yymsp[0].minor.yy19->pNextWin = yymsp[-2].minor.yy19; + yylhsminor.yy19 = yymsp[0].minor.yy19; +} + yymsp[-2].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 308: /* windowdefn ::= nm AS LP window RP */ +{ + if( ALWAYS(yymsp[-1].minor.yy19) ){ + yymsp[-1].minor.yy19->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + } + yylhsminor.yy19 = yymsp[-1].minor.yy19; +} + yymsp[-4].minor.yy19 = yylhsminor.yy19; + break; + case 309: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yymsp[-4].minor.yy19 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy19, yymsp[-2].minor.yy338, yymsp[-1].minor.yy338, 0); +} +#endif + break; + case 310: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yylhsminor.yy19 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy19, yymsp[-2].minor.yy338, yymsp[-1].minor.yy338, &yymsp[-5].minor.yy0); +} + yymsp[-5].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 311: /* window ::= ORDER BY sortlist frame_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yymsp[-3].minor.yy19 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy19, 0, yymsp[-1].minor.yy338, 0); +} +#endif + break; + case 312: /* window ::= nm ORDER BY sortlist frame_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yylhsminor.yy19 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy19, 0, yymsp[-1].minor.yy338, &yymsp[-4].minor.yy0); +} + yymsp[-4].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 313: /* window ::= frame_opt */ + case 332: /* filter_over ::= over_clause */ yytestcase(yyruleno==332); +{ + yylhsminor.yy19 = yymsp[0].minor.yy19; +} + yymsp[0].minor.yy19 = yylhsminor.yy19; + break; + case 314: /* window ::= nm frame_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yylhsminor.yy19 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy19, 0, 0, &yymsp[-1].minor.yy0); +} + yymsp[-1].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 315: /* frame_opt ::= */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yymsp[1].minor.yy19 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); +} +#endif + break; + case 316: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yylhsminor.yy19 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy60, yymsp[-1].minor.yy113.eType, yymsp[-1].minor.yy113.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy570); +} + yymsp[-2].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 317: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ +#ifdef SQLITE_OMIT_WINDOWFUNC + goto SqliteOmitWindowfunc; +#else +{ + yylhsminor.yy19 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy60, yymsp[-3].minor.yy113.eType, yymsp[-3].minor.yy113.pExpr, yymsp[-1].minor.yy113.eType, yymsp[-1].minor.yy113.pExpr, yymsp[0].minor.yy570); +} + yymsp[-5].minor.yy19 = yylhsminor.yy19; +#endif + break; + case 319: /* frame_bound_s ::= frame_bound */ + case 321: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==321); +{yylhsminor.yy113 = yymsp[0].minor.yy113;} + yymsp[0].minor.yy113 = yylhsminor.yy113; + break; + case 320: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 322: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==322); + case 324: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==324); +{yylhsminor.yy113.eType = yymsp[-1].major; yylhsminor.yy113.pExpr = 0;} + yymsp[-1].minor.yy113 = yylhsminor.yy113; + break; + case 323: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy113.eType = yymsp[0].major; yylhsminor.yy113.pExpr = yymsp[-1].minor.yy602;} + yymsp[-1].minor.yy113 = yylhsminor.yy113; + break; + case 325: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy570 = 0;} + break; + case 326: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy570 = yymsp[0].minor.yy570;} + break; + case 327: /* frame_exclude ::= NO OTHERS */ + case 328: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==328); +{yymsp[-1].minor.yy570 = yymsp[-1].major; /*A-overwrites-X*/} + break; + case 329: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy570 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 330: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy19 = yymsp[0].minor.yy19; } + break; + case 331: /* filter_over ::= filter_clause over_clause */ +{ + yymsp[0].minor.yy19->pFilter = yymsp[-1].minor.yy602; + yylhsminor.yy19 = yymsp[0].minor.yy19; +} + yymsp[-1].minor.yy19 = yylhsminor.yy19; + break; + case 333: /* filter_over ::= filter_clause */ +{ + yylhsminor.yy19 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy19 ){ + yylhsminor.yy19->eFrmType = TK_FILTER; + yylhsminor.yy19->pFilter = yymsp[0].minor.yy602; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy602); + } +} + yymsp[0].minor.yy19 = yylhsminor.yy19; + break; + case 334: /* over_clause ::= OVER LP window RP */ +{ + yymsp[-3].minor.yy19 = yymsp[-1].minor.yy19; + assert( yymsp[-3].minor.yy19!=0 ); +} + break; + case 335: /* over_clause ::= OVER nm */ +{ + yymsp[-1].minor.yy19 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy19 ){ + yymsp[-1].minor.yy19->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + } +} + break; + case 336: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy602 = yymsp[-1].minor.yy602; } + break; + default: + /* (337) input ::= cmdlist */ yytestcase(yyruleno==337); + /* (338) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==338); + /* (339) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=339); + /* (340) ecmd ::= SEMI */ yytestcase(yyruleno==340); + /* (341) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==341); + /* (342) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=342); + /* (343) trans_opt ::= */ yytestcase(yyruleno==343); + /* (344) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==344); + /* (345) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==345); + /* (346) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==346); + /* (347) savepoint_opt ::= */ yytestcase(yyruleno==347); + /* (348) cmd ::= create_table create_table_args */ yytestcase(yyruleno==348); + /* (349) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==349); + /* (350) columnlist ::= columnname carglist */ yytestcase(yyruleno==350); + /* (351) nm ::= ID|INDEXED */ yytestcase(yyruleno==351); + /* (352) nm ::= STRING */ yytestcase(yyruleno==352); + /* (353) nm ::= JOIN_KW */ yytestcase(yyruleno==353); + /* (354) typetoken ::= typename */ yytestcase(yyruleno==354); + /* (355) typename ::= ID|STRING */ yytestcase(yyruleno==355); + /* (356) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=357); + /* (358) carglist ::= carglist ccons */ yytestcase(yyruleno==358); + /* (359) carglist ::= */ yytestcase(yyruleno==359); + /* (360) ccons ::= NULL onconf */ yytestcase(yyruleno==360); + /* (361) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==361); + /* (362) ccons ::= AS generated */ yytestcase(yyruleno==362); + /* (363) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==363); + /* (364) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==364); + /* (365) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=365); + /* (366) tconscomma ::= */ yytestcase(yyruleno==366); + /* (367) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=367); + /* (368) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=368); + /* (369) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=369); + /* (370) oneselect ::= values */ yytestcase(yyruleno==370); + /* (371) sclp ::= selcollist COMMA */ yytestcase(yyruleno==371); + /* (372) as ::= ID|STRING */ yytestcase(yyruleno==372); + /* (373) returning ::= */ yytestcase(yyruleno==373); + /* (374) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==375); + /* (376) exprlist ::= nexprlist */ yytestcase(yyruleno==376); + /* (377) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=377); + /* (378) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=378); + /* (379) nmnum ::= ON */ yytestcase(yyruleno==379); + /* (380) nmnum ::= DELETE */ yytestcase(yyruleno==380); + /* (381) nmnum ::= DEFAULT */ yytestcase(yyruleno==381); + /* (382) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==382); + /* (383) foreach_clause ::= */ yytestcase(yyruleno==383); + /* (384) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==384); + /* (385) trnm ::= nm */ yytestcase(yyruleno==385); + /* (386) tridxby ::= */ yytestcase(yyruleno==386); + /* (387) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==387); + /* (388) database_kw_opt ::= */ yytestcase(yyruleno==388); + /* (389) kwcolumn_opt ::= */ yytestcase(yyruleno==389); + /* (390) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==390); + /* (391) vtabarglist ::= vtabarg */ yytestcase(yyruleno==391); + /* (392) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==392); + /* (393) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==393); + /* (394) anylist ::= */ yytestcase(yyruleno==394); + /* (395) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==395); + /* (396) anylist ::= anylist ANY */ yytestcase(yyruleno==396); + /* (397) with ::= */ yytestcase(yyruleno==397); + break; +/********** End reduce actions ************************************************/ +#ifdef SQLITE_OMIT_ANALYZE + SqliteOmitAnalyze: + sqlite3ErrorMsg(pParse, "ANALYZE not included in production builds"); + break; +#endif +#ifdef SQLITE_OMIT_ALTERTABLE + SqliteOmitAltertable: + sqlite3ErrorMsg(pParse, "ALTER not included in production builds"); + break; +#endif +#ifdef SQLITE_OMIT_WINDOWFUNC + SqliteOmitWindowfunc: + sqlite3ErrorMsg(pParse, "window functions not included in production builds"); + break; +#endif +#ifdef SQLITE_OMIT_VACUUM + SqliteOmitVacuum: + sqlite3ErrorMsg(pParse, "VACUUM not included in production builds"); + break; +#endif + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ + + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + if( TOKEN.z[0] ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + }else{ + sqlite3ErrorMsg(pParse, "incomplete input"); + } +/************ End %syntax_error code ******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + sqlite3ParserCTX_FETCH + sqlite3ParserARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); +#ifndef NDEBUG + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + break; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } +#endif + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos >= yypParser->yystack + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + } +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int sqlite3ParserFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} diff --git a/third_party/sqlite3/parse.inc b/third_party/sqlite3/parse.inc new file mode 100644 index 000000000..ba9679cf5 --- /dev/null +++ b/third_party/sqlite3/parse.inc @@ -0,0 +1,182 @@ +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_WITHOUT 25 +#define TK_COMMA 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_MATCH 46 +#define TK_LIKE_KW 47 +#define TK_BETWEEN 48 +#define TK_IN 49 +#define TK_ISNULL 50 +#define TK_NOTNULL 51 +#define TK_NE 52 +#define TK_EQ 53 +#define TK_GT 54 +#define TK_LE 55 +#define TK_LT 56 +#define TK_GE 57 +#define TK_ESCAPE 58 +#define TK_ID 59 +#define TK_COLUMNKW 60 +#define TK_DO 61 +#define TK_FOR 62 +#define TK_IGNORE 63 +#define TK_INITIALLY 64 +#define TK_INSTEAD 65 +#define TK_NO 66 +#define TK_KEY 67 +#define TK_OF 68 +#define TK_OFFSET 69 +#define TK_PRAGMA 70 +#define TK_RAISE 71 +#define TK_RECURSIVE 72 +#define TK_REPLACE 73 +#define TK_RESTRICT 74 +#define TK_ROW 75 +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_GENERATED 95 +#define TK_ALWAYS 96 +#define TK_MATERIALIZED 97 +#define TK_REINDEX 98 +#define TK_RENAME 99 +#define TK_CTIME_KW 100 +#define TK_ANY 101 +#define TK_BITAND 102 +#define TK_BITOR 103 +#define TK_LSHIFT 104 +#define TK_RSHIFT 105 +#define TK_PLUS 106 +#define TK_MINUS 107 +#define TK_STAR 108 +#define TK_SLASH 109 +#define TK_REM 110 +#define TK_CONCAT 111 +#define TK_COLLATE 112 +#define TK_BITNOT 113 +#define TK_ON 114 +#define TK_INDEXED 115 +#define TK_STRING 116 +#define TK_JOIN_KW 117 +#define TK_CONSTRAINT 118 +#define TK_DEFAULT 119 +#define TK_NULL 120 +#define TK_PRIMARY 121 +#define TK_UNIQUE 122 +#define TK_CHECK 123 +#define TK_REFERENCES 124 +#define TK_AUTOINCR 125 +#define TK_INSERT 126 +#define TK_DELETE 127 +#define TK_UPDATE 128 +#define TK_SET 129 +#define TK_DEFERRABLE 130 +#define TK_FOREIGN 131 +#define TK_DROP 132 +#define TK_UNION 133 +#define TK_ALL 134 +#define TK_EXCEPT 135 +#define TK_INTERSECT 136 +#define TK_SELECT 137 +#define TK_VALUES 138 +#define TK_DISTINCT 139 +#define TK_DOT 140 +#define TK_FROM 141 +#define TK_JOIN 142 +#define TK_USING 143 +#define TK_ORDER 144 +#define TK_GROUP 145 +#define TK_HAVING 146 +#define TK_LIMIT 147 +#define TK_WHERE 148 +#define TK_RETURNING 149 +#define TK_INTO 150 +#define TK_NOTHING 151 +#define TK_FLOAT 152 +#define TK_BLOB 153 +#define TK_INTEGER 154 +#define TK_VARIABLE 155 +#define TK_CASE 156 +#define TK_WHEN 157 +#define TK_THEN 158 +#define TK_ELSE 159 +#define TK_INDEX 160 +#define TK_ALTER 161 +#define TK_ADD 162 +#define TK_WINDOW 163 +#define TK_OVER 164 +#define TK_FILTER 165 +#define TK_COLUMN 166 +#define TK_AGG_FUNCTION 167 +#define TK_AGG_COLUMN 168 +#define TK_TRUEFALSE 169 +#define TK_ISNOT 170 +#define TK_FUNCTION 171 +#define TK_UMINUS 172 +#define TK_UPLUS 173 +#define TK_TRUTH 174 +#define TK_REGISTER 175 +#define TK_VECTOR 176 +#define TK_SELECT_COLUMN 177 +#define TK_IF_NULL_ROW 178 +#define TK_ASTERISK 179 +#define TK_SPAN 180 +#define TK_SPACE 181 +#define TK_ILLEGAL 182 diff --git a/third_party/sqlite3/pcache.c b/third_party/sqlite3/pcache.c new file mode 100644 index 000000000..00f601979 --- /dev/null +++ b/third_party/sqlite3/pcache.c @@ -0,0 +1,883 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements that page cache. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** A complete page cache is an instance of this structure. Every +** entry in the cache holds a single page of the database file. The +** btree layer only operates on the cached copy of the database pages. +** +** A page cache entry is "clean" if it exactly matches what is currently +** on disk. A page is "dirty" if it has been modified and needs to be +** persisted to disk. +** +** pDirty, pDirtyTail, pSynced: +** All dirty pages are linked into the doubly linked list using +** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order +** such that p was added to the list more recently than p->pDirtyNext. +** PCache.pDirty points to the first (newest) element in the list and +** pDirtyTail to the last (oldest). +** +** The PCache.pSynced variable is used to optimize searching for a dirty +** page to eject from the cache mid-transaction. It is better to eject +** a page that does not require a journal sync than one that does. +** Therefore, pSynced is maintained so that it *almost* always points +** to either the oldest page in the pDirty/pDirtyTail list that has a +** clear PGHDR_NEED_SYNC flag or to a page that is older than this one +** (so that the right page to eject can be found by following pDirtyPrev +** pointers). +*/ +struct PCache { + PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ + PgHdr *pSynced; /* Last synced page in dirty page list */ + int nRefSum; /* Sum of ref counts over all pages */ + int szCache; /* Configured cache size */ + int szSpill; /* Size before spilling occurs */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + u8 bPurgeable; /* True if pages are on backing store */ + u8 eCreate; /* eCreate value for for xFetch() */ + int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ + void *pStress; /* Argument to xStress */ + sqlite3_pcache *pCache; /* Pluggable cache module */ +}; + +/********************************** Test and Debug Logic **********************/ +/* +** Debug tracing macros. Enable by by changing the "0" to "1" and +** recompiling. +** +** When sqlite3PcacheTrace is 1, single line trace messages are issued. +** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries +** is displayed for many operations, resulting in a lot of output. +*/ +#if defined(SQLITE_DEBUG) && 0 + int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ + int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ +# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} + void pcacheDump(PCache *pCache){ + int N; + int i, j; + sqlite3_pcache_page *pLower; + PgHdr *pPg; + unsigned char *a; + + if( sqlite3PcacheTrace<2 ) return; + if( pCache->pCache==0 ) return; + N = sqlite3PcachePagecount(pCache); + if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; + for(i=1; i<=N; i++){ + pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); + if( pLower==0 ) continue; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf("\n"); + if( pPg->pPage==0 ){ + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); + } + } + } + #else +# define pcacheTrace(X) +# define pcacheDump(X) +#endif + +/* +** Check invariants on a PgHdr entry. Return true if everything is OK. +** Return false if any invariant is violated. +** +** This routine is for use inside of assert() statements only. For +** example: +** +** assert( sqlite3PcachePageSanity(pPg) ); +*/ +#ifdef SQLITE_DEBUG +int sqlite3PcachePageSanity(PgHdr *pPg){ + PCache *pCache; + assert( pPg!=0 ); + assert( pPg->pgno>0 || pPg->pPager==0 ); /* Page number is 1 or more */ + pCache = pPg->pCache; + assert( pCache!=0 ); /* Every page has an associated PCache */ + if( pPg->flags & PGHDR_CLEAN ){ + assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ + assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ + assert( pCache->pDirtyTail!=pPg ); + } + /* WRITEABLE pages must also be DIRTY */ + if( pPg->flags & PGHDR_WRITEABLE ){ + assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ + } + /* NEED_SYNC can be set independently of WRITEABLE. This can happen, + ** for example, when using the sqlite3PagerDontWrite() optimization: + ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. + ** (2) Page X moved to freelist, WRITEABLE is cleared + ** (3) Page X reused, WRITEABLE is set again + ** If NEED_SYNC had been cleared in step 2, then it would not be reset + ** in step 3, and page might be written into the database without first + ** syncing the rollback journal, which might cause corruption on a power + ** loss. + ** + ** Another example is when the database page size is smaller than the + ** disk sector size. When any page of a sector is journalled, all pages + ** in that sector are marked NEED_SYNC even if they are still CLEAN, just + ** in case they are later modified, since all pages in the same sector + ** must be journalled and synced before any of those pages can be safely + ** written. + */ + return 1; +} +#endif /* SQLITE_DEBUG */ + + +/********************************** Linked List Management ********************/ + +/* Allowed values for second argument to pcacheManageDirtyList() */ +#define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ +#define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ +#define PCACHE_DIRTYLIST_FRONT 3 /* Move pPage to the front of the list */ + +/* +** Manage pPage's participation on the dirty list. Bits of the addRemove +** argument determines what operation to do. The 0x01 bit means first +** remove pPage from the dirty list. The 0x02 means add pPage back to +** the dirty list. Doing both moves pPage to the front of the dirty list. +*/ +static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ + PCache *p = pPage->pCache; + + pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, + addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", + pPage->pgno)); + if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + p->pSynced = pPage->pDirtyPrev; + } + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + /* If there are now no dirty pages in the cache, set eCreate to 2. + ** This is an optimization that allows sqlite3PcacheFetch() to skip + ** searching for a dirty page to eject from the cache when it might + ** otherwise have to. */ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + assert( p->bPurgeable || p->eCreate==2 ); + if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ + assert( p->bPurgeable==0 || p->eCreate==1 ); + p->eCreate = 2; + } + } + } + if( addRemove & PCACHE_DIRTYLIST_ADD ){ + pPage->pDirtyPrev = 0; + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else{ + p->pDirtyTail = pPage; + if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + } + p->pDirty = pPage; + + /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set + ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** optimization, as if pSynced points to a page with the NEED_SYNC + ** flag set sqlite3PcacheFetchStress() searches through all newer + ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ + if( !p->pSynced + && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ + ){ + p->pSynced = pPage; + } + } + pcacheDump(p); +} + +/* +** Wrapper around the pluggable caches xUnpin method. If the cache is +** being used for an in-memory database, this function is a no-op. +*/ +static void pcacheUnpin(PgHdr *p){ + if( p->pCache->bPurgeable ){ + pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + pcacheDump(p->pCache); + } +} + +/* +** Compute the number of pages of cache requested. p->szCache is the +** cache size requested by the "PRAGMA cache_size" statement. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + /* IMPLEMENTATION-OF: R-42059-47211 If the argument N is positive then the + ** suggested cache size is set to N. */ + return p->szCache; + }else{ + /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + } +} + +/*************************************************** General Interfaces ****** +** +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. +*/ +int sqlite3PcacheInitialize(void){ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ + sqlite3PCacheSetDefault(); + assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); + } + return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); +} +void sqlite3PcacheShutdown(void){ + if( sqlite3GlobalConfig.pcache2.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ + sqlite3GlobalConfig.pcache2.xShutdown(sqlite3GlobalConfig.pcache2.pArg); + } +} + +/* +** Return the size in bytes of a PCache object. +*/ +int sqlite3PcacheSize(void){ return sizeof(PCache); } + +/* +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by +** calling sqlite3PcacheSize(). +** +** szExtra is some extra space allocated for each page. The first +** 8 bytes of the extra space will be zeroed as the page is allocated, +** but remaining content will be uninitialized. Though it is opaque +** to this module, the extra space really ends up being the MemPage +** structure in the pager. +*/ +int sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + memset(p, 0, sizeof(PCache)); + p->szPage = 1; + p->szExtra = szExtra; + assert( szExtra>=8 ); /* First 8 bytes will be zeroed */ + p->bPurgeable = bPurgeable; + p->eCreate = 2; + p->xStress = xStress; + p->pStress = pStress; + p->szCache = 100; + p->szSpill = 1; + pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); + return sqlite3PcacheSetPageSize(p, szPage); +} + +/* +** Change the page size for PCache object. The caller must ensure that there +** are no outstanding page references when this function is called. +*/ +int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); + if( pCache->szPage ){ + sqlite3_pcache *pNew; + pNew = sqlite3GlobalConfig.pcache2.xCreate( + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable + ); + if( pNew==0 ) return SQLITE_NOMEM_BKPT; + sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } + pCache->pCache = pNew; + pCache->szPage = szPage; + pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); + } + return SQLITE_OK; +} + +/* +** Try to obtain a page from the cache. +** +** This routine returns a pointer to an sqlite3_pcache_page object if +** such an object is already in cache, or if a new one is created. +** This routine returns a NULL pointer if the object was not in cache +** and could not be created. +** +** The createFlags should be 0 to check for existing pages and should +** be 3 (not 1, but 3) to try to create a new page. +** +** If the createFlag is 0, then NULL is always returned if the page +** is not already in the cache. If createFlag is 1, then a new page +** is created only if that can be done without spilling dirty pages +** and without exceeding the cache size limit. +** +** The caller needs to invoke sqlite3PcacheFetchFinish() to properly +** initialize the sqlite3_pcache_page object and convert it into a +** PgHdr object. The sqlite3PcacheFetch() and sqlite3PcacheFetchFinish() +** routines are split this way for performance reasons. When separated +** they can both (usually) operate without having to push values to +** the stack on entry and pop them back off on exit, which saves a +** lot of pushing and popping. +*/ +sqlite3_pcache_page *sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag /* If true, create page if it does not exist already */ +){ + int eCreate; + sqlite3_pcache_page *pRes; + + assert( pCache!=0 ); + assert( pCache->pCache!=0 ); + assert( createFlag==3 || createFlag==0 ); + assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); + + /* eCreate defines what to do if the page does not exist. + ** 0 Do not allocate a new page. (createFlag==0) + ** 1 Allocate a new page if doing so is inexpensive. + ** (createFlag==1 AND bPurgeable AND pDirty) + ** 2 Allocate a new page even it doing so is difficult. + ** (createFlag==1 AND !(bPurgeable AND pDirty) + */ + eCreate = createFlag & pCache->eCreate; + assert( eCreate==0 || eCreate==1 || eCreate==2 ); + assert( createFlag==0 || pCache->eCreate==eCreate ); + assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); + pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + createFlag?" create":"",pRes)); + return pRes; +} + +/* +** If the sqlite3PcacheFetch() routine is unable to allocate a new +** page because no clean pages are available for reuse and the cache +** size limit has been reached, then this routine can be invoked to +** try harder to allocate a page. This routine might invoke the stress +** callback to spill dirty pages to the journal. It will then try to +** allocate the new page and will only fail to allocate a new page on +** an OOM error. +** +** This routine should be invoked only after sqlite3PcacheFetch() fails. +*/ +int sqlite3PcacheFetchStress( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + sqlite3_pcache_page **ppPage /* Write result here */ +){ + PgHdr *pPg; + if( pCache->eCreate==2 ) return 0; + + if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + ** + ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC + ** flag is currently referenced, then the following may leave pSynced + ** set incorrectly (pointing to other than the LRU page with NEED_SYNC + ** cleared). This is Ok, as pSynced is just an optimization. */ + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache), + numberOfCachePages(pCache)); +#endif + pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); + rc = pCache->xStress(pCache->pStress, pPg); + pcacheDump(pCache); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; + } + } + } + *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; +} + +/* +** This is a helper routine for sqlite3PcacheFetchFinish() +** +** In the uncommon case where the page being fetched has not been +** initialized, this routine is invoked to do the initialization. +** This routine is broken out into a separate function since it +** requires extra stack manipulation that can be avoided in the common +** case. +*/ +static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + assert( pPage!=0 ); + pPgHdr = (PgHdr*)pPage->pExtra; + assert( pPgHdr->pPage==0 ); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, 8); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + pPgHdr->flags = PGHDR_CLEAN; + return sqlite3PcacheFetchFinish(pCache,pgno,pPage); +} + +/* +** This routine converts the sqlite3_pcache_page object returned by +** sqlite3PcacheFetch() into an initialized PgHdr object. This routine +** must be called after sqlite3PcacheFetch() in order to get a usable +** result. +*/ +PgHdr *sqlite3PcacheFetchFinish( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + + assert( pPage!=0 ); + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + return pcacheFetchFinishWithInit(pCache, pgno, pPage); + } + pCache->nRefSum++; + pPgHdr->nRef++; + assert( sqlite3PcachePageSanity(pPgHdr) ); + return pPgHdr; +} + +/* +** Decrement the reference count on a page. If the page is clean and the +** reference count drops to 0, then it is made eligible for recycling. +*/ +void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ + if( p->flags&PGHDR_CLEAN ){ + pcacheUnpin(p); + }else{ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + } + } +} + +/* +** Increase the reference count of a supplied page by 1. +*/ +void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>0); + assert( sqlite3PcachePageSanity(p) ); + p->nRef++; + p->pCache->nRefSum++; +} + +/* +** Drop a page from the cache. There must be exactly one reference to the +** page. This function deletes that reference, so after it returns the +** page pointed to by p is invalid. +*/ +void sqlite3PcacheDrop(PgHdr *p){ + assert( p->nRef==1 ); + assert( sqlite3PcachePageSanity(p) ); + if( p->flags&PGHDR_DIRTY ){ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + } + p->pCache->nRefSum--; + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); +} + +/* +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. +*/ +void sqlite3PcacheMakeDirty(PgHdr *p){ + assert( p->nRef>0 ); + assert( sqlite3PcachePageSanity(p) ); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ + p->flags &= ~PGHDR_DONT_WRITE; + if( p->flags & PGHDR_CLEAN ){ + p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); + assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + } + assert( sqlite3PcachePageSanity(p) ); + } +} + +/* +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. +*/ +void sqlite3PcacheMakeClean(PgHdr *p){ + assert( sqlite3PcachePageSanity(p) ); + assert( (p->flags & PGHDR_DIRTY)!=0 ); + assert( (p->flags & PGHDR_CLEAN)==0 ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); + if( p->nRef==0 ){ + pcacheUnpin(p); + } +} + +/* +** Make every page in the cache clean. +*/ +void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAN-ALL\n",pCache)); + while( (p = pCache->pDirty)!=0 ){ + sqlite3PcacheMakeClean(p); + } +} + +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Clear the PGHDR_NEED_SYNC flag from all dirty pages. +*/ +void sqlite3PcacheClearSyncFlags(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~PGHDR_NEED_SYNC; + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Change the page number of page p to newPgno. +*/ +void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + PCache *pCache = p->pCache; + assert( p->nRef>0 ); + assert( newPgno>0 ); + assert( sqlite3PcachePageSanity(p) ); + pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); + p->pgno = newPgno; + if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + } +} + +/* +** Drop every cache entry whose page number is greater than "pgno". The +** caller must ensure that there are no outstanding references to any pages +** other than page 1 with a page number greater than pgno. +** +** If there is a reference to page 1 and the pgno parameter passed to this +** function is 0, then the data area associated with page 1 is zeroed, but +** the page object is not dropped. +*/ +void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + if( pCache->pCache ){ + PgHdr *p; + PgHdr *pNext; + pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pDirtyNext; + /* This routine never gets call with a positive pgno except right + ** after sqlite3PcacheCleanAll(). So if there are dirty pages, + ** it must be that pgno==0. + */ + assert( p->pgno>0 ); + if( p->pgno>pgno ){ + assert( p->flags&PGHDR_DIRTY ); + sqlite3PcacheMakeClean(p); + } + } + if( pgno==0 && pCache->nRefSum ){ + sqlite3_pcache_page *pPage1; + pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); + if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because + ** pCache->nRefSum>0 */ + memset(pPage1->pBuf, 0, pCache->szPage); + pgno = 1; + } + } + sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); + } +} + +/* +** Close a cache. +*/ +void sqlite3PcacheClose(PCache *pCache){ + assert( pCache->pCache!=0 ); + pcacheTrace(("%p.CLOSE\n",pCache)); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); +} + +/* +** Discard the contents of the cache. +*/ +void sqlite3PcacheClear(PCache *pCache){ + sqlite3PcacheTruncate(pCache, 0); +} + +/* +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not bother fixing the pDirtyPrev pointers. +*/ +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + assert( pA!=0 && pB!=0 ); + for(;;){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } + }else{ + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } + } + } + return result.pDirty; +} + +/* +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pDirtyPrev pointers are +** corrupted by this sort. +** +** Since there cannot be more than 2^31 distinct pages in a database, +** there cannot be more than 31 buckets required by the merge sorter. +** One extra bucket is added to catch overflow in case something +** ever changes to make the previous sentence incorrect. +*/ +#define N_SORT_BUCKET 32 +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; ALWAYS(ipDirty; p; p=p->pDirtyNext){ + p->pDirty = p->pDirtyNext; + } + return pcacheSortDirtyList(pCache->pDirty); +} + +/* +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. +*/ +int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRefSum; +} + +/* +** Return the number of references to the page supplied as an argument. +*/ +int sqlite3PcachePageRefcount(PgHdr *p){ + return p->nRef; +} + +/* +** Return the total number of pages in the cache. +*/ +int sqlite3PcachePagecount(PCache *pCache){ + assert( pCache->pCache!=0 ); + return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); +} + +#ifdef SQLITE_TEST +/* +** Get the suggested cache-size value. +*/ +int sqlite3PcacheGetCachesize(PCache *pCache){ + return numberOfCachePages(pCache); +} +#endif + +/* +** Set the suggested cache-size value. +*/ +void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + assert( pCache->pCache!=0 ); + pCache->szCache = mxPage; + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); +} + +/* +** Set the suggested cache-spill value. Make no changes if if the +** argument is zero. Return the effective cache-spill size, which will +** be the larger of the szSpill and szCache. +*/ +int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ + int res; + assert( p->pCache!=0 ); + if( mxPage ){ + if( mxPage<0 ){ + mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra)); + } + p->szSpill = mxPage; + } + res = numberOfCachePages(p); + if( resszSpill ) res = p->szSpill; + return res; +} + +/* +** Free up as much memory as possible from the page cache. +*/ +void sqlite3PcacheShrink(PCache *pCache){ + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); +} + +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } + +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; +} + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if there are one or more dirty pages in the cache. Else false. +*/ +int sqlite3PCacheIsDirty(PCache *pCache){ + return (pCache->pDirty!=0); +} +#endif + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* +** For all dirty pages currently in the cache, invoke the specified +** callback. This is only used if the SQLITE_CHECK_PAGES macro is +** defined. +*/ +void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *pDirty; + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ + xIter(pDirty); + } +} +#endif diff --git a/third_party/sqlite3/pcache.inc b/third_party/sqlite3/pcache.inc new file mode 100644 index 000000000..77f3c0089 --- /dev/null +++ b/third_party/sqlite3/pcache.inc @@ -0,0 +1,191 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. +*/ +/* clang-format off */ + +#ifndef _PCACHE_H_ + +typedef struct PgHdr PgHdr; +typedef struct PCache PCache; + +/* +** Every page in the cache is controlled by an instance of the following +** structure. +*/ +struct PgHdr { + sqlite3_pcache_page *pPage; /* Pcache object page handle */ + void *pData; /* Page data */ + void *pExtra; /* Extra content */ + PCache *pCache; /* PRIVATE: Cache that owns this page */ + PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ + Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ +#ifdef SQLITE_CHECK_PAGES + u32 pageHash; /* Hash of page content */ +#endif + u16 flags; /* PGHDR flags defined below */ + + /********************************************************************** + ** Elements above, except pCache, are public. All that follow are + ** private to pcache.c and should not be accessed by other modules. + ** pCache is grouped with the public elements for efficiency. + */ + i16 nRef; /* Number of users of this page */ + PgHdr *pDirtyNext; /* Next element in list of dirty pages */ + PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ + /* NB: pDirtyNext and pDirtyPrev are undefined if the + ** PgHdr object is not dirty */ +}; + +/* Bit values for PgHdr.flags */ +#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ +#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ +#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ +#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ +#define PGHDR_MMAP 0x020 /* This is an mmap page object */ + +#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ + +/* Initialize and shutdown the page cache subsystem */ +int sqlite3PcacheInitialize(void); +void sqlite3PcacheShutdown(void); + +/* Page cache buffer management: +** These routines implement SQLITE_CONFIG_PAGECACHE. +*/ +void sqlite3PCacheBufferSetup(void *, int sz, int n); + +/* Create a new pager cache. +** Under memory stress, invoke xStress to try to make pages clean. +** Only clean and unpinned pages can be reclaimed. +*/ +int sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*, PgHdr*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *pToInit /* Preallocated space for the PCache */ +); + +/* Modify the page-size after the cache has been created. */ +int sqlite3PcacheSetPageSize(PCache *, int); + +/* Return the size in bytes of a PCache object. Used to preallocate +** storage space. +*/ +int sqlite3PcacheSize(void); + +/* One release per successful fetch. Page is pinned until released. +** Reference counted. +*/ +sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); +void sqlite3PcacheRelease(PgHdr*); + +void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ +void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ +void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ +void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); + +/* Change a page number. Used by incr-vacuum. */ +void sqlite3PcacheMove(PgHdr*, Pgno); + +/* Remove all pages with pgno>x. Reset the cache if x==0 */ +void sqlite3PcacheTruncate(PCache*, Pgno x); + +/* Get a list of all dirty pages in the cache, sorted by page number */ +PgHdr *sqlite3PcacheDirtyList(PCache*); + +/* Reset and close the cache object */ +void sqlite3PcacheClose(PCache*); + +/* Clear flags from pages of the page cache */ +void sqlite3PcacheClearSyncFlags(PCache *); + +/* Discard the contents of the cache */ +void sqlite3PcacheClear(PCache*); + +/* Return the total number of outstanding page references */ +int sqlite3PcacheRefCount(PCache*); + +/* Increment the reference count of an existing page */ +void sqlite3PcacheRef(PgHdr*); + +int sqlite3PcachePageRefcount(PgHdr*); + +/* Return the total number of pages stored in the cache */ +int sqlite3PcachePagecount(PCache*); + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* Iterate through all dirty pages currently stored in the cache. This +** interface is only available if SQLITE_CHECK_PAGES is defined when the +** library is built. +*/ +void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); +#endif + +#if defined(SQLITE_DEBUG) +/* Check invariants on a PgHdr object */ +int sqlite3PcachePageSanity(PgHdr*); +#endif + +/* Set and get the suggested cache-size for the specified pager-cache. +** +** If no global maximum is configured, then the system attempts to limit +** the total number of pages cached by purgeable pager-caches to the sum +** of the suggested cache-sizes. +*/ +void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST +int sqlite3PcacheGetCachesize(PCache *); +#endif + +/* Set or get the suggested spill-size for the specified pager-cache. +** +** The spill-size is the minimum number of pages in cache before the cache +** will attempt to spill dirty pages by calling xStress. +*/ +int sqlite3PcacheSetSpillsize(PCache *, int); + +/* Free up as much memory as possible from the page cache */ +void sqlite3PcacheShrink(PCache*); + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* Try to return memory used by the pcache module to the main memory heap */ +int sqlite3PcacheReleaseMemory(int); +#endif + +#ifdef SQLITE_TEST +void sqlite3PcacheStats(int*,int*,int*,int*); +#endif + +void sqlite3PCacheSetDefault(void); + +/* Return the header size */ +int sqlite3HeaderSizePcache(void); +int sqlite3HeaderSizePcache1(void); + +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +int sqlite3PCacheIsDirty(PCache *pCache); +#endif + +#endif /* _PCACHE_H_ */ diff --git a/third_party/sqlite3/pcache1.c b/third_party/sqlite3/pcache1.c new file mode 100644 index 000000000..11109b170 --- /dev/null +++ b/third_party/sqlite3/pcache1.c @@ -0,0 +1,1279 @@ +/* +** 2008 November 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements the default page cache implementation (the +** sqlite3_pcache interface). It also contains part of the implementation +** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. +** If the default page cache implementation is overridden, then neither of +** these two features are available. +** +** A Page cache line looks like this: +** +** ------------------------------------------------------------- +** | database page content | PgHdr1 | MemPage | PgHdr | +** ------------------------------------------------------------- +** +** The database page content is up front (so that buffer overreads tend to +** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage +** is the extension added by the btree.c module containing information such +** as the database page number and how that database page is used. PgHdr +** is added by the pcache.c layer and contains information used to keep track +** of which pages are "dirty". PgHdr1 is an extension added by this +** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. +** PgHdr1 contains information needed to look up a page by its page number. +** The superclass sqlite3_pcache_page.pBuf points to the start of the +** database page content and sqlite3_pcache_page.pExtra points to PgHdr. +** +** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at +** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The +** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this +** size can vary according to architecture, compile-time options, and +** SQLite library version number. +** +** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained +** using a separate memory allocation from the database page content. This +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more +** than a power of two with the memory allocator rounding up to the next +** power of two, and leaving the rounded-up space unused. +** +** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates +** with this module. Information is passed back and forth as PgHdr1 pointers. +** +** The pcache.c and pager.c modules deal pointers to PgHdr objects. +** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +typedef struct PCache1 PCache1; +typedef struct PgHdr1 PgHdr1; +typedef struct PgFreeslot PgFreeslot; +typedef struct PGroup PGroup; + +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +** +** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** but causes a 2-byte gap in the structure for most architectures (since +** pointers must be either 4 or 8-byte aligned). As this structure is located +** in memory directly after the associated page data, if the database is +** corrupt, code at the b-tree layer may overread the page buffer and +** read part of this structure before the corruption is detected. This +** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of unitialized memory +** in the structure. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ +}; + +/* +** A page is pinned if it is not on the LRU list. To be "pinned" means +** that the page is in active use and must not be deallocated. +*/ +#define PAGE_IS_PINNED(p) ((p)->pLruNext==0) +#define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) + +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +** of one or more PCaches that are able to recycle each other's unpinned +** pages when they are under memory pressure. A PGroup is an instance of +** the following object. +** +** This page cache implementation works in one of two modes: +** +** (1) Every PCache is the sole member of its own PGroup. There is +** one PGroup per PCache. +** +** (2) There is a single global PGroup that all PCaches are a member +** of. +** +** Mode 1 uses more memory (since PCache instances are not able to rob +** unused pages from other PCaches) but it also operates without a mutex, +** and is therefore often faster. Mode 2 requires a mutex in order to be +** threadsafe, but recycles pages more efficiently. +** +** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single +** PGroup which is the pcache1.grp global variable and its mutex is +** SQLITE_MUTEX_STATIC_LRU. +*/ +struct PGroup { + sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ + unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ + unsigned int nMinPage; /* Sum of nMin for purgeable caches */ + unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ + unsigned int nPurgeable; /* Number of purgeable pages allocated */ + PgHdr1 lru; /* The beginning and end of the LRU list */ +}; + +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. +*/ +struct PCache1 { + /* Cache configuration parameters. Page size (szPage) and the purgeable + ** flag (bPurgeable) and the pnPurgeable pointer are all set when the + ** cache is created and are never changed thereafter. nMax may be + ** modified at any time by a call to the pcache1Cachesize() method. + ** The PGroup mutex must be held when accessing nMax. + */ + PGroup *pGroup; /* PGroup this cache belongs to */ + unsigned int *pnPurgeable; /* Pointer to pGroup->nPurgeable */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ + int bPurgeable; /* True if cache is purgeable */ + unsigned int nMin; /* Minimum number of pages reserved */ + unsigned int nMax; /* Configured "cache_size" value */ + unsigned int n90pct; /* nMax*9/10 */ + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + unsigned int nPurgeableDummy; /* pnPurgeable points here when not used*/ + + /* Hash table of all pages. The following variables may only be accessed + ** when the accessor is holding the PGroup mutex. + */ + unsigned int nRecyclable; /* Number of pages in the LRU list */ + unsigned int nPage; /* Total number of pages in apHash */ + unsigned int nHash; /* Number of slots in apHash[] */ + PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ +}; + +/* +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. +*/ +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; + +/* +** Global data used by this cache. +*/ +static SQLITE_WSD struct PCacheGlobal { + PGroup grp; /* The global PGroup for mode (2) */ + + /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The + ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all + ** fixed at sqlite3_initialize() time and do not require mutex protection. + ** The nFreeSlot and pFree values do require mutex protection. + */ + int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ + int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ + /* Above requires no mutex. Use mutex below for variable that follow. */ + sqlite3_mutex *mutex; /* Mutex for accessing the following: */ + PgFreeslot *pFree; /* Free page blocks */ + int nFreeSlot; /* Number of unused pcache slots */ + /* The following value requires a mutex to change. We skip the mutex on + ** reading because (1) most platforms read a 32-bit integer atomically and + ** (2) even if an incorrect value is read, no great harm is done since this + ** is really just an optimization. */ + int bUnderPressure; /* True if low on PAGECACHE memory */ +} pcache1_g; + +/* +** All code in this file should access the global structure above via the +** alias "pcache1". This ensures that the WSD emulation is used when +** compiling for systems that do not support real WSD. +*/ +#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) + +/* +** Macros to enter and leave the PCache LRU mutex. +*/ +#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 +# define pcache1EnterMutex(X) assert((X)->mutex==0) +# define pcache1LeaveMutex(X) assert((X)->mutex==0) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 +#else +# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 +#endif + +/******************************************************************************/ +/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + + +/* +** This function is called during initialization if a static buffer is +** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE +** verb to sqlite3_config(). Parameter pBuf points to an allocation large +** enough to contain 'n' buffers of 'sz' bytes each. +** +** This routine is called from sqlite3_initialize() and so it is guaranteed +** to be serialized already. There is no need for further mutexing. +*/ +void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + if( pcache1.isInit ){ + PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; + if( n==0 ) sz = 0; + sz = ROUNDDOWN8(sz); + pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); + pcache1.pStart = pBuf; + pcache1.pFree = 0; + pcache1.bUnderPressure = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache1.pFree; + pcache1.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache1.pEnd = pBuf; + } +} + +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + i64 szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; + }else{ + szBulk = -1024 * (i64)pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ + szBulk = pCache->szAlloc*(i64)pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + do{ + PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->isAnchor = 0; + pX->pNext = pCache->pFree; + pX->pLruPrev = 0; /* Initializing this saves a valgrind error */ + pCache->pFree = pX; + zBulk += pCache->szAlloc; + }while( --nBulk ); + } + return pCache->pFree!=0; +} + +/* +** Malloc function used within this file to allocate space from the buffer +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls +** back to sqlite3Malloc(). +** +** Multiple threads can run this routine at the same time. Global variables +** in pcache1 need to be protected via mutex. +*/ +static void *pcache1Alloc(int nByte){ + void *p = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + if( nByte<=pcache1.szSlot ){ + sqlite3_mutex_enter(pcache1.mutex); + p = (PgHdr1 *)pcache1.pFree; + if( p ){ + pcache1.pFree = pcache1.pFree->pNext; + pcache1.nFreeSlot--; + pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); + } + sqlite3_mutex_leave(pcache1.mutex); + } + if( p==0 ){ + /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool. Get + ** it from sqlite3Malloc instead. + */ + p = sqlite3Malloc(nByte); +#ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + if( p ){ + int sz = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3_mutex_leave(pcache1.mutex); + } +#endif + sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); + } + return p; +} + +/* +** Free an allocated buffer obtained from pcache1Alloc(). +*/ +static void pcache1Free(void *p){ + if( p==0 ) return; + if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ + PgFreeslot *pSlot; + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_USED, 1); + pSlot = (PgFreeslot*)p; + pSlot->pNext = pcache1.pFree; + pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + pcache1.bUnderPressure = pcache1.nFreeSlot=pcache1.pStart && ppGroup->mutex) ); + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + assert( pCache->pFree!=0 ); + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); +#endif + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } +#else + pPg = pcache1Alloc(pCache->szAlloc); +#endif + if( benignMalloc ){ sqlite3EndBenignMalloc(); } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); +#endif + if( pPg==0 ) return 0; +#ifndef SQLITE_PCACHE_SEPARATE_HEADER + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif + p->page.pBuf = pPg; + p->page.pExtra = &p[1]; + p->isBulkLocal = 0; + p->isAnchor = 0; + p->pLruPrev = 0; /* Initializing this saves a valgrind error */ + } + (*pCache->pnPurgeable)++; + return p; +} + +/* +** Free a page object allocated by pcache1AllocPage(). +*/ +static void pcache1FreePage(PgHdr1 *p){ + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ + pcache1Free(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + sqlite3_free(p); +#endif + } + (*pCache->pnPurgeable)--; +} + +/* +** Malloc function used by SQLite to obtain space from the buffer configured +** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer +** exists, this function falls back to sqlite3Malloc(). +*/ +void *sqlite3PageMalloc(int sz){ + assert( sz<=65536+8 ); /* These allocations are never very large */ + return pcache1Alloc(sz); +} + +/* +** Free an allocated buffer obtained from sqlite3PageMalloc(). +*/ +void sqlite3PageFree(void *p){ + pcache1Free(p); +} + + +/* +** Return true if it desirable to avoid allocating a new page cache +** entry. +** +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory but the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. +*/ +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ + return pcache1.bUnderPressure; + }else{ + return sqlite3HeapNearlyFull(); + } +} + +/******************************************************************************/ +/******** General Implementation Functions ************************************/ + +/* +** This function is used to resize the hash table used by the cache passed +** as the first argument. +** +** The PCache mutex must be held when this function is called. +*/ +static void pcache1ResizeHash(PCache1 *p){ + PgHdr1 **apNew; + unsigned int nNew; + unsigned int i; + + assert( sqlite3_mutex_held(p->pGroup->mutex) ); + + nNew = p->nHash*2; + if( nNew<256 ){ + nNew = 256; + } + + pcache1LeaveMutex(p->pGroup); + if( p->nHash ){ sqlite3BeginBenignMalloc(); } + apNew = (PgHdr1 **)sqlite3MallocZero(sizeof(PgHdr1 *)*nNew); + if( p->nHash ){ sqlite3EndBenignMalloc(); } + pcache1EnterMutex(p->pGroup); + if( apNew ){ + for(i=0; inHash; i++){ + PgHdr1 *pPage; + PgHdr1 *pNext = p->apHash[i]; + while( (pPage = pNext)!=0 ){ + unsigned int h = pPage->iKey % nNew; + pNext = pPage->pNext; + pPage->pNext = apNew[h]; + apNew[h] = pPage; + } + } + sqlite3_free(p->apHash); + p->apHash = apNew; + p->nHash = nNew; + } +} + +/* +** This function is used internally to remove the page pPage from the +** PGroup LRU list, if is part of it. If pPage is not part of the PGroup +** LRU list, then this function is a no-op. +** +** The PGroup mutex must be held when this function is called. +*/ +static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ + assert( pPage!=0 ); + assert( PAGE_IS_UNPINNED(pPage) ); + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); + assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) ); + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ + assert( pPage->isAnchor==0 ); + assert( pPage->pCache->pGroup->lru.isAnchor==1 ); + pPage->pCache->nRecyclable--; + return pPage; +} + + +/* +** Remove the page supplied as an argument from the hash table +** (PCache1.apHash structure) that it is currently stored in. +** Also free the page if freePage is true. +** +** The PGroup mutex must be held when this function is called. +*/ +static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ + unsigned int h; + PCache1 *pCache = pPage->pCache; + PgHdr1 **pp; + + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + h = pPage->iKey % pCache->nHash; + for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; + + pCache->nPage--; + if( freeFlag ) pcache1FreePage(pPage); +} + +/* +** If there are currently more than nMaxPage pages allocated, try +** to recycle pages to reduce the number allocated to nMaxPage. +*/ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; + assert( sqlite3_mutex_held(pGroup->mutex) ); + while( pGroup->nPurgeable>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ + assert( p->pCache->pGroup==pGroup ); + assert( PAGE_IS_UNPINNED(p) ); + pcache1PinPage(p); + pcache1RemoveFromHash(p, 1); + } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; + } +} + +/* +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this +** criteria are unpinned before they are discarded. +** +** The PCache mutex must be held when this function is called. +*/ +static void pcache1TruncateUnsafe( + PCache1 *pCache, /* The cache to truncate */ + unsigned int iLimit /* Drop pages with this pgno or larger */ +){ + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; + PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; + while( (pPage = *pp)!=0 ){ + if( pPage->iKey>=iLimit ){ + pCache->nPage--; + *pp = pPage->pNext; + if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage); + pcache1FreePage(pPage); + }else{ + pp = &pPage->pNext; + TESTONLY( if( nPage>=0 ) nPage++; ) + } + } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; + } + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); +} + +/******************************************************************************/ +/******** sqlite3_pcache Methods **********************************************/ + +/* +** Implementation of the sqlite3_pcache.xInit method. +*/ +static int pcache1Init(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit==0 ); + memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + +#if SQLITE_THREADSAFE + if( sqlite3GlobalConfig.bCoreMutex ){ + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); + } +#endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; + } + pcache1.grp.mxPinned = 10; + pcache1.isInit = 1; + return SQLITE_OK; +} + +/* +** Implementation of the sqlite3_pcache.xShutdown method. +** Note that the static mutex allocated in xInit does +** not need to be freed. +*/ +static void pcache1Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit!=0 ); + memset(&pcache1, 0, sizeof(pcache1)); +} + +/* forward declaration */ +static void pcache1Destroy(sqlite3_pcache *p); + +/* +** Implementation of the sqlite3_pcache.xCreate method. +** +** Allocate a new cache. +*/ +static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ + PCache1 *pCache; /* The newly created page cache */ + PGroup *pGroup; /* The group the new page cache will belong to */ + int sz; /* Bytes of memory required to allocate the new cache */ + + assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); + assert( szExtra < 300 ); + + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; + pCache = (PCache1 *)sqlite3MallocZero(sz); + if( pCache ){ + if( pcache1.separateCache ){ + pGroup = (PGroup*)&pCache[1]; + pGroup->mxPinned = 10; + }else{ + pGroup = &pcache1.grp; + } + pcache1EnterMutex(pGroup); + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } + pCache->pGroup = pGroup; + pCache->szPage = szPage; + pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); + pCache->bPurgeable = (bPurgeable ? 1 : 0); + pcache1ResizeHash(pCache); + if( bPurgeable ){ + pCache->nMin = 10; + pGroup->nMinPage += pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->pnPurgeable = &pGroup->nPurgeable; + }else{ + pCache->pnPurgeable = &pCache->nPurgeableDummy; + } + pcache1LeaveMutex(pGroup); + if( pCache->nHash==0 ){ + pcache1Destroy((sqlite3_pcache*)pCache); + pCache = 0; + } + } + return (sqlite3_pcache *)pCache; +} + +/* +** Implementation of the sqlite3_pcache.xCachesize method. +** +** Configure the cache_size limit for a cache. +*/ +static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ + PCache1 *pCache = (PCache1 *)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + pcache1EnterMutex(pGroup); + pGroup->nMaxPage += (nMax - pCache->nMax); + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->nMax = nMax; + pCache->n90pct = pCache->nMax*9/10; + pcache1EnforceMaxPage(pCache); + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xShrink method. +** +** Free up as much memory as possible. +*/ +static void pcache1Shrink(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1*)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + int savedMaxPage; + pcache1EnterMutex(pGroup); + savedMaxPage = pGroup->nMaxPage; + pGroup->nMaxPage = 0; + pcache1EnforceMaxPage(pCache); + pGroup->nMaxPage = savedMaxPage; + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xPagecount method. +*/ +static int pcache1Pagecount(sqlite3_pcache *p){ + int n; + PCache1 *pCache = (PCache1*)p; + pcache1EnterMutex(pCache->pGroup); + n = pCache->nPage; + pcache1LeaveMutex(pCache->pGroup); + return n; +} + + +/* +** Implement steps 3, 4, and 5 of the pcache1Fetch() algorithm described +** in the header of the pcache1Fetch() procedure. +** +** This steps are broken out into a separate procedure because they are +** usually not needed, and by avoiding the stack initialization required +** for these steps, the main pcache1Fetch() procedure can run faster. +*/ +static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( + PCache1 *pCache, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PGroup *pGroup = pCache->pGroup; + PgHdr1 *pPage = 0; + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclablenPage>=pCache->nHash ) pcache1ResizeHash(pCache); + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable + && !pGroup->lru.pLruPrev->isAnchor + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ + PCache1 *pOther; + pPage = pGroup->lru.pLruPrev; + assert( PAGE_IS_UNPINNED(pPage) ); + pcache1RemoveFromHash(pPage, 0); + pcache1PinPage(pPage); + pOther = pPage->pCache; + if( pOther->szAlloc != pCache->szAlloc ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nPurgeable -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + pPage = pcache1AllocPage(pCache, createFlag==1); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + if( iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + } + return pPage; +} + +/* +** Implementation of the sqlite3_pcache.xFetch method. +** +** Fetch a page by key value. +** +** Whether or not a new page may be allocated by this function depends on +** the value of the createFlag argument. 0 means do not allocate a new +** page. 1 means allocate a new page if space is easily available. 2 +** means to try really hard to allocate a new page. +** +** For a non-purgeable cache (a cache used as the storage for an in-memory +** database) there is really no difference between createFlag 1 and 2. So +** the calling function (pcache.c) will never have a createFlag of 1 on +** a non-purgeable cache. +** +** There are three different approaches to obtaining space for a page, +** depending on the value of parameter createFlag (which may be 0, 1 or 2). +** +** 1. Regardless of the value of createFlag, the cache is searched for a +** copy of the requested page. If one is found, it is returned. +** +** 2. If createFlag==0 and the page is not already in the cache, NULL is +** returned. +** +** 3. If createFlag is 1, and the page is not already in the cache, then +** return NULL (do not allocate a new page) if any of the following +** conditions are true: +** +** (a) the number of pages pinned by the cache is greater than +** PCache1.nMax, or +** +** (b) the number of pages pinned by the cache is greater than +** the sum of nMax for all purgeable caches, less the sum of +** nMin for all other purgeable caches, or +** +** 4. If none of the first three conditions apply and the cache is marked +** as purgeable, and if one of the following is true: +** +** (a) The number of pages allocated for the cache is already +** PCache1.nMax, or +** +** (b) The number of pages allocated for all purgeable caches is +** already equal to or greater than the sum of nMax for all +** purgeable caches, +** +** (c) The system is under memory pressure and wants to avoid +** unnecessary pages cache entry allocations +** +** then attempt to recycle a page from the LRU list. If it is the right +** size, return the recycled buffer. Otherwise, free the buffer and +** proceed to step 5. +** +** 5. Otherwise, allocate and return a new page buffer. +** +** There are two versions of this routine. pcache1FetchWithMutex() is +** the general case. pcache1FetchNoMutex() is a faster implementation for +** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper +** invokes the appropriate routine. +*/ +static PgHdr1 *pcache1FetchNoMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = 0; + + /* Step 1: Search the hash table for an existing entry. */ + pPage = pCache->apHash[iKey % pCache->nHash]; + while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } + + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ + if( pPage ){ + if( PAGE_IS_UNPINNED(pPage) ){ + return pcache1PinPage(pPage); + }else{ + return pPage; + } + }else if( createFlag ){ + /* Steps 3, 4, and 5 implemented by this subroutine */ + return pcache1FetchStage2(pCache, iKey, createFlag); + }else{ + return 0; + } +} +#if PCACHE1_MIGHT_USE_GROUP_MUTEX +static PgHdr1 *pcache1FetchWithMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage; + + pcache1EnterMutex(pCache->pGroup); + pPage = pcache1FetchNoMutex(p, iKey, createFlag); + assert( pPage==0 || pCache->iMaxKey>=iKey ); + pcache1LeaveMutex(pCache->pGroup); + return pPage; +} +#endif +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ +#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) + PCache1 *pCache = (PCache1 *)p; +#endif + + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + assert( pCache->nHash>0 ); +#if PCACHE1_MIGHT_USE_GROUP_MUTEX + if( pCache->pGroup->mutex ){ + return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + }else +#endif + { + return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); + } +} + + +/* +** Implementation of the sqlite3_pcache.xUnpin method. +** +** Mark a page as unpinned (eligible for asynchronous recycling). +*/ +static void pcache1Unpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int reuseUnlikely +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PGroup *pGroup = pCache->pGroup; + + assert( pPage->pCache==pCache ); + pcache1EnterMutex(pGroup); + + /* It is an error to call this function if the page is already + ** part of the PGroup LRU list. + */ + assert( pPage->pLruNext==0 ); + assert( PAGE_IS_PINNED(pPage) ); + + if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ + pcache1RemoveFromHash(pPage, 1); + }else{ + /* Add the page to the PGroup LRU list. */ + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; + pCache->nRecyclable++; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xRekey method. +*/ +static void pcache1Rekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned int iOld, + unsigned int iNew +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PgHdr1 **pp; + unsigned int h; + assert( pPage->iKey==iOld ); + assert( pPage->pCache==pCache ); + + pcache1EnterMutex(pCache->pGroup); + + h = iOld%pCache->nHash; + pp = &pCache->apHash[h]; + while( (*pp)!=pPage ){ + pp = &(*pp)->pNext; + } + *pp = pPage->pNext; + + h = iNew%pCache->nHash; + pPage->iKey = iNew; + pPage->pNext = pCache->apHash[h]; + pCache->apHash[h] = pPage; + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xTruncate method. +** +** Discard all unpinned pages in the cache with a page number equal to +** or greater than parameter iLimit. Any pinned pages with a page number +** equal to or greater than iLimit are implicitly unpinned. +*/ +static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(pCache->pGroup); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xDestroy method. +** +** Destroy a cache allocated using pcache1Create(). +*/ +static void pcache1Destroy(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup = pCache->pGroup; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); + pcache1EnterMutex(pGroup); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); + assert( pGroup->nMaxPage >= pCache->nMax ); + pGroup->nMaxPage -= pCache->nMax; + assert( pGroup->nMinPage >= pCache->nMin ); + pGroup->nMinPage -= pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1EnforceMaxPage(pCache); + pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); + sqlite3_free(pCache->apHash); + sqlite3_free(pCache); +} + +/* +** This function is called during initialization (sqlite3_initialize()) to +** install the default pluggable cache module, assuming the user has not +** already provided an alternative. +*/ +void sqlite3PCacheSetDefault(void){ + static const sqlite3_pcache_methods2 defaultMethods = { + 1, /* iVersion */ + 0, /* pArg */ + pcache1Init, /* xInit */ + pcache1Shutdown, /* xShutdown */ + pcache1Create, /* xCreate */ + pcache1Cachesize, /* xCachesize */ + pcache1Pagecount, /* xPagecount */ + pcache1Fetch, /* xFetch */ + pcache1Unpin, /* xUnpin */ + pcache1Rekey, /* xRekey */ + pcache1Truncate, /* xTruncate */ + pcache1Destroy, /* xDestroy */ + pcache1Shrink /* xShrink */ + }; + sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); +} + +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } + +/* +** Return the global mutex used by this PCACHE implementation. The +** sqlite3_status() routine needs access to this mutex. +*/ +sqlite3_mutex *sqlite3Pcache1Mutex(void){ + return pcache1.mutex; +} + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqlite3_free()ed. +** +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +int sqlite3PcacheReleaseMemory(int nReq){ + int nFree = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + assert( sqlite3_mutex_notheld(pcache1.mutex) ); + if( sqlite3GlobalConfig.pPage==0 ){ + PgHdr1 *p; + pcache1EnterMutex(&pcache1.grp); + while( (nReq<0 || nFreeisAnchor==0 + ){ + nFree += pcache1MemSize(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + nFree += sqlite3MemSize(p); +#endif + assert( PAGE_IS_UNPINNED(p) ); + pcache1PinPage(p); + pcache1RemoveFromHash(p, 1); + } + pcache1LeaveMutex(&pcache1.grp); + } + return nFree; +} +#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ + +#ifdef SQLITE_TEST +/* +** This function is used by test procedures to inspect the internal state +** of the global cache. +*/ +void sqlite3PcacheStats( + int *pnCurrent, /* OUT: Total number of pages cached */ + int *pnMax, /* OUT: Global maximum cache size */ + int *pnMin, /* OUT: Sum of PCache1.nMin for purgeable caches */ + int *pnRecyclable /* OUT: Total number of pages available for recycling */ +){ + PgHdr1 *p; + int nRecyclable = 0; + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ + assert( PAGE_IS_UNPINNED(p) ); + nRecyclable++; + } + *pnCurrent = pcache1.grp.nPurgeable; + *pnMax = (int)pcache1.grp.nMaxPage; + *pnMin = (int)pcache1.grp.nMinPage; + *pnRecyclable = nRecyclable; +} +#endif diff --git a/third_party/sqlite3/pragma.c b/third_party/sqlite3/pragma.c new file mode 100644 index 000000000..b22edbd8a --- /dev/null +++ b/third_party/sqlite3/pragma.c @@ -0,0 +1,2594 @@ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the PRAGMA command. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/*************************************************************************** +** The "pragma.h" include file is an automatically generated file that +** that includes the PragType_XXXX macro definitions and the aPragmaName[] +** object. This ensures that the aPragmaName[] table is arranged in +** lexicographical order to facility a binary search of the pragma name. +** Do not edit pragma.h directly. Edit and rerun the script in at +** ../tool/mkpragmatab.tcl. */ +#include "third_party/sqlite3/pragma.inc" + +/* +** Interpret the given string as a safety level. Return 0 for OFF, +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** unrecognized string argument. The FULL and EXTRA option is disallowed +** if the omitFull parameter it 1. +** +** Note that the values returned are one less that the values that +** should be passed into sqlite3BtreeSetSafetyLevel(). The is done +** to support legacy SQL code. The safety level used to be boolean +** and older scripts may have used numbers 0 for OFF and 1 for ON. +*/ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ + /* 123456789 123456789 123 */ + static const char zText[] = "onoffalseyestruextrafull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; + /* on no off false yes true extra full */ + int i, n; + if( sqlite3Isdigit(*z) ){ + return (u8)sqlite3Atoi(z); + } + n = sqlite3Strlen30(z); + for(i=0; i=0&&i<=2)?i:0); +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Interpret the given string as a temp db location. Return 1 for file +** backed temporary databases, 2 for the Red-Black tree in memory database +** and 0 to use the compile-time default. +*/ +static int getTempStore(const char *z){ + if( z[0]>='0' && z[0]<='2' ){ + return z[0] - '0'; + }else if( sqlite3StrICmp(z, "file")==0 ){ + return 1; + }else if( sqlite3StrICmp(z, "memory")==0 ){ + return 2; + }else{ + return 0; + } +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Invalidate temp storage, either when the temp storage is changed +** from default, or when 'file' and the temp_store_directory has changed +*/ +static int invalidateTempStorage(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt!=0 ){ + if( !db->autoCommit + || sqlite3BtreeTxnState(db->aDb[1].pBt)!=SQLITE_TXN_NONE + ){ + sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " + "from within a transaction"); + return SQLITE_ERROR; + } + sqlite3BtreeClose(db->aDb[1].pBt); + db->aDb[1].pBt = 0; + sqlite3ResetAllSchemasOfConnection(db); + } + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** If the TEMP database is open, close it and mark the database schema +** as needing reloading. This must be done when using the SQLITE_TEMP_STORE +** or DEFAULT_TEMP_STORE pragmas. +*/ +static int changeTempStorage(Parse *pParse, const char *zStorageType){ + int ts = getTempStore(zStorageType); + sqlite3 *db = pParse->db; + if( db->temp_store==ts ) return SQLITE_OK; + if( invalidateTempStorage( pParse ) != SQLITE_OK ){ + return SQLITE_ERROR; + } + db->temp_store = (u8)ts; + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +/* +** Set result column names for a pragma. +*/ +static void setPragmaResultColumnNames( + Vdbe *v, /* The query under construction */ + const PragmaName *pPragma /* The pragma */ +){ + u8 n = pPragma->nPragCName; + sqlite3VdbeSetNumCols(v, n==0 ? 1 : n); + if( n==0 ){ + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, pPragma->zName, SQLITE_STATIC); + }else{ + int i, j; + for(i=0, j=pPragma->iPragCName; iautoCommit ){ + Db *pDb = db->aDb; + int n = db->nDb; + assert( SQLITE_FullFSync==PAGER_FULLFSYNC ); + assert( SQLITE_CkptFullFSync==PAGER_CKPT_FULLFSYNC ); + assert( SQLITE_CacheSpill==PAGER_CACHESPILL ); + assert( (PAGER_FULLFSYNC | PAGER_CKPT_FULLFSYNC | PAGER_CACHESPILL) + == PAGER_FLAGS_MASK ); + assert( (pDb->safety_level & PAGER_SYNCHRONOUS_MASK)==pDb->safety_level ); + while( (n--) > 0 ){ + if( pDb->pBt ){ + sqlite3BtreeSetPagerFlags(pDb->pBt, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK) ); + } + pDb++; + } + } +} +#else +# define setAllPagerFlags(X) /* no-op */ +#endif + + +/* +** Return a human-readable name for a constraint resolution action. +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +static const char *actionName(u8 action){ + const char *zName; + switch( action ){ + case OE_SetNull: zName = "SET NULL"; break; + case OE_SetDflt: zName = "SET DEFAULT"; break; + case OE_Cascade: zName = "CASCADE"; break; + case OE_Restrict: zName = "RESTRICT"; break; + default: zName = "NO ACTION"; + assert( action==OE_None ); break; + } + return zName; +} +#endif + + +/* +** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants +** defined in pager.h. This function returns the associated lowercase +** journal-mode name. +*/ +const char *sqlite3JournalModename(int eMode){ + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory" +#ifndef SQLITE_OMIT_WAL + , "wal" +#endif + }; + assert( PAGER_JOURNALMODE_DELETE==0 ); + assert( PAGER_JOURNALMODE_PERSIST==1 ); + assert( PAGER_JOURNALMODE_OFF==2 ); + assert( PAGER_JOURNALMODE_TRUNCATE==3 ); + assert( PAGER_JOURNALMODE_MEMORY==4 ); + assert( PAGER_JOURNALMODE_WAL==5 ); + assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + + if( eMode==ArraySize(azModeName) ) return 0; + return azModeName[eMode]; +} + +/* +** Locate a pragma in the aPragmaName[] array. +*/ +static const PragmaName *pragmaLocate(const char *zName){ + int upr, lwr, mid = 0, rc; + lwr = 0; + upr = ArraySize(aPragmaName)-1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + rc = sqlite3_stricmp(zName, aPragmaName[mid].zName); + if( rc==0 ) break; + if( rc<0 ){ + upr = mid - 1; + }else{ + lwr = mid + 1; + } + } + return lwr>upr ? 0 : &aPragmaName[mid]; +} + +/* +** Create zero or more entries in the output for the SQL functions +** defined by FuncDef p. +*/ +static void pragmaFunclistLine( + Vdbe *v, /* The prepared statement being created */ + FuncDef *p, /* A particular function definition */ + int isBuiltin, /* True if this is a built-in function */ + int showInternFuncs /* True if showing internal functions */ +){ + for(; p; p=p->pNext){ + const char *zType; + static const u32 mask = + SQLITE_DETERMINISTIC | + SQLITE_DIRECTONLY | + SQLITE_SUBTYPE | + SQLITE_INNOCUOUS | + SQLITE_FUNC_INTERNAL + ; + static const char *azEnc[] = { 0, "utf8", "utf16le", "utf16be" }; + + assert( SQLITE_FUNC_ENCMASK==0x3 ); + assert( strcmp(azEnc[SQLITE_UTF8],"utf8")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16LE],"utf16le")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16BE],"utf16be")==0 ); + + if( p->xSFunc==0 ) continue; + if( (p->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && showInternFuncs==0 + ){ + continue; + } + if( p->xValue!=0 ){ + zType = "w"; + }else if( p->xFinalize!=0 ){ + zType = "a"; + }else{ + zType = "s"; + } + sqlite3VdbeMultiLoad(v, 1, "sissii", + p->zName, isBuiltin, + zType, azEnc[p->funcFlags&SQLITE_FUNC_ENCMASK], + p->nArg, + (p->funcFlags & mask) ^ SQLITE_INNOCUOUS + ); + } +} + + +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with a value of the +** string held in register 3. Decrement the result count in register 1 +** and halt if the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + return addr; +} + +/* +** Process a pragma statement. +** +** Pragmas are of this form: +** +** PRAGMA [schema.]id [= value] +** +** The identifier might also be a string. The value is a string, and +** identifier, or a number. If minusFlag is true, then the value is +** a number that was preceded by a minus sign. +** +** If the left side is "database.id" then pId1 is the database name +** and pId2 is the id. If the left side is just "id" then pId1 is the +** id and pId2 is any empty string. +*/ +void sqlite3Pragma( + Parse *pParse, + Token *pId1, /* First part of [schema.]id field */ + Token *pId2, /* Second part of [schema.]id field, or NULL */ + Token *pValue, /* Token for , or NULL */ + int minusFlag /* True if a '-' sign preceded */ +){ + char *zLeft = 0; /* Nul-terminated UTF-8 string */ + char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ + const char *zDb = 0; /* The database name */ + Token *pId; /* Pointer to token */ + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int iDb; /* Database index for */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + const PragmaName *pPragma; /* The pragma */ + + if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); + pParse->nMem = 2; + + /* Interpret the [schema.] part of the pragma statement. iDb is the + ** index of the database this pragma is being applied to in db.aDb[]. */ + iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); + if( iDb<0 ) return; + pDb = &db->aDb[iDb]; + + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. + */ + if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ + return; + } + + zLeft = sqlite3NameFromToken(db, pId); + if( !zLeft ) return; + if( minusFlag ){ + zRight = sqlite3MPrintf(db, "-%T", pValue); + }else{ + zRight = sqlite3NameFromToken(db, pValue); + } + + assert( pId2 ); + zDb = pId2->n>0 ? pDb->zDbSName : 0; + if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ + goto pragma_out; + } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + ** + ** IMPLEMENTATION-OF: R-12238-55120 Whenever a PRAGMA statement is parsed, + ** an SQLITE_FCNTL_PRAGMA file control is sent to the open sqlite3_file + ** object corresponding to the database file to which the pragma + ** statement refers. + ** + ** IMPLEMENTATION-OF: R-29875-31678 The argument to the SQLITE_FCNTL_PRAGMA + ** file control is an array of pointers to strings (char**) in which the + ** second element of the array is the name of the pragma and the third + ** element is the argument to the pragma or NULL if the pragma has no + ** argument. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + db->busyHandler.nBusy = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, aFcntl[0], SQLITE_TRANSIENT); + returnSingleText(v, aFcntl[0]); + sqlite3_free(aFcntl[0]); + goto pragma_out; + } + if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + goto pragma_out; + } + + /* Locate the pragma in the lookup table */ + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ) goto pragma_out; + + /* Make sure the database schema is loaded if the pragma requires that */ + if( (pPragma->mPragFlg & PragFlg_NeedSchema)!=0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + } + + /* Register the result column names for pragmas that return results */ + if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 + && ((pPragma->mPragFlg & PragFlg_NoColumns1)==0 || zRight==0) + ){ + setPragmaResultColumnNames(v, pPragma); + } + + /* Jump to the appropriate pragma handler */ + switch( pPragma->ePragTyp ){ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + /* + ** PRAGMA [schema.]default_cache_size + ** PRAGMA [schema.]default_cache_size=N + ** + ** The first form reports the current persistent setting for the + ** page cache size. The value returned is the maximum number of + ** pages in the page cache. The second form sets both the current + ** page cache size value and the persistent page cache size value + ** stored in the database file. + ** + ** Older versions of SQLite would set the default cache size to a + ** negative number to indicate synchronous=OFF. These days, synchronous + ** is always on by default regardless of the sign of the default cache + ** size. But continue to take the absolute value of the default cache + ** size of historical compatibility. + */ + case PragTyp_DEFAULT_CACHE_SIZE: { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList getCacheSize[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 2, 0}, + { OP_Subtract, 1, 2, 1}, + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, + { OP_ResultRow, 1, 1, 0}, + }; + VdbeOp *aOp; + sqlite3VdbeUsesBtree(v, iDb); + if( !zRight ){ + pParse->nMem += 2; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; + }else{ + int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, size); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } +#endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + /* + ** PRAGMA [schema.]page_size + ** PRAGMA [schema.]page_size=N + ** + ** The first form reports the current setting for the + ** database page size in bytes. The second form sets the + ** database page size value. The value can only be set if + ** the database has not yet been created. + */ + case PragTyp_PAGE_SIZE: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; + returnSingleInt(v, size); + }else{ + /* Malloc may fail when setting the page-size, as there is an internal + ** buffer that the pager module resizes using sqlite3_realloc(). + */ + db->nextPagesize = sqlite3Atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,0,0) ){ + sqlite3OomFault(db); + } + } + break; + } + + /* + ** PRAGMA [schema.]secure_delete + ** PRAGMA [schema.]secure_delete=ON/OFF/FAST + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports the new value. + */ + case PragTyp_SECURE_DELETE: { + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + if( sqlite3_stricmp(zRight, "fast")==0 ){ + b = 2; + }else{ + b = sqlite3GetBoolean(zRight, 0); + } + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(v, b); + break; + } + + /* + ** PRAGMA [schema.]max_page_count + ** PRAGMA [schema.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** + ** The absolute value of N is used. This is undocumented and might + ** change. The only purpose is to provide an easy way to test + ** the sqlite3AbsInt32() function. + ** + ** PRAGMA [schema.]page_count + ** + ** Return the number of pages in the specified database. + */ + case PragTyp_PAGE_COUNT: { + int iReg; + i64 x = 0; + sqlite3CodeVerifySchema(pParse, iDb); + iReg = ++pParse->nMem; + if( sqlite3Tolower(zLeft[0])=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + if( zRight && sqlite3DecOrHexToI64(zRight,&x)==0 ){ + if( x<0 ) x = 0; + else if( x>0xfffffffe ) x = 0xfffffffe; + }else{ + x = 0; + } + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, (int)x); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); + break; + } + + /* + ** PRAGMA [schema.]locking_mode + ** PRAGMA [schema.]locking_mode = (normal|exclusive) + */ + case PragTyp_LOCKING_MODE: { + const char *zRet = "normal"; + int eMode = getLockingMode(zRight); + + if( pId2->n==0 && eMode==PAGER_LOCKINGMODE_QUERY ){ + /* Simple "PRAGMA locking_mode;" statement. This is a query for + ** the current default locking mode (which may be different to + ** the locking-mode of the main database). + */ + eMode = db->dfltLockMode; + }else{ + Pager *pPager; + if( pId2->n==0 ){ + /* This indicates that no database name was specified as part + ** of the PRAGMA command. In this case the locking-mode must be + ** set on all attached databases, as well as the main db file. + ** + ** Also, the sqlite3.dfltLockMode variable is set so that + ** any subsequently attached databases also use the specified + ** locking mode. + */ + int ii; + assert(pDb==&db->aDb[0]); + for(ii=2; iinDb; ii++){ + pPager = sqlite3BtreePager(db->aDb[ii].pBt); + sqlite3PagerLockingMode(pPager, eMode); + } + db->dfltLockMode = (u8)eMode; + } + pPager = sqlite3BtreePager(pDb->pBt); + eMode = sqlite3PagerLockingMode(pPager, eMode); + } + + assert( eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ + zRet = "exclusive"; + } + returnSingleText(v, zRet); + break; + } + + /* + ** PRAGMA [schema.]journal_mode + ** PRAGMA [schema.]journal_mode = + ** (delete|persist|off|truncate|memory|wal|off) + */ + case PragTyp_JOURNAL_MODE: { + int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ + int ii; /* Loop counter */ + + if( zRight==0 ){ + /* If there is no "=MODE" part of the pragma, do a query for the + ** current mode */ + eMode = PAGER_JOURNALMODE_QUERY; + }else{ + const char *zMode; + int n = sqlite3Strlen30(zRight); + for(eMode=0; (zMode = sqlite3JournalModename(eMode))!=0; eMode++){ + if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; + } + if( !zMode ){ + /* If the "=MODE" part does not match any known journal mode, + ** then do a query */ + eMode = PAGER_JOURNALMODE_QUERY; + } + if( eMode==PAGER_JOURNALMODE_OFF && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow journal-mode "OFF" in defensive since the database + ** can become corrupted using ordinary SQL when the journal is off */ + eMode = PAGER_JOURNALMODE_QUERY; + } + } + if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ + /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ + iDb = 0; + pId2->n = 1; + } + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3VdbeUsesBtree(v, ii); + sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + } + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + break; + } + + /* + ** PRAGMA [schema.]journal_size_limit + ** PRAGMA [schema.]journal_size_limit=N + ** + ** Get or set the size limit on rollback journal files. + */ + case PragTyp_JOURNAL_SIZE_LIMIT: { + Pager *pPager = sqlite3BtreePager(pDb->pBt); + i64 iLimit = -2; + if( zRight ){ + sqlite3DecOrHexToI64(zRight, &iLimit); + if( iLimit<-1 ) iLimit = -1; + } + iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); + returnSingleInt(v, iLimit); + break; + } + +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + + /* + ** PRAGMA [schema.]auto_vacuum + ** PRAGMA [schema.]auto_vacuum=N + ** + ** Get or set the value of the database 'auto-vacuum' parameter. + ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_AUTO_VACUUM: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + returnSingleInt(v, sqlite3BtreeGetAutoVacuum(pBt)); + }else{ + int eAuto = getAutoVacuum(zRight); + assert( eAuto>=0 && eAuto<=2 ); + db->nextAutovac = (u8)eAuto; + /* Call SetAutoVacuum() to set initialize the internal auto and + ** incr-vacuum flags. This is required in case this connection + ** creates the database file. It is important that it is created + ** as an auto-vacuum capable db. + */ + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ + /* When setting the auto_vacuum mode to either "full" or + ** "incremental", write the value of meta[6] in the database + ** file. Before writing to meta[6], check that meta[3] indicates + ** that this really is an auto-vacuum capable database. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList setMeta6[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, + { OP_If, 1, 0, 0}, /* 2 */ + { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 0}, /* 4 */ + }; + VdbeOp *aOp; + int iAddr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[2].p2 = iAddr+4; + aOp[4].p1 = iDb; + aOp[4].p3 = eAuto - 1; + sqlite3VdbeUsesBtree(v, iDb); + } + } + break; + } +#endif + + /* + ** PRAGMA [schema.]incremental_vacuum(N) + ** + ** Do N steps of incremental vacuuming on a database. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_INCREMENTAL_VACUUM: { + int iLimit, addr; + if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ + iLimit = 0x7fffffff; + } + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_ResultRow, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + break; + } +#endif + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + /* + ** PRAGMA [schema.]cache_size + ** PRAGMA [schema.]cache_size=N + ** + ** The first form reports the current local setting for the + ** page cache size. The second form sets the local + ** page cache size value. If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + */ + case PragTyp_CACHE_SIZE: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, pDb->pSchema->cache_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } + + /* + ** PRAGMA [schema.]cache_spill + ** PRAGMA cache_spill=BOOLEAN + ** PRAGMA [schema.]cache_spill=N + ** + ** The first form reports the current local setting for the + ** page cache spill size. The second form turns cache spill on + ** or off. When turnning cache spill on, the size is set to the + ** current cache_size. The third form sets a spill size that + ** may be different form the cache size. + ** If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + ** + ** If the number of cache_spill pages is less then the number of + ** cache_size pages, no spilling occurs until the page count exceeds + ** the number of cache_size pages. + ** + ** The cache_spill=BOOLEAN setting applies to all attached schemas, + ** not just the schema specified. + */ + case PragTyp_CACHE_SPILL: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, + (db->flags & SQLITE_CacheSpill)==0 ? 0 : + sqlite3BtreeSetSpillSize(pDb->pBt,0)); + }else{ + int size = 1; + if( sqlite3GetInt32(zRight, &size) ){ + sqlite3BtreeSetSpillSize(pDb->pBt, size); + } + if( sqlite3GetBoolean(zRight, size!=0) ){ + db->flags |= SQLITE_CacheSpill; + }else{ + db->flags &= ~(u64)SQLITE_CacheSpill; + } + setAllPagerFlags(db); + } + break; + } + + /* + ** PRAGMA [schema.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + case PragTyp_MMAP_SIZE: { + sqlite3_int64 sz; +#if SQLITE_MAX_MMAP_SIZE>0 + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3DecOrHexToI64(zRight, &sz); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_MMAP_SIZE, &sz); +#else + sz = 0; + rc = SQLITE_OK; +#endif + if( rc==SQLITE_OK ){ + returnSingleInt(v, sz); + }else if( rc!=SQLITE_NOTFOUND ){ + pParse->nErr++; + pParse->rc = rc; + } + break; + } + + /* + ** PRAGMA temp_store + ** PRAGMA temp_store = "default"|"memory"|"file" + ** + ** Return or set the local value of the temp_store flag. Changing + ** the local value does not make changes to the disk file and the default + ** value will be restored the next time the database is opened. + ** + ** Note that it is possible for the library compile-time options to + ** override this setting + */ + case PragTyp_TEMP_STORE: { + if( !zRight ){ + returnSingleInt(v, db->temp_store); + }else{ + changeTempStorage(pParse, zRight); + } + break; + } + + /* + ** PRAGMA temp_store_directory + ** PRAGMA temp_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the temp_store_directory flag. Changing + ** the value sets a specific directory to be used for temporary files. + ** Setting to a null string reverts to the default temporary directory search. + ** If temporary directory is changed, then invalidateTempStorage. + ** + */ + case PragTyp_TEMP_STORE_DIRECTORY: { + if( !zRight ){ + returnSingleText(v, sqlite3_temp_directory); + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + if( SQLITE_TEMP_STORE==0 + || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) + || (SQLITE_TEMP_STORE==2 && db->temp_store==1) + ){ + invalidateTempStorage(pParse); + } + sqlite3_free(sqlite3_temp_directory); + if( zRight[0] ){ + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_temp_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } + +#if SQLITE_OS_WIN + /* + ** PRAGMA data_store_directory + ** PRAGMA data_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the data_store_directory flag. Changing + ** the value sets a specific directory to be used for database files that + ** were specified with a relative pathname. Setting to a null string reverts + ** to the default database directory, which for database files specified with + ** a relative path will probably be based on the current directory for the + ** process. Database file specified with an absolute path are not impacted + ** by this setting, regardless of its value. + ** + */ + case PragTyp_DATA_STORE_DIRECTORY: { + if( !zRight ){ + returnSingleText(v, sqlite3_data_directory); + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + sqlite3_free(sqlite3_data_directory); + if( zRight[0] ){ + sqlite3_data_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_data_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE + /* + ** PRAGMA [schema.]lock_proxy_file + ** PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + case PragTyp_LOCK_PROXY_FILE: { + if( !zRight ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + char *proxy_file_path = NULL; + sqlite3_file *pFile = sqlite3PagerFile(pPager); + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, + &proxy_file_path); + returnSingleText(v, proxy_file_path); + }else{ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + sqlite3_file *pFile = sqlite3PagerFile(pPager); + int res; + if( zRight[0] ){ + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + zRight); + } else { + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + NULL); + } + if( res!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); + goto pragma_out; + } + } + break; + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + + /* + ** PRAGMA [schema.]synchronous + ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA + ** + ** Return or set the local value of the synchronous flag. Changing + ** the local value does not make changes to the disk file and the + ** default value will be restored the next time the database is + ** opened. + */ + case PragTyp_SYNCHRONOUS: { + if( !zRight ){ + returnSingleInt(v, pDb->safety_level-1); + }else{ + if( !db->autoCommit ){ + sqlite3ErrorMsg(pParse, + "Safety level may not be changed inside a transaction"); + }else if( iDb!=1 ){ + int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; + if( iLevel==0 ) iLevel = 1; + pDb->safety_level = iLevel; + pDb->bSyncSet = 1; + setAllPagerFlags(db); + } + } + break; + } +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_FLAG_PRAGMAS + case PragTyp_FLAG: { + if( zRight==0 ){ + setPragmaResultColumnNames(v, pPragma); + returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); + }else{ + u64 mask = pPragma->iArg; /* Mask of bits to set or clear. */ + if( db->autoCommit==0 ){ + /* Foreign key support may not be enabled or disabled while not + ** in auto-commit mode. */ + mask &= ~(SQLITE_ForeignKeys); + } +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevel==UAUTH_User ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif + + if( sqlite3GetBoolean(zRight, 0) ){ + db->flags |= mask; + }else{ + db->flags &= ~mask; + if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp0(v, OP_Expire); + setAllPagerFlags(db); + } + break; + } +#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ + +#ifndef SQLITE_OMIT_SCHEMA_PRAGMAS + /* + ** PRAGMA table_info(
) + ** + ** Return a single row for each column of the named table. The columns of + ** the returned data set are: + ** + ** cid: Column id (numbered from left to right, starting at 0) + ** name: Column name + ** type: Column declaration type. + ** notnull: True if 'NOT NULL' is part of column declaration + ** dflt_value: The default value for the column, if any. + ** pk: Non-zero for PK fields. + */ + case PragTyp_TABLE_INFO: if( zRight ){ + Table *pTab; + sqlite3CodeVerifyNamedSchema(pParse, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab ){ + int i, k; + int nHidden = 0; + Column *pCol; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + pParse->nMem = 7; + sqlite3ViewGetColumnNames(pParse, pTab); + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + int isHidden = 0; + if( pCol->colFlags & COLFLAG_NOINSERT ){ + if( pPragma->iArg==0 ){ + nHidden++; + continue; + } + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + isHidden = 2; /* GENERATED ALWAYS AS ... VIRTUAL */ + }else if( pCol->colFlags & COLFLAG_STORED ){ + isHidden = 3; /* GENERATED ALWAYS AS ... STORED */ + }else{ assert( pCol->colFlags & COLFLAG_HIDDEN ); + isHidden = 1; /* HIDDEN */ + } + } + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} + } + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN || isHidden>=2 ); + sqlite3VdbeMultiLoad(v, 1, pPragma->iArg ? "issisii" : "issisi", + i-nHidden, + pCol->zName, + sqlite3ColumnType(pCol,""), + pCol->notNull ? 1 : 0, + pCol->pDflt && isHidden<2 ? pCol->pDflt->u.zToken : 0, + k, + isHidden); + } + } + } + break; + +#ifdef SQLITE_DEBUG + case PragTyp_STATS: { + Index *pIdx; + HashElem *i; + pParse->nMem = 5; + sqlite3CodeVerifySchema(pParse, iDb); + for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + sqlite3VdbeMultiLoad(v, 1, "ssiii", + pTab->zName, + 0, + pTab->szTabRow, + pTab->nRowLogEst, + pTab->tabFlags); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeMultiLoad(v, 2, "siiiX", + pIdx->zName, + pIdx->szIdxRow, + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); + } + } + } + break; +#endif + + case PragTyp_INDEX_INFO: if( zRight ){ + Index *pIdx; + Table *pTab; + pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + /* If there is no index named zRight, check to see if there is a + ** WITHOUT ROWID table named zRight, and if there is, show the + ** structure of the PRIMARY KEY index for that table. */ + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab && !HasRowid(pTab) ){ + pIdx = sqlite3PrimaryKeyIndex(pTab); + } + } + if( pIdx ){ + int iIdxDb = sqlite3SchemaToIndex(db, pIdx->pSchema); + int i; + int mx; + if( pPragma->iArg ){ + /* PRAGMA index_xinfo (newer version with more rows and columns) */ + mx = pIdx->nColumn; + pParse->nMem = 6; + }else{ + /* PRAGMA index_info (legacy version) */ + mx = pIdx->nKeyCol; + pParse->nMem = 3; + } + pTab = pIdx->pTable; + sqlite3CodeVerifySchema(pParse, iIdxDb); + assert( pParse->nMem<=pPragma->nPragCName ); + for(i=0; iaiColumn[i]; + sqlite3VdbeMultiLoad(v, 1, "iisX", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zName); + if( pPragma->iArg ){ + sqlite3VdbeMultiLoad(v, 4, "isiX", + pIdx->aSortOrder[i], + pIdx->azColl[i], + inKeyCol); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); + } + } + } + break; + + case PragTyp_INDEX_LIST: if( zRight ){ + Index *pIdx; + Table *pTab; + int i; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pParse->nMem = 5; + sqlite3CodeVerifySchema(pParse, iTabDb); + for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ + const char *azOrigin[] = { "c", "u", "pk" }; + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); + } + } + } + break; + + case PragTyp_DATABASE_LIST: { + int i; + pParse->nMem = 3; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt==0 ) continue; + assert( db->aDb[i].zDbSName!=0 ); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zDbSName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); + } + } + break; + + case PragTyp_COLLATION_LIST: { + int i = 0; + HashElem *p; + pParse->nMem = 2; + for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(p); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); + } + } + break; + +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + case PragTyp_FUNCTION_LIST: { + int i; + HashElem *j; + FuncDef *p; + int showInternFunc = (db->mDbFlags & DBFLAG_InternalFunc)!=0; + pParse->nMem = 6; + for(i=0; iu.pHash ){ + pragmaFunclistLine(v, p, 1, showInternFunc); + } + } + for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){ + p = (FuncDef*)sqliteHashData(j); + pragmaFunclistLine(v, p, 0, showInternFunc); + } + } + break; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + case PragTyp_MODULE_LIST: { + HashElem *j; + pParse->nMem = 1; + for(j=sqliteHashFirst(&db->aModule); j; j=sqliteHashNext(j)){ + Module *pMod = (Module*)sqliteHashData(j); + sqlite3VdbeMultiLoad(v, 1, "s", pMod->zName); + } + } + break; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + case PragTyp_PRAGMA_LIST: { + int i; + for(i=0; ipFKey; + if( pFK ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + int i = 0; + pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iTabDb); + while(pFK){ + int j; + for(j=0; jnCol; j++){ + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); + } + ++i; + pFK = pFK->pNextFrom; + } + } + } + } + break; +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + case PragTyp_FOREIGN_KEY_CHECK: { + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeLoadString(v, regResult, pTab->zName); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + if( pParent==0 ) continue; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + } + }else{ + k = 0; + break; + } + } + assert( pParse->nErr>0 || pFK==0 ); + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + pIdx = 0; + aiCols = 0; + if( pParent ){ + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 || db->mallocFailed ); + } + addrOk = sqlite3VdbeMakeLabel(pParse); + + /* Generate code to read the child key values into registers + ** regRow..regRow+n. If any of the child key values are NULL, this + ** row cannot cause an FK violation. Jump directly to addrOk in + ** this case. */ + for(j=0; jnCol; j++){ + int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + + /* Generate code to query the parent index for a matching parent + ** key. If a match is found, jump to addrOk. */ + if( pIdx ){ + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); + }else if( pParent ){ + int jmp = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v); + sqlite3VdbeGoto(v, addrOk); + assert( pFK->nCol==1 || db->mallocFailed ); + } + + /* Generate code to report an FK violation to the caller. */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1); + } + sqlite3VdbeMultiLoad(v, regResult+2, "siX", pFK->zTo, i-1); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + } + } + break; +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA + /* Reinstall the LIKE and GLOB functions. The variant of LIKE + ** used will be case sensitive or not depending on the RHS. + */ + case PragTyp_CASE_SENSITIVE_LIKE: { + if( zRight ){ + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight, 0)); + } + } + break; +#endif /* SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA */ + +#ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX +# define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 +#endif + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of + ** integrity_check designed to detect most database corruption + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time wherease integrity_check is O(NlogN). + ** + ** The maximum nubmer of errors is 100 by default. A different default + ** can be specified using a numeric parameter N. + ** + ** Or, the parameter N can be the name of a table. In that case, only + ** the one table named is verified. The freelist is only verified if + ** the named table is "sqlite_schema" (or one of its aliases). + ** + ** All schemas are checked by default. To check just a single + ** schema, use the form: + ** + ** PRAGMA schema.integrity_check; + */ + case PragTyp_INTEGRITY_CHECK: { + int i, j, addr, mxErr; + Table *pObjTab = 0; /* Check only this one table, if not NULL */ + + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); + + /* If the PRAGMA command was of the form "PRAGMA .integrity_check", + ** then iDb is set to the index of the database identified by . + ** In this case, the integrity of database iDb only is verified by + ** the VDBE created below. + ** + ** Otherwise, if the command was simply "PRAGMA integrity_check" (or + ** "PRAGMA quick_check"), then iDb is set to 0. In this case, set iDb + ** to -1 here, to indicate that the VDBE should verify the integrity + ** of all attached databases. */ + assert( iDb>=0 ); + assert( iDb==0 || pId2->z ); + if( pId2->z==0 ) iDb = -1; + + /* Initialize the VDBE program */ + pParse->nMem = 6; + + /* Set the maximum error count */ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + if( zRight ){ + if( sqlite3GetInt32(zRight, &mxErr) ){ + if( mxErr<=0 ){ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + } + }else{ + pObjTab = sqlite3LocateTable(pParse, 0, zRight, + iDb>=0 ? db->aDb[iDb].zDbSName : 0); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ + + /* Do an integrity check on each database file */ + for(i=0; inDb; i++){ + HashElem *x; /* For looping over tables in the schema */ + Hash *pTbls; /* Set of all tables in the schema */ + int *aRoot; /* Array of root page numbers of all btrees */ + int cnt = 0; /* Number of entries in aRoot[] */ + int mxIdx = 0; /* Maximum number of indexes for any table */ + + if( OMIT_TEMPDB && i==1 ) continue; + if( iDb>=0 && i!=iDb ) continue; + + sqlite3CodeVerifySchema(pParse, i); + + /* Do an integrity check of the B-Tree + ** + ** Begin by finding the root pages numbers + ** for all tables and indices in the database. + */ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + pTbls = &db->aDb[i].pSchema->tblHash; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); /* Current table */ + Index *pIdx; /* An index on pTab */ + int nIdx; /* Number of indexes on pTab */ + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + if( cnt==0 ) continue; + if( pObjTab ) cnt++; + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + cnt = 0; + if( pObjTab ) aRoot[++cnt] = 0; + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + aRoot[++cnt] = pIdx->tnum; + } + } + aRoot[0] = cnt; + + /* Make sure sufficient number of registers have been allocated */ + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3ClearTempRegCache(pParse); + + /* Do the b-tree integrity checks */ + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeChangeP5(v, (u8)i); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Concat, 2, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + + /* Make sure all the indices are constructed correctly. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx, *pPk; + Index *pPrior = 0; + int loopTop; + int iDataCur, iIdxCur; + int r1 = -1; + + if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ + if( pObjTab && pObjTab!=pTab ) continue; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, + 1, 0, &iDataCur, &iIdxCur); + /* reg[7] counts the number of entries in the table. + ** reg[8+i] counts the number of entries in the i-th index + */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ + } + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); + sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); + loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + if( !isQuick ){ + /* Sanity check on record header decoding */ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } + /* Verify that all NOT NULL columns really are NOT NULL */ + for(j=0; jnCol; j++){ + char *zErr; + int jmp2; + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, jmp2); + } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0); + if( db->mallocFailed==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(pParse); + int addrCkOk = sqlite3VdbeMakeLabel(pParse); + char *zErr; + int k; + pParse->iSelfTab = iDataCur + 1; + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + pParse->iSelfTab = 0; + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, addrCkOk); + } + sqlite3ExprListDelete(db, pCheck); + } + if( !isQuick ){ /* Omit the remaining tests for quick_check */ + /* Validate index entries for the current row */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(pParse); + if( pPk==pIdx ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; + sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + pIdx->nColumn); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " missing from index "); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp4 = integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(pParse); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeGoto(v, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); + sqlite3ResolvePartIdxLabel(pParse, jmp3); + } + } + sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, loopTop-1); + if( !isQuick ){ + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + } + } + } + { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, /* 3 */ + { OP_Halt, 0, 0, 0}, /* 4 */ + { OP_String8, 0, 3, 0}, /* 5 */ + { OP_Goto, 0, 3, 0}, /* 6 */ + }; + VdbeOp *aOp; + + aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + if( aOp ){ + aOp[0].p2 = 1-mxErr; + aOp[2].p4type = P4_STATIC; + aOp[2].p4.z = "ok"; + aOp[5].p4type = P4_STATIC; + aOp[5].p4.z = (char*)sqlite3ErrStr(SQLITE_CORRUPT); + } + sqlite3VdbeChangeP3(v, 0, sqlite3VdbeCurrentAddr(v)-2); + } + } + break; +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_UTF16 + /* + ** PRAGMA encoding + ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" + ** + ** In its first form, this pragma returns the encoding of the main + ** database. If the database is not initialized, it is initialized now. + ** + ** The second form of this pragma is a no-op if the main database file + ** has not already been initialized. In this case it sets the default + ** encoding that will be used for the main database file if a new file + ** is created. If an existing main database file is opened, then the + ** default text encoding for the existing database is used. + ** + ** In all cases new databases created using the ATTACH command are + ** created to use the same default text encoding as the main database. If + ** the main database has not been initialized and/or created when ATTACH + ** is executed, this is done before the ATTACH operation. + ** + ** In the second form this pragma sets the text encoding to be used in + ** new database files created using this database handle. It is only + ** useful if invoked immediately after the main database i + */ + case PragTyp_ENCODING: { + static const struct EncName { + char *zName; + u8 enc; + } encnames[] = { + { "UTF8", SQLITE_UTF8 }, + { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ + { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ + { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ + { "UTF16le", SQLITE_UTF16LE }, + { "UTF16be", SQLITE_UTF16BE }, + { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ + { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ + { 0, 0 } + }; + const struct EncName *pEnc; + if( !zRight ){ /* "PRAGMA encoding" */ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); + assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); + assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); + returnSingleText(v, encnames[ENC(pParse->db)].zName); + }else{ /* "PRAGMA encoding = XXX" */ + /* Only change the value of sqlite.enc if the database handle is not + ** initialized. If the main database exists, the new sqlite.enc value + ** will be overwritten when the schema is next loaded. If it does not + ** already exists, it will be created to use the new encoding value. + */ + if( (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ + for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ + if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ + u8 enc = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = enc; + sqlite3SetTextEncoding(db, enc); + break; + } + } + if( !pEnc->zName ){ + sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight); + } + } + } + } + break; +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + /* + ** PRAGMA [schema.]schema_version + ** PRAGMA [schema.]schema_version = + ** + ** PRAGMA [schema.]user_version + ** PRAGMA [schema.]user_version = + ** + ** PRAGMA [schema.]freelist_count + ** + ** PRAGMA [schema.]data_version + ** + ** PRAGMA [schema.]application_id + ** PRAGMA [schema.]application_id = + ** + ** The pragma's schema_version and user_version are used to set or get + ** the value of the schema-version and user-version, respectively. Both + ** the schema-version and the user-version are 32-bit signed integers + ** stored in the database header. + ** + ** The schema-cookie is usually only manipulated internally by SQLite. It + ** is incremented by SQLite whenever the database schema is modified (by + ** creating or dropping a table or index). The schema version is used by + ** SQLite each time a query is executed to ensure that the internal cache + ** of the schema used when compiling the SQL query matches the schema of + ** the database against which the compiled query is actually executed. + ** Subverting this mechanism by using "PRAGMA schema_version" to modify + ** the schema-version is potentially dangerous and may lead to program + ** crashes or database corruption. Use with caution! + ** + ** The user-version is not used internally by SQLite. It may be used by + ** applications for any purpose. + */ + case PragTyp_HEADER_VALUE: { + int iCookie = pPragma->iArg; /* Which cookie to read or write */ + sqlite3VdbeUsesBtree(v, iDb); + if( zRight && (pPragma->mPragFlg & PragFlg_ReadOnly)==0 ){ + /* Write the specified cookie value */ + static const VdbeOpList setCookie[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_SetCookie, 0, 0, 0}, /* 1 */ + }; + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p2 = iCookie; + aOp[1].p3 = sqlite3Atoi(zRight); + aOp[1].p5 = 1; + }else{ + /* Read the specified cookie value */ + static const VdbeOpList readCookie[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, 0}, /* 1 */ + { OP_ResultRow, 1, 1, 0} + }; + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p3 = iCookie; + sqlite3VdbeReusable(v); + } + } + break; +#endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + /* + ** PRAGMA compile_options + ** + ** Return the names of all compile-time options used in this build, + ** one option per row. + */ + case PragTyp_COMPILE_OPTIONS: { + int i = 0; + const char *zOpt; + pParse->nMem = 1; + while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ + sqlite3VdbeLoadString(v, 1, zOpt); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + sqlite3VdbeReusable(v); + } + break; +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +#ifndef SQLITE_OMIT_WAL + /* + ** PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate + ** + ** Checkpoint the database. + */ + case PragTyp_WAL_CHECKPOINT: { + int iBt = (pId2->z?iDb:SQLITE_MAX_DB); + int eMode = SQLITE_CHECKPOINT_PASSIVE; + if( zRight ){ + if( sqlite3StrICmp(zRight, "full")==0 ){ + eMode = SQLITE_CHECKPOINT_FULL; + }else if( sqlite3StrICmp(zRight, "restart")==0 ){ + eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; + } + } + pParse->nMem = 3; + sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + break; + + /* + ** PRAGMA wal_autocheckpoint + ** PRAGMA wal_autocheckpoint = N + ** + ** Configure a database connection to automatically checkpoint a database + ** after accumulating N frames in the log. Or query for the current value + ** of N. + */ + case PragTyp_WAL_AUTOCHECKPOINT: { + if( zRight ){ + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); + } + returnSingleInt(v, + db->xWalCallback==sqlite3WalDefaultHook ? + SQLITE_PTR_TO_INT(db->pWalArg) : 0); + } + break; +#endif + + /* + ** PRAGMA shrink_memory + ** + ** IMPLEMENTATION-OF: R-23445-46109 This pragma causes the database + ** connection on which it is invoked to free up as much memory as it + ** can, by calling sqlite3_db_release_memory(). + */ + case PragTyp_SHRINK_MEMORY: { + sqlite3_db_release_memory(db); + break; + } + + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all + ** of the optimizations listed above except Debug Mode, including new + ** optimizations that have not yet been invented. If new optimizations are + ** ever added that should be off by default, those off-by-default + ** optimizations will have bitmasks of 0x10000 or larger. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (3) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xfffe; + } + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; /* Always analyze if any index lacks statistics */ + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + break; + } + + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + /*case PragTyp_BUSY_TIMEOUT*/ default: { + assert( pPragma->ePragTyp==PragTyp_BUSY_TIMEOUT ); + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(v, db->busyTimeout); + break; + } + + /* + ** PRAGMA soft_heap_limit + ** PRAGMA soft_heap_limit = N + ** + ** IMPLEMENTATION-OF: R-26343-45930 This pragma invokes the + ** sqlite3_soft_heap_limit64() interface with the argument N, if N is + ** specified and is a non-negative integer. + ** IMPLEMENTATION-OF: R-64451-07163 The soft_heap_limit pragma always + ** returns the same integer that would be returned by the + ** sqlite3_soft_heap_limit64(-1) C-language function. + */ + case PragTyp_SOFT_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_soft_heap_limit64(N); + } + returnSingleInt(v, sqlite3_soft_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA hard_heap_limit + ** PRAGMA hard_heap_limit = N + ** + ** Invoke sqlite3_hard_heap_limit64() to query or set the hard heap + ** limit. The hard heap limit can be activated or lowered by this + ** pragma, but not raised or deactivated. Only the + ** sqlite3_hard_heap_limit64() C-language API can raise or deactivate + ** the hard heap limit. This allows an application to set a heap limit + ** constraint that cannot be relaxed by an untrusted SQL script. + */ + case PragTyp_HARD_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); + if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); + } + returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); + } + returnSingleInt(v, sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + break; + } + + /* + ** PRAGMA analysis_limit + ** PRAGMA analysis_limit = N + ** + ** Configure the maximum number of rows that ANALYZE will examine + ** in each index that it looks at. Return the new limit. + */ + case PragTyp_ANALYSIS_LIMIT: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + db->nAnalysisLimit = (int)(N&0x7fffffff); + } + returnSingleInt(v, db->nAnalysisLimit); + break; + } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* + ** Report the current state of file logs for all databases + */ + case PragTyp_LOCK_STATUS: { + static const char *const azLockName[] = { + "unlocked", "shared", "reserved", "pending", "exclusive" + }; + int i; + pParse->nMem = 2; + for(i=0; inDb; i++){ + Btree *pBt; + const char *zState = "unknown"; + int j; + if( db->aDb[i].zDbSName==0 ) continue; + pBt = db->aDb[i].pBt; + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ + zState = "closed"; + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, + SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ + zState = azLockName[j]; + } + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); + } + break; + } +#endif + +#if defined(SQLITE_ENABLE_CEROD) + case PragTyp_ACTIVATE_EXTENSIONS: if( zRight ){ + if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ + sqlite3_activate_cerod(&zRight[6]); + } + } + break; +#endif + + } /* End of the PRAGMA switch */ + + /* The following block is a no-op unless SQLITE_DEBUG is defined. Its only + ** purpose is to execute assert() statements to verify that if the + ** PragFlg_NoColumns1 flag is set and the caller specified an argument + ** to the PRAGMA, the implementation has not added any OP_ResultRow + ** instructions to the VM. */ + if( (pPragma->mPragFlg & PragFlg_NoColumns1) && zRight ){ + sqlite3VdbeVerifyNoResultRow(v); + } + +pragma_out: + sqlite3DbFree(db, zLeft); + sqlite3DbFree(db, zRight); +} +#ifndef SQLITE_OMIT_VIRTUALTABLE +/***************************************************************************** +** Implementation of an eponymous virtual table that runs a pragma. +** +*/ +typedef struct PragmaVtab PragmaVtab; +typedef struct PragmaVtabCursor PragmaVtabCursor; +struct PragmaVtab { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection to which it belongs */ + const PragmaName *pName; /* Name of the pragma */ + u8 nHidden; /* Number of hidden columns */ + u8 iHidden; /* Index of the first hidden column */ +}; +struct PragmaVtabCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pPragma; /* The pragma statement to run */ + sqlite_int64 iRowid; /* Current rowid */ + char *azArg[2]; /* Value of the argument and schema */ +}; + +/* +** Pragma virtual table module xConnect method. +*/ +static int pragmaVtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + const PragmaName *pPragma = (const PragmaName*)pAux; + PragmaVtab *pTab = 0; + int rc; + int i, j; + char cSep = '('; + StrAccum acc; + char zBuf[200]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3_str_appendall(&acc, "CREATE TABLE x"); + for(i=0, j=pPragma->iPragCName; inPragCName; i++, j++){ + sqlite3_str_appendf(&acc, "%c\"%s\"", cSep, pragCName[j]); + cSep = ','; + } + if( i==0 ){ + sqlite3_str_appendf(&acc, "(\"%s\"", pPragma->zName); + i++; + } + j = 0; + if( pPragma->mPragFlg & PragFlg_Result1 ){ + sqlite3_str_appendall(&acc, ",arg HIDDEN"); + j++; + } + if( pPragma->mPragFlg & (PragFlg_SchemaOpt|PragFlg_SchemaReq) ){ + sqlite3_str_appendall(&acc, ",schema HIDDEN"); + j++; + } + sqlite3_str_append(&acc, ")", 1); + sqlite3StrAccumFinish(&acc); + assert( strlen(zBuf) < sizeof(zBuf)-1 ); + rc = sqlite3_declare_vtab(db, zBuf); + if( rc==SQLITE_OK ){ + pTab = (PragmaVtab*)sqlite3_malloc(sizeof(PragmaVtab)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(PragmaVtab)); + pTab->pName = pPragma; + pTab->db = db; + pTab->iHidden = i; + pTab->nHidden = j; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Pragma virtual table module xDisconnect method. +*/ +static int pragmaVtabDisconnect(sqlite3_vtab *pVtab){ + PragmaVtab *pTab = (PragmaVtab*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* Figure out the best index to use to search a pragma virtual table. +** +** There are not really any index choices. But we want to encourage the +** query planner to give == constraints on as many hidden parameters as +** possible, and especially on the first hidden parameter. So return a +** high cost if hidden parameters are unconstrained. +*/ +static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + PragmaVtab *pTab = (PragmaVtab*)tab; + const struct sqlite3_index_constraint *pConstraint; + int i, j; + int seen[2]; + + pIdxInfo->estimatedCost = (double)1; + if( pTab->nHidden==0 ){ return SQLITE_OK; } + pConstraint = pIdxInfo->aConstraint; + seen[0] = 0; + seen[1] = 0; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn < pTab->iHidden ) continue; + j = pConstraint->iColumn - pTab->iHidden; + assert( j < 2 ); + seen[j] = i+1; + } + if( seen[0]==0 ){ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + return SQLITE_OK; + } + j = seen[0]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 1; + pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1]==0 ) return SQLITE_OK; + pIdxInfo->estimatedCost = (double)20; + pIdxInfo->estimatedRows = 20; + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + return SQLITE_OK; +} + +/* Create a new cursor for the pragma virtual table */ +static int pragmaVtabOpen(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ + PragmaVtabCursor *pCsr; + pCsr = (PragmaVtabCursor*)sqlite3_malloc(sizeof(*pCsr)); + if( pCsr==0 ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(PragmaVtabCursor)); + pCsr->base.pVtab = pVtab; + *ppCursor = &pCsr->base; + return SQLITE_OK; +} + +/* Clear all content from pragma virtual table cursor. */ +static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ + int i; + sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + for(i=0; iazArg); i++){ + sqlite3_free(pCsr->azArg[i]); + pCsr->azArg[i] = 0; + } +} + +/* Close a pragma virtual table cursor */ +static int pragmaVtabClose(sqlite3_vtab_cursor *cur){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)cur; + pragmaVtabCursorClear(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* Advance the pragma virtual table cursor to the next row */ +static int pragmaVtabNext(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + int rc = SQLITE_OK; + + /* Increment the xRowid value */ + pCsr->iRowid++; + assert( pCsr->pPragma ); + if( SQLITE_ROW!=sqlite3_step(pCsr->pPragma) ){ + rc = sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + pragmaVtabCursorClear(pCsr); + } + return rc; +} + +/* +** Pragma virtual table module xFilter method. +*/ +static int pragmaVtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + int rc; + int i, j; + StrAccum acc; + char *zSql; + + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + pragmaVtabCursorClear(pCsr); + j = (pTab->pName->mPragFlg & PragFlg_Result1)!=0 ? 0 : 1; + for(i=0; iazArg) ); + assert( pCsr->azArg[j]==0 ); + if( zText ){ + pCsr->azArg[j] = sqlite3_mprintf("%s", zText); + if( pCsr->azArg[j]==0 ){ + return SQLITE_NOMEM; + } + } + } + sqlite3StrAccumInit(&acc, 0, 0, 0, pTab->db->aLimit[SQLITE_LIMIT_SQL_LENGTH]); + sqlite3_str_appendall(&acc, "PRAGMA "); + if( pCsr->azArg[1] ){ + sqlite3_str_appendf(&acc, "%Q.", pCsr->azArg[1]); + } + sqlite3_str_appendall(&acc, pTab->pName->zName); + if( pCsr->azArg[0] ){ + sqlite3_str_appendf(&acc, "=%Q", pCsr->azArg[0]); + } + zSql = sqlite3StrAccumFinish(&acc); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pPragma, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + return rc; + } + return pragmaVtabNext(pVtabCursor); +} + +/* +** Pragma virtual table module xEof method. +*/ +static int pragmaVtabEof(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + return (pCsr->pPragma==0); +} + +/* The xColumn method simply returns the corresponding column from +** the PRAGMA. +*/ +static int pragmaVtabColumn( + sqlite3_vtab_cursor *pVtabCursor, + sqlite3_context *ctx, + int i +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + if( iiHidden ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pPragma, i)); + }else{ + sqlite3_result_text(ctx, pCsr->azArg[i-pTab->iHidden],-1,SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Pragma virtual table module xRowid method. +*/ +static int pragmaVtabRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *p){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + *p = pCsr->iRowid; + return SQLITE_OK; +} + +/* The pragma virtual table object */ +static const sqlite3_module pragmaVtabModule = { + 0, /* iVersion */ + 0, /* xCreate - create a table */ + pragmaVtabConnect, /* xConnect - connect to an existing table */ + pragmaVtabBestIndex, /* xBestIndex - Determine search strategy */ + pragmaVtabDisconnect, /* xDisconnect - Disconnect from a table */ + 0, /* xDestroy - Drop a table */ + pragmaVtabOpen, /* xOpen - open a cursor */ + pragmaVtabClose, /* xClose - close a cursor */ + pragmaVtabFilter, /* xFilter - configure scan constraints */ + pragmaVtabNext, /* xNext - advance a cursor */ + pragmaVtabEof, /* xEof */ + pragmaVtabColumn, /* xColumn - read data */ + pragmaVtabRowid, /* xRowid - read data */ + 0, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +/* +** Check to see if zTabName is really the name of a pragma. If it is, +** then register an eponymous virtual table for that pragma and return +** a pointer to the Module object for the new virtual table. +*/ +Module *sqlite3PragmaVtabRegister(sqlite3 *db, const char *zName){ + const PragmaName *pName; + assert( sqlite3_strnicmp(zName, "pragma_", 7)==0 ); + pName = pragmaLocate(zName+7); + if( pName==0 ) return 0; + if( (pName->mPragFlg & (PragFlg_Result0|PragFlg_Result1))==0 ) return 0; + assert( sqlite3HashFind(&db->aModule, zName)==0 ); + return sqlite3VtabCreateModule(db, zName, &pragmaVtabModule, (void*)pName, 0); +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#endif /* SQLITE_OMIT_PRAGMA */ diff --git a/third_party/sqlite3/pragma.inc b/third_party/sqlite3/pragma.inc new file mode 100644 index 000000000..1d2187362 --- /dev/null +++ b/third_party/sqlite3/pragma.inc @@ -0,0 +1,649 @@ +/* DO NOT EDIT! +** This file is automatically generated by the script at +** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit +** that script and rerun it. +*/ +/* clang-format off */ + +/* The various pragma types */ +#define PragTyp_ACTIVATE_EXTENSIONS 0 +#define PragTyp_ANALYSIS_LIMIT 1 +#define PragTyp_HEADER_VALUE 2 +#define PragTyp_AUTO_VACUUM 3 +#define PragTyp_FLAG 4 +#define PragTyp_BUSY_TIMEOUT 5 +#define PragTyp_CACHE_SIZE 6 +#define PragTyp_CACHE_SPILL 7 +#define PragTyp_CASE_SENSITIVE_LIKE 8 +#define PragTyp_COLLATION_LIST 9 +#define PragTyp_COMPILE_OPTIONS 10 +#define PragTyp_DATA_STORE_DIRECTORY 11 +#define PragTyp_DATABASE_LIST 12 +#define PragTyp_DEFAULT_CACHE_SIZE 13 +#define PragTyp_ENCODING 14 +#define PragTyp_FOREIGN_KEY_CHECK 15 +#define PragTyp_FOREIGN_KEY_LIST 16 +#define PragTyp_FUNCTION_LIST 17 +#define PragTyp_HARD_HEAP_LIMIT 18 +#define PragTyp_INCREMENTAL_VACUUM 19 +#define PragTyp_INDEX_INFO 20 +#define PragTyp_INDEX_LIST 21 +#define PragTyp_INTEGRITY_CHECK 22 +#define PragTyp_JOURNAL_MODE 23 +#define PragTyp_JOURNAL_SIZE_LIMIT 24 +#define PragTyp_LOCK_PROXY_FILE 25 +#define PragTyp_LOCKING_MODE 26 +#define PragTyp_PAGE_COUNT 27 +#define PragTyp_MMAP_SIZE 28 +#define PragTyp_MODULE_LIST 29 +#define PragTyp_OPTIMIZE 30 +#define PragTyp_PAGE_SIZE 31 +#define PragTyp_PRAGMA_LIST 32 +#define PragTyp_SECURE_DELETE 33 +#define PragTyp_SHRINK_MEMORY 34 +#define PragTyp_SOFT_HEAP_LIMIT 35 +#define PragTyp_SYNCHRONOUS 36 +#define PragTyp_TABLE_INFO 37 +#define PragTyp_TEMP_STORE 38 +#define PragTyp_TEMP_STORE_DIRECTORY 39 +#define PragTyp_THREADS 40 +#define PragTyp_WAL_AUTOCHECKPOINT 41 +#define PragTyp_WAL_CHECKPOINT 42 +#define PragTyp_LOCK_STATUS 43 +#define PragTyp_STATS 44 + +/* Property flags associated with various pragma. */ +#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ +#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ +#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ +#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ +#define PragFlg_Result0 0x10 /* Acts as query when no argument */ +#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ +#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ +#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ + +/* Names of columns for pragmas that return multi-column result +** or that return single-column results where the name of the +** result column is different from the name of the pragma +*/ +static const char *const pragCName[] = { + /* 0 */ "id", /* Used by: foreign_key_list */ + /* 1 */ "seq", + /* 2 */ "table", + /* 3 */ "from", + /* 4 */ "to", + /* 5 */ "on_update", + /* 6 */ "on_delete", + /* 7 */ "match", + /* 8 */ "cid", /* Used by: table_xinfo */ + /* 9 */ "name", + /* 10 */ "type", + /* 11 */ "notnull", + /* 12 */ "dflt_value", + /* 13 */ "pk", + /* 14 */ "hidden", + /* table_info reuses 8 */ + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "name", /* Used by: function_list */ + /* 22 */ "builtin", + /* 23 */ "type", + /* 24 */ "enc", + /* 25 */ "narg", + /* 26 */ "flags", + /* 27 */ "tbl", /* Used by: stats */ + /* 28 */ "idx", + /* 29 */ "wdth", + /* 30 */ "hght", + /* 31 */ "flgs", + /* 32 */ "seq", /* Used by: index_list */ + /* 33 */ "name", + /* 34 */ "unique", + /* 35 */ "origin", + /* 36 */ "partial", + /* 37 */ "table", /* Used by: foreign_key_check */ + /* 38 */ "rowid", + /* 39 */ "parent", + /* 40 */ "fkid", + /* index_info reuses 15 */ + /* 41 */ "seq", /* Used by: database_list */ + /* 42 */ "name", + /* 43 */ "file", + /* 44 */ "busy", /* Used by: wal_checkpoint */ + /* 45 */ "log", + /* 46 */ "checkpointed", + /* collation_list reuses 32 */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", + /* 49 */ "cache_size", /* Used by: default_cache_size */ + /* module_list pragma_list reuses 9 */ + /* 50 */ "timeout", /* Used by: busy_timeout */ +}; + +/* Definitions of all built-in pragmas */ +typedef struct PragmaName { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlg; /* Zero or more PragFlg_XXX values */ + u8 iPragCName; /* Start of column names in pragCName[] */ + u8 nPragCName; /* Num of col names. 0 means use pragma name */ + u64 iArg; /* Extra argument */ +} PragmaName; +static const PragmaName aPragmaName[] = { +#if defined(SQLITE_ENABLE_CEROD) + {/* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "analysis_limit", + /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_APPLICATION_ID }, +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + {/* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) + {/* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_AutoIndex }, +#endif +#endif + {/* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 50, 1, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_CACHE_SPILL, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) + {/* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CellSizeCk }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CkptFullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 32, 2, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) + {/* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CountRows }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN + {/* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_DATA_VERSION }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, + /* ColNames: */ 41, 3, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + {/* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 49, 1, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_DeferFKs }, +#endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NullCallback }, +#endif +#if !defined(SQLITE_OMIT_UTF16) + {/* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 37, 4, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) + {/* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 8, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ForeignKeys }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullColNames }, + {/* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "function_list", + /* ePragTyp: */ PragTyp_FUNCTION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 21, 6, + /* iArg: */ 0 }, +#endif +#endif + {/* zName: */ "hard_heap_limit", + /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_CHECK) + {/* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_IgnoreChecks }, +#endif +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + {/* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 15, 3, + /* iArg: */ 0 }, + {/* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 32, 5, + /* iArg: */ 0 }, + {/* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 15, 6, + /* iArg: */ 1 }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + {/* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "legacy_alter_table", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_LegacyAlter }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE + {/* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + {/* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 47, 2, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_VIRTUALTABLE) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "module_list", + /* ePragTyp: */ PragTyp_MODULE_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#endif +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ParserTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "pragma_list", + /* ePragTyp: */ PragTyp_PRAGMA_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_QueryOnly }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + {/* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReadUncommit }, + {/* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_RecTriggers }, + {/* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReverseOrder }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_SCHEMA_VERSION }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ShortColNames }, +#endif + {/* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) + {/* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 27, 5, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 6, + /* iArg: */ 0 }, + {/* zName: */ "table_xinfo", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 7, + /* iArg: */ 1 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "trusted_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_TrustedSchema }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_USER_VERSION }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + {/* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeEQP }, + {/* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeListing }, + {/* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_WAL) + {/* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 44, 3, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, +#endif +}; +/* Number of pragmas: 67 on by default, 77 total. */ diff --git a/third_party/sqlite3/prepare.c b/third_party/sqlite3/prepare.c new file mode 100644 index 000000000..c7575105e --- /dev/null +++ b/third_party/sqlite3/prepare.c @@ -0,0 +1,1014 @@ +/* +** 2005 May 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_prepare() +** interface, and routines that contribute to loading the database schema +** from disk. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** Fill the InitData structure with an error message that indicates +** that the database is corrupt. +*/ +static void corruptSchema( + InitData *pData, /* Initialization context */ + char **azObj, /* Type and name of object being parsed */ + const char *zExtra /* Error information */ +){ + sqlite3 *db = pData->db; + if( db->mallocFailed ){ + pData->rc = SQLITE_NOMEM_BKPT; + }else if( pData->pzErrMsg[0]!=0 ){ + /* A error message has already been generated. Do not overwrite it */ + }else if( pData->mInitFlags & (INITFLAG_AlterRename|INITFLAG_AlterDrop) ){ + *pData->pzErrMsg = sqlite3MPrintf(db, + "error in %s %s after %s: %s", azObj[0], azObj[1], + (pData->mInitFlags & INITFLAG_AlterRename) ? "rename" : "drop column", + zExtra + ); + pData->rc = SQLITE_ERROR; + }else if( db->flags & SQLITE_WriteSchema ){ + pData->rc = SQLITE_CORRUPT_BKPT; + }else{ + char *z; + const char *zObj = azObj[1] ? azObj[1] : "?"; + z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); + if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); + *pData->pzErrMsg = z; + pData->rc = SQLITE_CORRUPT_BKPT; + } +} + +/* +** Check to see if any sibling index (another index on the same table) +** of pIndex has the same root page number, and if it does, return true. +** This would indicate a corrupt schema. +*/ +int sqlite3IndexHasDuplicateRootPage(Index *pIndex){ + Index *p; + for(p=pIndex->pTable->pIndex; p; p=p->pNext){ + if( p->tnum==pIndex->tnum && p!=pIndex ) return 1; + } + return 0; +} + +/* forward declaration */ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +); + + +/* +** This is the callback routine for the code that initializes the +** database. See sqlite3Init() below for additional information. +** This routine is also called from the OP_ParseSchema opcode of the VDBE. +** +** Each callback contains the following information: +** +** argv[0] = type of object: "table", "index", "trigger", or "view". +** argv[1] = name of thing being created +** argv[2] = associated table if an index or trigger +** argv[3] = root page number for table or index. 0 for trigger or view. +** argv[4] = SQL text for the CREATE statement. +** +*/ +int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ + InitData *pData = (InitData*)pInit; + sqlite3 *db = pData->db; + int iDb = pData->iDb; + + assert( argc==5 ); + UNUSED_PARAMETER2(NotUsed, argc); + assert( sqlite3_mutex_held(db->mutex) ); + db->mDbFlags |= DBFLAG_EncodingFixed; + pData->nInitRow++; + if( db->mallocFailed ){ + corruptSchema(pData, argv, 0); + return 1; + } + + assert( iDb>=0 && iDbnDb ); + if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ + if( argv[3]==0 ){ + corruptSchema(pData, argv, 0); + }else if( argv[4] + && 'c'==sqlite3UpperToLower[(unsigned char)argv[4][0]] + && 'r'==sqlite3UpperToLower[(unsigned char)argv[4][1]] ){ + /* Call the parser to process a CREATE TABLE, INDEX or VIEW. + ** But because db->init.busy is set to 1, no VDBE code is generated + ** or executed. All the parser does is build the internal data + ** structures that describe the table, index, or view. + ** + ** No other valid SQL statement, other than the variable CREATE statements, + ** can begin with the letters "C" and "R". Thus, it is not possible run + ** any other kind of statement while parsing the schema, even a corrupt + ** schema. + */ + int rc; + u8 saved_iDb = db->init.iDb; + sqlite3_stmt *pStmt; + TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ + + assert( db->init.busy ); + db->init.iDb = iDb; + if( sqlite3GetUInt32(argv[3], &db->init.newTnum)==0 + || (db->init.newTnum>pData->mxPage && pData->mxPage>0) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } + } + db->init.orphanTrigger = 0; + db->init.azInit = argv; + pStmt = 0; + TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, 0); + rc = db->errCode; + assert( (rc&0xFF)==(rcp&0xFF) ); + db->init.iDb = saved_iDb; + /* assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); */ + if( SQLITE_OK!=rc ){ + if( db->init.orphanTrigger ){ + assert( iDb==1 ); + }else{ + if( rc > pData->rc ) pData->rc = rc; + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ + corruptSchema(pData, argv, sqlite3_errmsg(db)); + } + } + } + sqlite3_finalize(pStmt); + }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ + corruptSchema(pData, argv, 0); + }else{ + /* If the SQL column is blank it means this is an index that + ** was created to be the PRIMARY KEY or to fulfill a UNIQUE + ** constraint for a CREATE TABLE. The index should have already + ** been created when we processed the CREATE TABLE. All we have + ** to do here is record the root page number for that index. + */ + Index *pIndex; + pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); + if( pIndex==0 ){ + corruptSchema(pData, argv, "orphan index"); + }else + if( sqlite3GetUInt32(argv[3],&pIndex->tnum)==0 + || pIndex->tnum<2 + || pIndex->tnum>pData->mxPage + || sqlite3IndexHasDuplicateRootPage(pIndex) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } + } + } + return 0; +} + +/* +** Attempt to read the database schema and initialize internal +** data structures for a single database file. The index of the +** database file is given by iDb. iDb==0 is used for the main +** database. iDb==1 should never be used. iDb>=2 is used for +** auxiliary databases. Return one of the SQLITE_ error codes to +** indicate success or failure. +*/ +int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ + int rc; + int i; +#ifndef SQLITE_OMIT_DEPRECATED + int size; +#endif + Db *pDb; + char const *azArg[6]; + int meta[5]; + InitData initData; + const char *zSchemaTabName; + int openedTransaction = 0; + int mask = ((db->mDbFlags & DBFLAG_EncodingFixed) | ~DBFLAG_EncodingFixed); + + assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ); + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + + db->init.busy = 1; + + /* Construct the in-memory representation schema tables (sqlite_schema or + ** sqlite_temp_schema) by invoking the parser directly. The appropriate + ** table name will be inserted automatically by the parser so we can just + ** use the abbreviation "x" here. The parser will also automatically tag + ** the schema table as read-only. */ + azArg[0] = "table"; + azArg[1] = zSchemaTabName = SCHEMA_TABLE(iDb); + azArg[2] = azArg[1]; + azArg[3] = "1"; + azArg[4] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage int,sql text)"; + azArg[5] = 0; + initData.db = db; + initData.iDb = iDb; + initData.rc = SQLITE_OK; + initData.pzErrMsg = pzErrMsg; + initData.mInitFlags = mFlags; + initData.nInitRow = 0; + initData.mxPage = 0; + sqlite3InitCallback(&initData, 5, (char **)azArg, 0); + db->mDbFlags &= mask; + if( initData.rc ){ + rc = initData.rc; + goto error_out; + } + + /* Create a cursor to hold the database open + */ + pDb = &db->aDb[iDb]; + if( pDb->pBt==0 ){ + assert( iDb==1 ); + DbSetProperty(db, 1, DB_SchemaLoaded); + rc = SQLITE_OK; + goto error_out; + } + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed before this function returns. */ + sqlite3BtreeEnter(pDb->pBt); + if( sqlite3BtreeTxnState(pDb->pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0); + if( rc!=SQLITE_OK ){ + sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); + goto initone_error_out; + } + openedTransaction = 1; + } + + /* Get the database meta information. + ** + ** Meta values are as follows: + ** meta[0] Schema cookie. Changes with each schema change. + ** meta[1] File format of schema layer. + ** meta[2] Size of the page cache. + ** meta[3] Largest rootpage (auto/incr_vacuum mode) + ** meta[4] Db text encoding. 1:UTF-8 2:UTF-16LE 3:UTF-16BE + ** meta[5] User version + ** meta[6] Incremental vacuum mode + ** meta[7] unused + ** meta[8] unused + ** meta[9] unused + ** + ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to + ** the possible values of meta[4]. + */ + for(i=0; ipBt, i+1, (u32 *)&meta[i]); + } + if( (db->flags & SQLITE_ResetDatabase)!=0 ){ + memset(meta, 0, sizeof(meta)); + } + pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; + + /* If opening a non-empty database, check the text encoding. For the + ** main database, set sqlite3.enc to the encoding of the main database. + ** For an attached db, it is an error if the encoding is not the same + ** as sqlite3.enc. + */ + if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ + if( iDb==0 && (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ + u8 encoding; +#ifndef SQLITE_OMIT_UTF16 + /* If opening the main database, set ENC(db). */ + encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; + if( encoding==0 ) encoding = SQLITE_UTF8; +#else + encoding = SQLITE_UTF8; +#endif + sqlite3SetTextEncoding(db, encoding); + }else{ + /* If opening an attached database, the encoding much match ENC(db) */ + if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ + sqlite3SetString(pzErrMsg, db, "attached databases must use the same" + " text encoding as main database"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + } + } + pDb->pSchema->enc = ENC(db); + + if( pDb->pSchema->cache_size==0 ){ +#ifndef SQLITE_OMIT_DEPRECATED + size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); + if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } + pDb->pSchema->cache_size = size; +#else + pDb->pSchema->cache_size = SQLITE_DEFAULT_CACHE_SIZE; +#endif + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + + /* + ** file_format==1 Version 3.0.0. + ** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN + ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults + ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants + */ + pDb->pSchema->file_format = (u8)meta[BTREE_FILE_FORMAT-1]; + if( pDb->pSchema->file_format==0 ){ + pDb->pSchema->file_format = 1; + } + if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ + sqlite3SetString(pzErrMsg, db, "unsupported file format"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + + /* Ticket #2804: When we open a database in the newer file format, + ** clear the legacy_file_format pragma flag so that a VACUUM will + ** not downgrade the database and thus invalidate any descending + ** indices that the user might have created. + */ + if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ + db->flags &= ~(u64)SQLITE_LegacyFileFmt; + } + + /* Read the schema information out of the schema tables + */ + assert( db->init.busy ); + initData.mxPage = sqlite3BtreeLastPage(pDb->pBt); + { + char *zSql; + zSql = sqlite3MPrintf(db, + "SELECT*FROM\"%w\".%s ORDER BY rowid", + db->aDb[iDb].zDbSName, zSchemaTabName); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + sqlite3_xauth xAuth; + xAuth = db->xAuth; + db->xAuth = 0; +#endif + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; + } +#endif + if( rc==SQLITE_OK ) rc = initData.rc; + sqlite3DbFree(db, zSql); +#ifndef SQLITE_OMIT_ANALYZE + if( rc==SQLITE_OK ){ + sqlite3AnalysisLoad(db, iDb); + } +#endif + } + if( db->mallocFailed ){ + rc = SQLITE_NOMEM_BKPT; + sqlite3ResetAllSchemasOfConnection(db); + } + if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + /* Black magic: If the SQLITE_NoSchemaError flag is set, then consider + ** the schema loaded, even if errors occurred. In this situation the + ** current sqlite3_prepare() operation will fail, but the following one + ** will attempt to compile the supplied statement against whatever subset + ** of the schema was loaded before the error occurred. The primary + ** purpose of this is to allow access to the sqlite_schema table + ** even when its contents have been corrupted. + */ + DbSetProperty(db, iDb, DB_SchemaLoaded); + rc = SQLITE_OK; + } + + /* Jump here for an error that occurs after successfully allocating + ** curMain and calling sqlite3BtreeEnter(). For an error that occurs + ** before that point, jump to error_out. + */ +initone_error_out: + if( openedTransaction ){ + sqlite3BtreeCommit(pDb->pBt); + } + sqlite3BtreeLeave(pDb->pBt); + +error_out: + if( rc ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + } + sqlite3ResetOneSchema(db, iDb); + } + db->init.busy = 0; + return rc; +} + +/* +** Initialize all database files - the main database file, the file +** used to store temporary tables, and any additional database files +** created using ATTACH statements. Return a success code. If an +** error occurs, write an error message into *pzErrMsg. +** +** After a database is initialized, the DB_SchemaLoaded bit is set +** bit is set in the flags field of the Db structure. +*/ +int sqlite3Init(sqlite3 *db, char **pzErrMsg){ + int i, rc; + int commit_internal = !(db->mDbFlags&DBFLAG_SchemaChange); + + assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); + assert( db->init.busy==0 ); + ENC(db) = SCHEMA_ENC(db); + assert( db->nDb>0 ); + /* Do the main schema first */ + if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 0, pzErrMsg, 0); + if( rc ) return rc; + } + /* All other schemas after the main schema. The "temp" schema must be last */ + for(i=db->nDb-1; i>0; i--){ + assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) ); + if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, i, pzErrMsg, 0); + if( rc ) return rc; + } + } + if( commit_internal ){ + sqlite3CommitInternalChanges(db); + } + return SQLITE_OK; +} + +/* +** This routine is a no-op if the database schema is already initialized. +** Otherwise, the schema is loaded. An error code is returned. +*/ +int sqlite3ReadSchema(Parse *pParse){ + int rc = SQLITE_OK; + sqlite3 *db = pParse->db; + assert( sqlite3_mutex_held(db->mutex) ); + if( !db->init.busy ){ + rc = sqlite3Init(db, &pParse->zErrMsg); + if( rc!=SQLITE_OK ){ + pParse->rc = rc; + pParse->nErr++; + }else if( db->noSharedCache ){ + db->mDbFlags |= DBFLAG_SchemaKnownOk; + } + } + return rc; +} + + +/* +** Check schema cookies in all databases. If any cookie is out +** of date set pParse->rc to SQLITE_SCHEMA. If all schema cookies +** make no changes to pParse->rc. +*/ +static void schemaIsValid(Parse *pParse){ + sqlite3 *db = pParse->db; + int iDb; + int rc; + int cookie; + + assert( pParse->checkSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + for(iDb=0; iDbnDb; iDb++){ + int openedTransaction = 0; /* True if a transaction is opened */ + Btree *pBt = db->aDb[iDb].pBt; /* Btree database to read cookie from */ + if( pBt==0 ) continue; + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed immediately after reading the meta-value. */ + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + } + if( rc!=SQLITE_OK ) return; + openedTransaction = 1; + } + + /* Read the schema cookie from the database. If it does not match the + ** value stored as part of the in-memory schema representation, + ** set Parse.rc to SQLITE_SCHEMA. */ + sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + sqlite3ResetOneSchema(db, iDb); + pParse->rc = SQLITE_SCHEMA; + } + + /* Close the transaction, if one was opened. */ + if( openedTransaction ){ + sqlite3BtreeCommit(pBt); + } + } +} + +/* +** Convert a schema pointer into the iDb index that indicates +** which database file in db->aDb[] the schema refers to. +** +** If the same database is attached more than once, the first +** attached database is returned. +*/ +int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ + int i = -32768; + + /* If pSchema is NULL, then return -32768. This happens when code in + ** expr.c is trying to resolve a reference to a transient table (i.e. one + ** created by a sub-select). In this case the return value of this + ** function should never be used. + ** + ** We return -32768 instead of the more usual -1 simply because using + ** -32768 as the incorrect index into db->aDb[] is much + ** more likely to cause a segfault than -1 (of course there are assert() + ** statements too, but it never hurts to play the odds) and + ** -32768 will still fit into a 16-bit signed integer. + */ + assert( sqlite3_mutex_held(db->mutex) ); + if( pSchema ){ + for(i=0; 1; i++){ + assert( inDb ); + if( db->aDb[i].pSchema==pSchema ){ + break; + } + } + assert( i>=0 && inDb ); + } + return i; +} + +/* +** Free all memory allocations in the pParse object +*/ +void sqlite3ParserReset(Parse *pParse){ + sqlite3 *db = pParse->db; + while( pParse->pCleanup ){ + ParseCleanup *pCleanup = pParse->pCleanup; + pParse->pCleanup = pCleanup->pNext; + pCleanup->xCleanup(db, pCleanup->pPtr); + sqlite3DbFreeNN(db, pCleanup); + } + sqlite3DbFree(db, pParse->aLabel); + if( pParse->pConstExpr ){ + sqlite3ExprListDelete(db, pParse->pConstExpr); + } + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; + } + pParse->disableLookaside = 0; +} + +/* +** Add a new cleanup operation to a Parser. The cleanup should happen when +** the parser object is destroyed. But, beware: the cleanup might happen +** immediately. +** +** Use this mechanism for uncommon cleanups. There is a higher setup +** cost for this mechansim (an extra malloc), so it should not be used +** for common cleanups that happen on most calls. But for less +** common cleanups, we save a single NULL-pointer comparison in +** sqlite3ParserReset(), which reduces the total CPU cycle count. +** +** If a memory allocation error occurs, then the cleanup happens immediately. +** When either SQLITE_DEBUG or SQLITE_COVERAGE_TEST are defined, the +** pParse->earlyCleanup flag is set in that case. Calling code show verify +** that test cases exist for which this happens, to guard against possible +** use-after-free errors following an OOM. The preferred way to do this is +** to immediately follow the call to this routine with: +** +** testcase( pParse->earlyCleanup ); +** +** This routine returns a copy of its pPtr input (the third parameter) +** except if an early cleanup occurs, in which case it returns NULL. So +** another way to check for early cleanup is to check the return value. +** Or, stop using the pPtr parameter with this call and use only its +** return value thereafter. Something like this: +** +** pObj = sqlite3ParserAddCleanup(pParse, destructor, pObj); +*/ +void *sqlite3ParserAddCleanup( + Parse *pParse, /* Destroy when this Parser finishes */ + void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ + void *pPtr /* Pointer to object to be cleaned up */ +){ + ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + if( pCleanup ){ + pCleanup->pNext = pParse->pCleanup; + pParse->pCleanup = pCleanup; + pCleanup->pPtr = pPtr; + pCleanup->xCleanup = xCleanup; + }else{ + xCleanup(pParse->db, pPtr); + pPtr = 0; +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + pParse->earlyCleanup = 1; +#endif + } + return pPtr; +} + +/* +** Compile the UTF-8 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + char *zErrMsg = 0; /* Error message */ + int rc = SQLITE_OK; /* Result code */ + int i; /* Loop counter */ + Parse sParse; /* Parsing context */ + + memset(&sParse, 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pReprepare = pReprepare; + assert( ppStmt && *ppStmt==0 ); + /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ + assert( sqlite3_mutex_held(db->mutex) ); + + /* For a long-term use prepared statement avoid the use of + ** lookaside memory. + */ + if( prepFlags & SQLITE_PREPARE_PERSISTENT ){ + sParse.disableLookaside++; + DisableLookaside; + } + sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + + /* Check to verify that it is possible to get a read lock on all + ** database schemas. The inability to get a read lock indicates that + ** some other database connection is holding a write-lock, which in + ** turn means that the other connection has made uncommitted changes + ** to the schema. + ** + ** Were we to proceed and prepare the statement against the uncommitted + ** schema changes and if those schema changes are subsequently rolled + ** back and different changes are made in their place, then when this + ** prepared statement goes to run the schema cookie would fail to detect + ** the schema change. Disaster would follow. + ** + ** This thread is currently holding mutexes on all Btrees (because + ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it + ** is not possible for another thread to start a new schema change + ** while this routine is running. Hence, we do not need to hold + ** locks on the schema, we just need to make sure nobody else is + ** holding them. + ** + ** Note that setting READ_UNCOMMITTED overrides most lock detection, + ** but it does *not* override schema lock detection, so this all still + ** works even if READ_UNCOMMITTED is set. + */ + if( !db->noSharedCache ){ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zDbSName; + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommit ); + goto end_prepare; + } + } + } + } + + sqlite3VtabUnlockList(db); + + sParse.db = db; + if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ + char *zSqlCopy; + int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + testcase( nBytes==mxLen ); + testcase( nBytes==mxLen+1 ); + if( nBytes>mxLen ){ + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); + rc = sqlite3ApiExit(db, SQLITE_TOOBIG); + goto end_prepare; + } + zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); + if( zSqlCopy ){ + sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; + sqlite3DbFree(db, zSqlCopy); + }else{ + sParse.zTail = &zSql[nBytes]; + } + }else{ + sqlite3RunParser(&sParse, zSql, &zErrMsg); + } + assert( 0==sParse.nQueryLoop ); + + if( pzTail ){ + *pzTail = sParse.zTail; + } + + if( db->init.busy==0 ){ + sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail-zSql), prepFlags); + } + if( db->mallocFailed ){ + sParse.rc = SQLITE_NOMEM_BKPT; + } + if( sParse.rc!=SQLITE_OK && sParse.rc!=SQLITE_DONE ){ + if( sParse.checkSchema ){ + schemaIsValid(&sParse); + } + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + assert( 0==(*ppStmt) ); + rc = sParse.rc; + if( zErrMsg ){ + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); + sqlite3DbFree(db, zErrMsg); + }else{ + sqlite3Error(db, rc); + } + }else{ + assert( zErrMsg==0 ); + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; + rc = SQLITE_OK; + sqlite3ErrorClear(db); + } + + + /* Delete any TriggerPrg structures allocated while parsing this statement. */ + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; + sqlite3DbFree(db, pT); + } + +end_prepare: + + sqlite3ParserReset(&sParse); + return rc; +} +static int sqlite3LockAndPrepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pOld, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + int cnt = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + do{ + /* Make multiple attempts to compile the SQL, until it either succeeds + ** or encounters a permanent error. A schema problem after one schema + ** reset is considered a permanent error. */ + rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); + assert( rc==SQLITE_OK || *ppStmt==0 ); + }while( rc==SQLITE_ERROR_RETRY + || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt++)==0) ); + sqlite3BtreeLeaveAll(db); + rc = sqlite3ApiExit(db, rc); + assert( (rc&db->errMask)==rc ); + db->busyHandler.nBusy = 0; + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** Rerun the compilation of a statement after a schema change. +** +** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, +** if the statement cannot be recompiled because another connection has +** locked the sqlite3_schema table, return SQLITE_LOCKED. If any other error +** occurs, return SQLITE_SCHEMA. +*/ +int sqlite3Reprepare(Vdbe *p){ + int rc; + sqlite3_stmt *pNew; + const char *zSql; + sqlite3 *db; + u8 prepFlags; + + assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); + zSql = sqlite3_sql((sqlite3_stmt *)p); + assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ + db = sqlite3VdbeDb(p); + assert( sqlite3_mutex_held(db->mutex) ); + prepFlags = sqlite3VdbePrepareFlags(p); + rc = sqlite3LockAndPrepare(db, zSql, -1, prepFlags, p, &pNew, 0); + if( rc ){ + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + assert( pNew==0 ); + return rc; + }else{ + assert( pNew!=0 ); + } + sqlite3VdbeSwap((Vdbe*)pNew, p); + sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); + sqlite3VdbeResetStepResult((Vdbe*)pNew); + sqlite3VdbeFinalize((Vdbe*)pNew); + return SQLITE_OK; +} + + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +int sqlite3_prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + /* EVIDENCE-OF: R-37923-12173 The sqlite3_prepare_v2() interface works + ** exactly the same as sqlite3_prepare_v3() with a zero prepFlags + ** parameter. + ** + ** Proof in that the 5th parameter to sqlite3LockAndPrepare is 0 */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,0, + ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); + return rc; +} +int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + /* EVIDENCE-OF: R-56861-42673 sqlite3_prepare_v3() differs from + ** sqlite3_prepare_v2() only in having the extra prepFlags parameter, + ** which is a bit array consisting of zero or more of the + ** SQLITE_PREPARE_* flags. + ** + ** Proof by comparison to the implementation of sqlite3_prepare_v2() + ** directly above. */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + 0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); + return rc; +} + + +#ifndef SQLITE_OMIT_UTF16 +/* +** Compile the UTF-16 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + /* This function currently works by first transforming the UTF-16 + ** encoded string to UTF-8, then invoking sqlite3_prepare(). The + ** tricky bit is figuring out the pointer to return in *pzTail. + */ + char *zSql8; + const char *zTail8 = 0; + int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ + return SQLITE_MISUSE_BKPT; + } + if( nBytes>=0 ){ + int sz; + const char *z = (const char*)zSql; + for(sz=0; szmutex); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); + if( zSql8 ){ + rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8); + } + + if( zTail8 && pzTail ){ + /* If sqlite3_prepare returns a tail pointer, we calculate the + ** equivalent pointer into the UTF-16 string by counting the unicode + ** characters between zSql8 and zTail8, and then returning a pointer + ** the same number of characters into the UTF-16 string. + */ + int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + } + sqlite3DbFree(db, zSql8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +int sqlite3_prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} + +#endif /* SQLITE_OMIT_UTF16 */ diff --git a/third_party/sqlite3/printf.c b/third_party/sqlite3/printf.c new file mode 100644 index 000000000..f916d9f54 --- /dev/null +++ b/third_party/sqlite3/printf.c @@ -0,0 +1,1311 @@ +/* +** The "printf" code that follows dates from the 1980's. It is in +** the public domain. +** +************************************************************************** +** +** This file contains code for a set of "printf"-like routines. These +** routines format strings much like the printf() from the standard C +** library, though the implementation here has enhancements to support +** SQLite. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** Conversion types fall into various categories as defined by the +** following enumeration. +*/ +#define etRADIX 0 /* non-decimal integer types. %x %o */ +#define etFLOAT 1 /* Floating point. %f */ +#define etEXP 2 /* Exponentional notation. %e and %E */ +#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 4 /* Return number of characters processed so far. %n */ +#define etSTRING 5 /* Strings. %s */ +#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ +#define etPERCENT 7 /* Percent symbol. %% */ +#define etCHARX 8 /* Characters. %c */ +/* The rest are extensions, not normally found in printf() */ +#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCLIST 12 /* a pointer to a SrcList */ +#define etPOINTER 13 /* The %p conversion */ +#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ + +#define etINVALID 17 /* Any unrecognized conversion type */ + + +/* +** An "etByte" is an 8-bit unsigned value. +*/ +typedef unsigned char etByte; + +/* +** Each builtin conversion character (ex: the 'd' in "%d") is described +** by an instance of the following structure +*/ +typedef struct et_info { /* Information about each format field */ + char fmttype; /* The format field code letter */ + etByte base; /* The base for radix conversion */ + etByte flags; /* One or more of FLAG_ constants below */ + etByte type; /* Conversion paradigm */ + etByte charset; /* Offset into aDigits[] of the digits string */ + etByte prefix; /* Offset into aPrefix[] of the prefix string */ +} et_info; + +/* +** Allowed values for et_info.flags +*/ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ + + +/* +** The following table is searched linearly, so it is good to put the +** most frequently used conversion types first. +*/ +static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; +static const char aPrefix[] = "-x0\000X0"; +static const et_info fmtinfo[] = { + { 'd', 10, 1, etDECIMAL, 0, 0 }, + { 's', 0, 4, etSTRING, 0, 0 }, + { 'g', 0, 1, etGENERIC, 30, 0 }, + { 'z', 0, 4, etDYNSTRING, 0, 0 }, + { 'q', 0, 4, etSQLESCAPE, 0, 0 }, + { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, + { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, + { 'c', 0, 0, etCHARX, 0, 0 }, + { 'o', 8, 0, etRADIX, 0, 2 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, + { 'x', 16, 0, etRADIX, 16, 1 }, + { 'X', 16, 0, etRADIX, 0, 4 }, +#ifndef SQLITE_OMIT_FLOATING_POINT + { 'f', 0, 1, etFLOAT, 0, 0 }, + { 'e', 0, 1, etEXP, 30, 0 }, + { 'E', 0, 1, etEXP, 14, 0 }, + { 'G', 0, 1, etGENERIC, 14, 0 }, +#endif + { 'i', 10, 1, etDECIMAL, 0, 0 }, + { 'n', 0, 0, etSIZE, 0, 0 }, + { '%', 0, 0, etPERCENT, 0, 0 }, + { 'p', 16, 0, etPOINTER, 0, 1 }, + + /* All the rest are undocumented and are for internal use only */ + { 'T', 0, 0, etTOKEN, 0, 0 }, + { 'S', 0, 0, etSRCLIST, 0, 0 }, + { 'r', 10, 1, etORDINAL, 0, 0 }, +}; + +/* Floating point constants used for rounding */ +static const double arRound[] = { + 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, + 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, +}; + +/* +** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point +** conversions will work. +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a double such that 0.1 <= *val < 10.0 +** Return the ascii code for the leading digit of *val, then +** multiply "*val" by 10.0 to renormalize. +** +** Example: +** input: *val = 3.14159 +** output: *val = 1.4159 function return = '3' +** +** The counter *cnt is incremented each time. After counter exceeds +** 16 (the number of significant digits in a 64-bit float) '0' is +** always returned. +*/ +static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ + int digit; + LONGDOUBLE_TYPE d; + if( (*cnt)<=0 ) return '0'; + (*cnt)--; + digit = (int)*val; + d = digit; + digit += '0'; + *val = (*val - d)*10.0; + return (char)digit; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + +/* +** Set the StrAccum object to an error mode. +*/ +static void setStrAccumError(StrAccum *p, u8 eError){ + assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); + p->accError = eError; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); +} + +/* +** Extra argument values from a PrintfArguments object +*/ +static sqlite3_int64 getIntArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return sqlite3_value_int64(p->apArg[p->nUsed++]); +} +static double getDoubleArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0.0; + return sqlite3_value_double(p->apArg[p->nUsed++]); +} +static char *getTextArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); +} + +/* +** Allocate memory for a temporary buffer needed for printf rendering. +** +** If the requested size of the temp buffer is larger than the size +** of the output buffer in pAccum, then cause an SQLITE_TOOBIG error. +** Do the size check before the memory allocation to prevent rogue +** SQL from requesting large allocations using the precision or width +** field of the printf() function. +*/ +static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ + char *z; + if( pAccum->accError ) return 0; + if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ + setStrAccumError(pAccum, SQLITE_TOOBIG); + return 0; + } + z = sqlite3DbMallocRaw(pAccum->db, n); + if( z==0 ){ + setStrAccumError(pAccum, SQLITE_NOMEM); + } + return z; +} + +/* +** On machines with a small stack size, you can redefine the +** SQLITE_PRINT_BUF_SIZE to be something smaller, if desired. +*/ +#ifndef SQLITE_PRINT_BUF_SIZE +# define SQLITE_PRINT_BUF_SIZE 70 +#endif +#define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ + +/* +** Hard limit on the precision of floating-point conversions. +*/ +#ifndef SQLITE_PRINTF_PRECISION_LIMIT +# define SQLITE_FP_PRECISION_LIMIT 100000000 +#endif + +/* +** Render a string given by "fmt" into the StrAccum object. +*/ +void sqlite3_str_vappendf( + sqlite3_str *pAccum, /* Accumulate results here */ + const char *fmt, /* Format string */ + va_list ap /* arguments */ +){ + int c; /* Next character in the format string */ + char *bufpt; /* Pointer to the conversion buffer */ + int precision; /* Precision of the current field */ + int length; /* Length of the field */ + int idx; /* A general purpose loop counter */ + int width; /* Width of the current field */ + etByte flag_leftjustify; /* True if "-" flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ + etByte flag_alternateform; /* True if "#" flag is present */ + etByte flag_altform2; /* True if "!" flag is present */ + etByte flag_zeropad; /* True if field width constant starts with zero */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ + etByte done; /* Loop termination flag */ + etByte cThousand; /* Thousands separator for %d and %u */ + etByte xtype = etINVALID; /* Conversion paradigm */ + u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ + char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ + sqlite_uint64 longvalue; /* Value for integer types */ + LONGDOUBLE_TYPE realvalue; /* Value for real types */ + const et_info *infop; /* Pointer to the appropriate info structure */ + char *zOut; /* Rendering buffer */ + int nOut; /* Size of the rendering buffer */ + char *zExtra = 0; /* Malloced memory used by some conversion */ +#ifndef SQLITE_OMIT_FLOATING_POINT + int exp, e2; /* exponent of real numbers */ + int nsd; /* Number of significant digits returned */ + double rounder; /* Used for rounding floating point values */ + etByte flag_dp; /* True if decimal point should be shown */ + etByte flag_rtz; /* True if trailing zeros should be removed */ +#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ + char buf[etBUFSIZE]; /* Conversion buffer */ + + /* pAccum never starts out with an empty buffer that was obtained from + ** malloc(). This precondition is required by the mprintf("%z...") + ** optimization. */ + assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + + bufpt = 0; + if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ + pArgList = va_arg(ap, PrintfArguments*); + bArgList = 1; + }else{ + bArgList = 0; + } + for(; (c=(*fmt))!=0; ++fmt){ + if( c!='%' ){ + bufpt = (char *)fmt; +#if HAVE_STRCHRNUL + fmt = strchrnul(fmt, '%'); +#else + do{ fmt++; }while( *fmt && *fmt != '%' ); +#endif + sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); + if( *fmt==0 ) break; + } + if( (c=(*++fmt))==0 ){ + sqlite3_str_append(pAccum, "%", 1); + break; + } + /* Find out what flags are present */ + flag_leftjustify = flag_prefix = cThousand = + flag_alternateform = flag_altform2 = flag_zeropad = 0; + done = 0; + width = 0; + flag_long = 0; + precision = -1; + do{ + switch( c ){ + case '-': flag_leftjustify = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; + case '#': flag_alternateform = 1; break; + case '!': flag_altform2 = 1; break; + case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; + default: done = 1; break; + case 'l': { + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + c = *++fmt; + flag_long = 2; + } + done = 1; + break; + } + case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': { + unsigned wx = c - '0'; + while( (c = *++fmt)>='0' && c<='9' ){ + wx = wx*10 + c - '0'; + } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c!='.' && c!='l' ){ + done = 1; + }else{ + fmt--; + } + break; + } + case '*': { + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( (c = fmt[1])!='.' && c!='l' ){ + c = *++fmt; + done = 1; + } + break; + } + case '.': { + c = *++fmt; + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + c = *++fmt; + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c=='l' ){ + --fmt; + }else{ + done = 1; + } + break; + } + } + }while( !done && (c=(*++fmt))!=0 ); + + /* Fetch the info entry for the field */ + infop = &fmtinfo[0]; + xtype = etINVALID; + for(idx=0; idxtype; + break; + } + } + + /* + ** At this point, variables are initialized as follows: + ** + ** flag_alternateform TRUE if a '#' is present. + ** flag_altform2 TRUE if a '!' is present. + ** flag_prefix '+' or ' ' or zero + ** flag_leftjustify TRUE if a '-' is present or if the + ** field width was negative. + ** flag_zeropad TRUE if the width began with 0. + ** flag_long 1 for "l", 2 for "ll" + ** width The specified field width. This is + ** always non-negative. Zero is the default. + ** precision The specified precision. The default + ** is -1. + ** xtype The class of the conversion. + ** infop Pointer to the appropriate info struct. + */ + assert( width>=0 ); + assert( precision>=(-1) ); + switch( xtype ){ + case etPOINTER: + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; + /* no break */ deliberate_fall_through + case etORDINAL: + case etRADIX: + cThousand = 0; + /* no break */ deliberate_fall_through + case etDECIMAL: + if( infop->flags & FLAG_SIGNED ){ + i64 v; + if( bArgList ){ + v = getIntArg(pArgList); + }else if( flag_long ){ + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } + }else{ + v = va_arg(ap,int); + } + if( v<0 ){ + testcase( v==SMALLEST_INT64 ); + testcase( v==(-1) ); + longvalue = ~v; + longvalue++; + prefix = '-'; + }else{ + longvalue = v; + prefix = flag_prefix; + } + }else{ + if( bArgList ){ + longvalue = (u64)getIntArg(pArgList); + }else if( flag_long ){ + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } + }else{ + longvalue = va_arg(ap,unsigned int); + } + prefix = 0; + } + if( longvalue==0 ) flag_alternateform = 0; + if( flag_zeropad && precision=4 || (longvalue/10)%10==1 ){ + x = 0; + } + *(--bufpt) = zOrd[x*2+1]; + *(--bufpt) = zOrd[x*2]; + } + { + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; + do{ /* Convert to ascii */ + *(--bufpt) = cset[longvalue%base]; + longvalue = longvalue/base; + }while( longvalue>0 ); + } + length = (int)(&zOut[nOut-1]-bufpt); + while( precision>length ){ + *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } + } + if( prefix ) *(--bufpt) = prefix; /* Add sign */ + if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ + const char *pre; + char x; + pre = &aPrefix[infop->prefix]; + for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; + } + length = (int)(&zOut[nOut-1]-bufpt); + break; + case etFLOAT: + case etEXP: + case etGENERIC: + if( bArgList ){ + realvalue = getDoubleArg(pArgList); + }else{ + realvalue = va_arg(ap,double); + } +#ifdef SQLITE_OMIT_FLOATING_POINT + length = 0; +#else + if( precision<0 ) precision = 6; /* Set default precision */ +#ifdef SQLITE_FP_PRECISION_LIMIT + if( precision>SQLITE_FP_PRECISION_LIMIT ){ + precision = SQLITE_FP_PRECISION_LIMIT; + } +#endif + if( realvalue<0.0 ){ + realvalue = -realvalue; + prefix = '-'; + }else{ + prefix = flag_prefix; + } + if( xtype==etGENERIC && precision>0 ) precision--; + testcase( precision>0xfff ); + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + exp = 0; + if( sqlite3IsNaN((double)realvalue) ){ + bufpt = "NaN"; + length = 3; + break; + } + if( realvalue>0.0 ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } + } + bufpt = buf; + /* + ** If the field type is etGENERIC, then convert to either etEXP + ** or etFLOAT, as appropriate. + */ + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + if( xtype==etGENERIC ){ + flag_rtz = !flag_alternateform; + if( exp<-4 || exp>precision ){ + xtype = etEXP; + }else{ + precision = precision - exp; + xtype = etFLOAT; + } + }else{ + flag_rtz = flag_altform2; + } + if( xtype==etEXP ){ + e2 = 0; + }else{ + e2 = exp; + } + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; + } + } + zOut = bufpt; + nsd = 16 + flag_altform2*10; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; + /* The sign in front of the number */ + if( prefix ){ + *(bufpt++) = prefix; + } + /* Digits prior to the decimal point */ + if( e2<0 ){ + *(bufpt++) = '0'; + }else{ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + } + /* The decimal point */ + if( flag_dp ){ + *(bufpt++) = '.'; + } + /* "0" digits after the decimal point but before the first + ** significant digit of the number */ + for(e2++; e2<0; precision--, e2++){ + assert( precision>0 ); + *(bufpt++) = '0'; + } + /* Significant digits after the decimal point */ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + /* Remove trailing zeros and the "." if no digits follow the "." */ + if( flag_rtz && flag_dp ){ + while( bufpt[-1]=='0' ) *(--bufpt) = 0; + assert( bufpt>zOut ); + if( bufpt[-1]=='.' ){ + if( flag_altform2 ){ + *(bufpt++) = '0'; + }else{ + *(--bufpt) = 0; + } + } + } + /* Add the "eNNN" suffix */ + if( xtype==etEXP ){ + *(bufpt++) = aDigits[infop->charset]; + if( exp<0 ){ + *(bufpt++) = '-'; exp = -exp; + }else{ + *(bufpt++) = '+'; + } + if( exp>=100 ){ + *(bufpt++) = (char)((exp/100)+'0'); /* 100's digit */ + exp %= 100; + } + *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ + *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ + } + *bufpt = 0; + + /* The converted number is in buf[] and zero terminated. Output it. + ** Note that the number is in the usual order, not reversed as with + ** integer conversions. */ + length = (int)(bufpt-zOut); + bufpt = zOut; + + /* Special case: Add leading zeros if the flag_zeropad flag is + ** set and we are not left justified */ + if( flag_zeropad && !flag_leftjustify && length < width){ + int i; + int nPad = width - length; + for(i=width; i>=nPad; i--){ + bufpt[i] = bufpt[i-nPad]; + } + i = prefix!=0; + while( nPad-- ) bufpt[i++] = '0'; + length = width; + } +#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ + break; + case etSIZE: + if( !bArgList ){ + *(va_arg(ap,int*)) = pAccum->nChar; + } + length = width = 0; + break; + case etPERCENT: + buf[0] = '%'; + bufpt = buf; + length = 1; + break; + case etCHARX: + if( bArgList ){ + bufpt = getTextArg(pArgList); + length = 1; + if( bufpt ){ + buf[0] = c = *(bufpt++); + if( (c&0xc0)==0xc0 ){ + while( length<4 && (bufpt[0]&0xc0)==0x80 ){ + buf[length++] = *(bufpt++); + } + } + }else{ + buf[0] = 0; + } + }else{ + unsigned int ch = va_arg(ap,unsigned int); + if( ch<0x00080 ){ + buf[0] = ch & 0xff; + length = 1; + }else if( ch<0x00800 ){ + buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); + buf[1] = 0x80 + (u8)(ch & 0x3f); + length = 2; + }else if( ch<0x10000 ){ + buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); + buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[2] = 0x80 + (u8)(ch & 0x3f); + length = 3; + }else{ + buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); + buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); + buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[3] = 0x80 + (u8)(ch & 0x3f); + length = 4; + } + } + if( precision>1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3_str_appendchar(pAccum, width-1, ' '); + width = 0; + } + while( precision-- > 1 ){ + sqlite3_str_append(pAccum, buf, length); + } + } + bufpt = buf; + flag_altform2 = 1; + goto adjust_width_for_utf8; + case etSTRING: + case etDYNSTRING: + if( bArgList ){ + bufpt = getTextArg(pArgList); + xtype = etSTRING; + }else{ + bufpt = va_arg(ap,char*); + } + if( bufpt==0 ){ + bufpt = ""; + }else if( xtype==etDYNSTRING ){ + if( pAccum->nChar==0 + && pAccum->mxAlloc + && width==0 + && precision<0 + && pAccum->accError==0 + ){ + /* Special optimization for sqlite3_mprintf("%z..."): + ** Extend an existing memory allocation rather than creating + ** a new one. */ + assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + pAccum->zText = bufpt; + pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt); + pAccum->nChar = 0x7fffffff & (int)strlen(bufpt); + pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED; + length = 0; + break; + } + zExtra = bufpt; + } + if( precision>=0 ){ + if( flag_altform2 ){ + /* Set length to the number of bytes needed in order to display + ** precision characters */ + unsigned char *z = (unsigned char*)bufpt; + while( precision-- > 0 && z[0] ){ + SQLITE_SKIP_UTF8(z); + } + length = (int)(z - (unsigned char*)bufpt); + }else{ + for(length=0; length0 ){ + /* Adjust width to account for extra bytes in UTF-8 characters */ + int ii = length - 1; + while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; + } + break; + case etSQLESCAPE: /* %q: Escape ' characters */ + case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* %w: Escape " characters */ + int i, j, k, n, isnull; + int needQuote; + char ch; + char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ + char *escarg; + + if( bArgList ){ + escarg = getTextArg(pArgList); + }else{ + escarg = va_arg(ap,char*); + } + isnull = escarg==0; + if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + /* For %q, %Q, and %w, the precision is the number of bytes (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. + */ + k = precision; + for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ + if( ch==q ) n++; + if( flag_altform2 && (ch&0xc0)==0xc0 ){ + while( (escarg[i+1]&0xc0)==0x80 ){ i++; } + } + } + needQuote = !isnull && xtype==etSQLESCAPE2; + n += i + 3; + if( n>etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, n); + if( bufpt==0 ) return; + }else{ + bufpt = buf; + } + j = 0; + if( needQuote ) bufpt[j++] = q; + k = i; + for(i=0; iprintfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pToken = va_arg(ap, Token*); + assert( bArgList==0 ); + if( pToken && pToken->n ){ + sqlite3_str_append(pAccum, (const char*)pToken->z, pToken->n); + } + length = width = 0; + break; + } + case etSRCLIST: { + SrcList *pSrc; + int k; + SrcItem *pItem; + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pSrc = va_arg(ap, SrcList*); + k = va_arg(ap, int); + pItem = &pSrc->a[k]; + assert( bArgList==0 ); + assert( k>=0 && knSrc ); + if( pItem->zDatabase ){ + sqlite3_str_appendall(pAccum, pItem->zDatabase); + sqlite3_str_append(pAccum, ".", 1); + } + sqlite3_str_appendall(pAccum, pItem->zName); + length = width = 0; + break; + } + default: { + assert( xtype==etINVALID ); + return; + } + }/* End switch over the format type */ + /* + ** The text of the conversion is pointed to by "bufpt" and is + ** "length" characters long. The field width is "width". Do + ** the output. Both length and width are in bytes, not characters, + ** at this point. If the "!" flag was present on string conversions + ** indicating that width and precision should be expressed in characters, + ** then the values have been translated prior to reaching this point. + */ + width -= length; + if( width>0 ){ + if( !flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + sqlite3_str_append(pAccum, bufpt, length); + if( flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + }else{ + sqlite3_str_append(pAccum, bufpt, length); + } + + if( zExtra ){ + sqlite3DbFree(pAccum->db, zExtra); + zExtra = 0; + } + }/* End for loop over the format string */ +} /* End of function */ + +/* +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==SQLITE_TOOBIG); + testcase(p->accError==SQLITE_NOMEM); + return 0; + } + if( p->mxAlloc==0 ){ + setStrAccumError(p, SQLITE_TOOBIG); + return p->nAlloc - p->nChar - 1; + }else{ + char *zOld = isMalloced(p) ? p->zText : 0; + i64 szNew = p->nChar; + szNew += (sqlite3_int64)N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } + if( szNew > p->mxAlloc ){ + sqlite3_str_reset(p); + setStrAccumError(p, SQLITE_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->db ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3Realloc(zOld, p->nAlloc); + } + if( zNew ){ + assert( p->zText!=0 || p->nChar==0 ); + if( !isMalloced(p) && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + sqlite3_str_reset(p); + setStrAccumError(p, SQLITE_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N copies of character c to the given string buffer. +*/ +void sqlite3_str_appendchar(sqlite3_str *p, int N, char c){ + testcase( p->nChar + (i64)N > 0x7fffffff ); + if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ + return; + } + while( (N--)>0 ) p->zText[p->nChar++] = c; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3_str_append() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3_str_append() routine can use fast calling semantics. +*/ +static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. +*/ +void sqlite3_str_append(sqlite3_str *p, const char *z, int N){ + assert( z!=0 || N==0 ); + assert( p->zText!=0 || p->nChar==0 || p->accError ); + assert( N>=0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); + if( p->nChar+N >= p->nAlloc ){ + enlargeAndAppend(p,z,N); + }else if( N ){ + assert( p->zText ); + p->nChar += N; + memcpy(&p->zText[p->nChar-N], z, N); + } +} + +/* +** Append the complete text of zero-terminated string z[] to the p string. +*/ +void sqlite3_str_appendall(sqlite3_str *p, const char *z){ + sqlite3_str_append(p, z, sqlite3Strlen30(z)); +} + + +/* +** Finish off a string by making sure it is zero-terminated. +** Return a pointer to the resulting string. Return a NULL +** pointer if any kind of error was encountered. +*/ +static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + char *zText; + assert( p->mxAlloc>0 && !isMalloced(p) ); + zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( zText ){ + memcpy(zText, p->zText, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + setStrAccumError(p, SQLITE_NOMEM); + } + p->zText = zText; + return zText; +} +char *sqlite3StrAccumFinish(StrAccum *p){ + if( p->zText ){ + p->zText[p->nChar] = 0; + if( p->mxAlloc>0 && !isMalloced(p) ){ + return strAccumFinishRealloc(p); + } + } + return p->zText; +} + +/* +** This singleton is an sqlite3_str object that is returned if +** sqlite3_malloc() fails to provide space for a real one. This +** sqlite3_str object accepts no new text and always returns +** an SQLITE_NOMEM error. +*/ +static sqlite3_str sqlite3OomStr = { + 0, 0, 0, 0, 0, SQLITE_NOMEM, 0 +}; + +/* Finalize a string created using sqlite3_str_new(). +*/ +char *sqlite3_str_finish(sqlite3_str *p){ + char *z; + if( p!=0 && p!=&sqlite3OomStr ){ + z = sqlite3StrAccumFinish(p); + sqlite3_free(p); + }else{ + z = 0; + } + return z; +} + +/* Return any error code associated with p */ +int sqlite3_str_errcode(sqlite3_str *p){ + return p ? p->accError : SQLITE_NOMEM; +} + +/* Return the current length of p in bytes */ +int sqlite3_str_length(sqlite3_str *p){ + return p ? p->nChar : 0; +} + +/* Return the current value for p */ +char *sqlite3_str_value(sqlite3_str *p){ + if( p==0 || p->nChar==0 ) return 0; + p->zText[p->nChar] = 0; + return p->zText; +} + +/* +** Reset an StrAccum string. Reclaim all malloced memory. +*/ +void sqlite3_str_reset(StrAccum *p){ + if( isMalloced(p) ){ + sqlite3DbFree(p->db, p->zText); + p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; + } + p->nAlloc = 0; + p->nChar = 0; + p->zText = 0; +} + +/* +** Initialize a string accumulator. +** +** p: The accumulator to be initialized. +** db: Pointer to a database connection. May be NULL. Lookaside +** memory is used if not NULL. db->mallocFailed is set appropriately +** when not NULL. +** zBase: An initial buffer. May be NULL in which case the initial buffer +** is malloced. +** n: Size of zBase in bytes. If total space requirements never exceed +** n then no memory allocations ever occur. +** mx: Maximum number of bytes to accumulate. If mx==0 then no memory +** allocations will ever occur. +*/ +void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ + p->zText = zBase; + p->db = db; + p->nAlloc = n; + p->mxAlloc = mx; + p->nChar = 0; + p->accError = 0; + p->printfFlags = 0; +} + +/* Allocate and initialize a new dynamic string object */ +sqlite3_str *sqlite3_str_new(sqlite3 *db){ + sqlite3_str *p = sqlite3_malloc64(sizeof(*p)); + if( p ){ + sqlite3StrAccumInit(p, 0, 0, 0, + db ? db->aLimit[SQLITE_LIMIT_LENGTH] : SQLITE_MAX_LENGTH); + }else{ + p = &sqlite3OomStr; + } + return p; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + assert( db!=0 ); + sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + acc.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_vappendf(&acc, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + if( acc.accError==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + return z; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc(). Omit the internal +** %-conversion extensions. +*/ +char *sqlite3_vmprintf(const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + sqlite3_str_vappendf(&acc, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc()(). Omit the internal +** %-conversion extensions. +*/ +char *sqlite3_mprintf(const char *zFormat, ...){ + va_list ap; + char *z; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + return z; +} + +/* +** sqlite3_snprintf() works like snprintf() except that it ignores the +** current locale settings. This is important for SQLite because we +** are not able to use a "," as the decimal point in place of "." as +** specified by some locales. +** +** Oops: The first two arguments of sqlite3_snprintf() are backwards +** from the snprintf() standard. Unfortunately, it is too late to change +** this without breaking compatibility, so we just have to live with the +** mistake. +** +** sqlite3_vsnprintf() is the varargs version. +*/ +char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ + StrAccum acc; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); + sqlite3_str_vappendf(&acc, zFormat, ap); + zBuf[acc.nChar] = 0; + return zBuf; +} +char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + char *z; + va_list ap; + va_start(ap,zFormat); + z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + va_end(ap); + return z; +} + +/* +** This is the routine that actually formats the sqlite3_log() message. +** We house it in a separate routine from sqlite3_log() to avoid using +** stack space on small-stack systems when logging is disabled. +** +** sqlite3_log() must render into a static buffer. It cannot dynamically +** allocate memory because it might be called while the memory allocator +** mutex is held. +** +** sqlite3_str_vappendf() might ask for *temporary* memory allocations for +** certain format characters (%q) or for very large precisions or widths. +** Care must be taken that any sqlite3_log() calls that occur while the +** memory mutex is held do not use these mechanisms. +*/ +static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ + StrAccum acc; /* String accumulator */ + char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); + sqlite3_str_vappendf(&acc, zFormat, ap); + sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, + sqlite3StrAccumFinish(&acc)); +} + +/* +** Format and write a message to the log if logging is enabled. +*/ +void sqlite3_log(int iErrCode, const char *zFormat, ...){ + va_list ap; /* Vararg list */ + if( sqlite3GlobalConfig.xLog ){ + va_start(ap, zFormat); + renderLogMsg(iErrCode, zFormat, ap); + va_end(ap); + } +} + +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) +/* +** A version of printf() that understands %lld. Used for debugging. +** The printf() built into some versions of windows does not understand %lld +** and segfaults if you give it a long long int. +*/ +void sqlite3DebugPrintf(const char *zFormat, ...){ + va_list ap; + StrAccum acc; + char zBuf[SQLITE_PRINT_BUF_SIZE*10]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + va_start(ap,zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); + va_end(ap); + sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else + fprintf(stdout,"%s", zBuf); + fflush(stdout); +#endif +} +#endif + + +/* +** variable-argument wrapper around sqlite3_str_vappendf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. +*/ +void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3_str_vappendf(p, zFormat, ap); + va_end(ap); +} diff --git a/third_party/sqlite3/random.c b/third_party/sqlite3/random.c new file mode 100644 index 000000000..2c613b53d --- /dev/null +++ b/third_party/sqlite3/random.c @@ -0,0 +1,135 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement a pseudo-random number +** generator (PRNG) for SQLite. +** +** Random numbers are used by some of the database backends in order +** to generate random integer keys for tables or random filenames. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + + +/* All threads share a single random number generator. +** This structure is the current state of the generator. +*/ +static SQLITE_WSD struct sqlite3PrngType { + unsigned char isInit; /* True if initialized */ + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ +} sqlite3Prng; + +/* +** Return N random bytes. +*/ +void sqlite3_randomness(int N, void *pBuf){ + unsigned char t; + unsigned char *zBuf = pBuf; + + /* The "wsdPrng" macro will resolve to the pseudo-random number generator + ** state vector. If writable static data is unsupported on the target, + ** we have to locate the state vector at run-time. In the more common + ** case where writable static data is supported, wsdPrng can refer directly + ** to the "sqlite3Prng" state vector declared above. + */ +#ifdef SQLITE_OMIT_WSD + struct sqlite3PrngType *p = &GLOBAL(struct sqlite3PrngType, sqlite3Prng); +# define wsdPrng p[0] +#else +# define wsdPrng sqlite3Prng +#endif + +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + + sqlite3_mutex_enter(mutex); + if( N<=0 || pBuf==0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + + /* Initialize the state of the random number generator once, + ** the first time this routine is called. The seed value does + ** not need to contain a lot of randomness since we are not + ** trying to do secure encryption or anything like that... + ** + ** Nothing in this file or anywhere else in SQLite does any kind of + ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random + ** number generator) not as an encryption device. + */ + if( !wsdPrng.isInit ){ + int i; + char k[256]; + wsdPrng.j = 0; + wsdPrng.i = 0; + sqlite3OsRandomness(sqlite3_vfs_find(0), 256, k); + for(i=0; i<256; i++){ + wsdPrng.s[i] = (u8)i; + } + for(i=0; i<256; i++){ + wsdPrng.j += wsdPrng.s[i] + k[i]; + t = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; + wsdPrng.s[i] = t; + } + wsdPrng.isInit = 1; + } + + assert( N>0 ); + do{ + wsdPrng.i++; + t = wsdPrng.s[wsdPrng.i]; + wsdPrng.j += t; + wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = t; + t += wsdPrng.s[wsdPrng.i]; + *(zBuf++) = wsdPrng.s[t]; + }while( --N ); + sqlite3_mutex_leave(mutex); +} + +#ifndef SQLITE_UNTESTABLE +/* +** For testing purposes, we sometimes want to preserve the state of +** PRNG and restore the PRNG to its saved state at a later time, or +** to reset the PRNG to its initial state. These routines accomplish +** those tasks. +** +** The sqlite3_test_control() interface calls these routines to +** control the PRNG. +*/ +static SQLITE_WSD struct sqlite3PrngType sqlite3SavedPrng; +void sqlite3PrngSaveState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + sizeof(sqlite3Prng) + ); +} +void sqlite3PrngRestoreState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + sizeof(sqlite3Prng) + ); +} +#endif /* SQLITE_UNTESTABLE */ diff --git a/third_party/sqlite3/resolve.c b/third_party/sqlite3/resolve.c new file mode 100644 index 000000000..4876c90fc --- /dev/null +++ b/third_party/sqlite3/resolve.c @@ -0,0 +1,2023 @@ +/* +** 2008 August 18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains routines used for walking the parser tree and +** resolve all identifiers by associating them with a particular +** table and column. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** Magic table number to mean the EXCLUDED table in an UPSERT statement. +*/ +#define EXCLUDED_TABLE_NUMBER 2 + +/* +** Walk the expression tree pExpr and increase the aggregate function +** depth (the Expr.op2 field) by N on every TK_AGG_FUNCTION node. +** This needs to occur when copying a TK_AGG_FUNCTION node from an +** outer query into an inner subquery. +** +** incrAggFunctionDepth(pExpr,n) is the main routine. incrAggDepth(..) +** is a helper function - a callback for the tree walker. +** +** See also the sqlite3WindowExtraAggFuncDepth() routine in window.c +*/ +static int incrAggDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; + return WRC_Continue; +} +static void incrAggFunctionDepth(Expr *pExpr, int N){ + if( N>0 ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = incrAggDepth; + w.u.n = N; + sqlite3WalkExpr(&w, pExpr); + } +} + +/* +** Turn the pExpr expression into an alias for the iCol-th column of the +** result set in pEList. +** +** If the reference is followed by a COLLATE operator, then make sure +** the COLLATE operator is preserved. For example: +** +** SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase; +** +** Should be transformed into: +** +** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; +** +** The nSubquery parameter specifies how many levels of subquery the +** alias is removed from the original expression. The usual value is +** zero but it might be more if the alias is contained within a subquery +** of the original expression. The Expr.op2 field of TK_AGG_FUNCTION +** structures must be increased by the nSubquery amount. +*/ +static void resolveAlias( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* A result set */ + int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ + Expr *pExpr, /* Transform this into an alias to the result set */ + int nSubquery /* Number of subqueries that the label is moving */ +){ + Expr *pOrig; /* The iCol-th column of the result set */ + Expr *pDup; /* Copy of pOrig */ + sqlite3 *db; /* The database connection */ + + assert( iCol>=0 && iColnExpr ); + pOrig = pEList->a[iCol].pExpr; + assert( pOrig!=0 ); + db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig, 0); + if( pDup!=0 ){ + incrAggFunctionDepth(pDup, nSubquery); + if( pExpr->op==TK_COLLATE ){ + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); + } + + /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This + ** prevents ExprDelete() from deleting the Expr structure itself, + ** allowing it to be repopulated by the memcpy() on the following line. + ** The pExpr->u.zToken might point into memory that will be freed by the + ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to + ** make a copy of the token before doing the sqlite3DbFree(). + */ + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(db, pExpr); + memcpy(pExpr, pDup, sizeof(*pExpr)); + if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ + assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); + pExpr->flags |= EP_MemToken; + } + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( pExpr->y.pWin!=0 ){ + pExpr->y.pWin->pOwner = pExpr; + }else{ + assert( db->mallocFailed ); + } + } + sqlite3DbFree(db, pDup); + } +} + + +/* +** Return TRUE if the name zCol occurs anywhere in the USING clause. +** +** Return FALSE if the USING clause is NULL or if it does not contain +** zCol. +*/ +static int nameInUsingClause(IdList *pUsing, const char *zCol){ + if( pUsing ){ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + } + } + return 0; +} + +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +int sqlite3MatchEName( + const struct ExprList_item *pItem, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + const char *zSpan; + if( pItem->eEName!=ENAME_TAB ) return 0; + zSpan = pItem->zEName; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} + +/* +** Return TRUE if the double-quoted string mis-feature should be supported. +*/ +static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ + if( db->init.busy ) return 1; /* Always support for legacy schemas */ + if( pTopNC->ncFlags & NC_IsDDL ){ + /* Currently parsing a DDL statement */ + if( sqlite3WritableSchema(db) && (db->flags & SQLITE_DqsDML)!=0 ){ + return 1; + } + return (db->flags & SQLITE_DqsDDL)!=0; + }else{ + /* Currently parsing a DML statement */ + return (db->flags & SQLITE_DqsDML)!=0; + } +} + +/* +** The argument is guaranteed to be a non-NULL Expr node of type TK_COLUMN. +** return the appropriate colUsed mask. +*/ +Bitmask sqlite3ExprColUsed(Expr *pExpr){ + int n; + Table *pExTab; + + n = pExpr->iColumn; + pExTab = pExpr->y.pTab; + assert( pExTab!=0 ); + if( (pExTab->tabFlags & TF_HasGenerated)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pExTab->nCol==BMS-1 ); + testcase( pExTab->nCol==BMS ); + return pExTab->nCol>=BMS ? ALLBITS : MASKBIT(pExTab->nCol)-1; + }else{ + testcase( n==BMS-1 ); + testcase( n==BMS ); + if( n>=BMS ) n = BMS-1; + return ((Bitmask)1)<iDb Set the index in db->aDb[] of the database X +** (even if X is implied). +** pExpr->iTable Set to the cursor number for the table obtained +** from pSrcList. +** pExpr->y.pTab Points to the Table structure of X.Y (even if +** X and/or Y are implied.) +** pExpr->iColumn Set to the column number within the table. +** pExpr->op Set to TK_COLUMN. +** pExpr->pLeft Any expression this points to is deleted +** pExpr->pRight Any expression this points to is deleted. +** +** The zDb variable is the name of the database (the "X"). This value may be +** NULL meaning that name is of the form Y.Z or Z. Any available database +** can be used. The zTable variable is the name of the table (the "Y"). This +** value can be NULL if zDb is also NULL. If zTable is NULL it +** means that the form of the name is Z and that columns from any table +** can be used. +** +** If the name cannot be resolved unambiguously, leave an error message +** in pParse and return WRC_Abort. Return WRC_Prune on success. +*/ +static int lookupName( + Parse *pParse, /* The parsing context */ + const char *zDb, /* Name of the database containing table, or NULL */ + const char *zTab, /* Name of table containing column, or NULL */ + const char *zCol, /* Name of the column. */ + NameContext *pNC, /* The name context used to resolve the name */ + Expr *pExpr /* Make this EXPR node point to the selected column */ +){ + int i, j; /* Loop counters */ + int cnt = 0; /* Number of matching column names */ + int cntTab = 0; /* Number of matching table names */ + int nSubquery = 0; /* How many levels of subquery */ + sqlite3 *db = pParse->db; /* The database connection */ + SrcItem *pItem; /* Use for looping over pSrcList items */ + SrcItem *pMatch = 0; /* The matching pSrcList item */ + NameContext *pTopNC = pNC; /* First namecontext in the list */ + Schema *pSchema = 0; /* Schema of the expression */ + int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ + Table *pTab = 0; /* Table hold the row */ + Column *pCol; /* A column of pTab */ + + assert( pNC ); /* the name context cannot be NULL. */ + assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + + /* Initialize the node to no-match */ + pExpr->iTable = -1; + ExprSetVVAProperty(pExpr, EP_NoReduce); + + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IsCheck ); + if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ + /* Silently ignore database qualifiers inside CHECK constraints and + ** partial indices. Do not raise errors because that might break + ** legacy and because it does not hurt anything to just ignore the + ** database name. */ + zDb = 0; + }else{ + for(i=0; inDb; i++){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + if( i==db->nDb && sqlite3StrICmp("main", zDb)==0 ){ + /* This branch is taken when the main database has been renamed + ** using SQLITE_DBCONFIG_MAINDBNAME. */ + pSchema = db->aDb[0].pSchema; + zDb = db->aDb[0].zDbSName; + } + } + } + + /* Start at the inner-most context and move outward until a match is found */ + assert( pNC && cnt==0 ); + do{ + ExprList *pEList; + SrcList *pSrcList = pNC->pSrcList; + + if( pSrcList ){ + for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + u8 hCol; + pTab = pItem->pTab; + assert( pTab!=0 && pTab->zName!=0 ); + assert( pTab->nCol>0 ); + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; jnExpr; j++){ + if( sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; + } + } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } + if( IN_RENAME_OBJECT && pItem->zAlias ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); + } + } + if( 0==(cntTab++) ){ + pMatch = pItem; + } + hCol = sqlite3StrIHash(zCol); + for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ + if( pCol->hName==hCol && sqlite3StrICmp(pCol->zName, zCol)==0 ){ + /* If there has been exactly one prior match and this match + ** is for the right-hand table of a NATURAL JOIN or is in a + ** USING clause, then skip this match. + */ + if( cnt==1 ){ + if( pItem->fg.jointype & JT_NATURAL ) continue; + if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + break; + } + } + } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->y.pTab = pMatch->pTab; + /* RIGHT JOIN not (yet) supported */ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } + pSchema = pExpr->y.pTab->pSchema; + } + } /* if( pSrcList ) */ + +#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference. Or + ** maybe it is an excluded.* from an upsert. Or maybe it is + ** a reference in the RETURNING clause to a table being modified. + */ + if( cnt==0 && zDb==0 ){ + pTab = 0; +#ifndef SQLITE_OMIT_TRIGGER + if( pParse->pTriggerTab!=0 ){ + int op = pParse->eTriggerOp; + assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); + if( pParse->bReturning ){ + if( (pNC->ncFlags & NC_UBaseReg)!=0 + && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + ){ + pExpr->iTable = op!=TK_DELETE; + pTab = pParse->pTriggerTab; + } + }else if( op!=TK_DELETE && zTab && sqlite3StrICmp("new",zTab) == 0 ){ + pExpr->iTable = 1; + pTab = pParse->pTriggerTab; + }else if( op!=TK_INSERT && zTab && sqlite3StrICmp("old",zTab)==0 ){ + pExpr->iTable = 0; + pTab = pParse->pTriggerTab; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ +#ifndef SQLITE_OMIT_UPSERT + if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ + Upsert *pUpsert = pNC->uNC.pUpsert; + if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ + pTab = pUpsert->pUpsertSrc->a[0].pTab; + pExpr->iTable = EXCLUDED_TABLE_NUMBER; + } + } +#endif /* SQLITE_OMIT_UPSERT */ + + if( pTab ){ + int iCol; + u8 hCol = sqlite3StrIHash(zCol); + pSchema = pTab->pSchema; + cntTab++; + for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ + if( pCol->hName==hCol && sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( iCol==pTab->iPKey ){ + iCol = -1; + } + break; + } + } + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + /* IMP: R-51414-32910 */ + iCol = -1; + } + if( iColnCol ){ + cnt++; + pMatch = 0; +#ifndef SQLITE_OMIT_UPSERT + if( pExpr->iTable==EXCLUDED_TABLE_NUMBER ){ + testcase( iCol==(-1) ); + if( IN_RENAME_OBJECT ){ + pExpr->iColumn = iCol; + pExpr->y.pTab = pTab; + eNewExprOp = TK_COLUMN; + }else{ + pExpr->iTable = pNC->uNC.pUpsert->regData + + sqlite3TableColumnToStorage(pTab, iCol); + eNewExprOp = TK_REGISTER; + } + }else +#endif /* SQLITE_OMIT_UPSERT */ + { + pExpr->y.pTab = pTab; + if( pParse->bReturning ){ + eNewExprOp = TK_REGISTER; + pExpr->iTable = pNC->uNC.iBaseReg + (pTab->nCol+1)*pExpr->iTable + + sqlite3TableColumnToStorage(pTab, iCol) + 1; + }else{ + pExpr->iColumn = (i16)iCol; + eNewExprOp = TK_TRIGGER; +#ifndef SQLITE_OMIT_TRIGGER + if( iCol<0 ){ + pExpr->affExpr = SQLITE_AFF_INTEGER; + }else if( pExpr->iTable==0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<ncFlags & (NC_IdxExpr|NC_GenCol))==0 + && sqlite3IsRowid(zCol) + && VisibleRowid(pMatch->pTab) + ){ + cnt = 1; + pExpr->iColumn = -1; + pExpr->affExpr = SQLITE_AFF_INTEGER; + } + + /* + ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z + ** might refer to an result-set alias. This happens, for example, when + ** we are resolving names in the WHERE clause of the following command: + ** + ** SELECT a+b AS x FROM table WHERE x<10; + ** + ** In cases like this, replace pExpr with a copy of the expression that + ** forms the result set entry ("a+b" in the example) and return immediately. + ** Note that the expression in the result set should have already been + ** resolved by the time the WHERE clause is resolved. + ** + ** The ability to use an output result-set column in the WHERE, GROUP BY, + ** or HAVING clauses, or as part of a larger expression in the ORDER BY + ** clause is not standard SQL. This is a (goofy) SQLite extension, that + ** is supported for backwards compatibility only. Hence, we issue a warning + ** on sqlite3_log() whenever the capability is used. + */ + if( (pNC->ncFlags & NC_UEList)!=0 + && cnt==0 + && zTab==0 + ){ + pEList = pNC->uNC.pEList; + assert( pEList!=0 ); + for(j=0; jnExpr; j++){ + char *zAs = pEList->a[j].zEName; + if( pEList->a[j].eEName==ENAME_NAME + && sqlite3_stricmp(zAs, zCol)==0 + ){ + Expr *pOrig; + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->x.pList==0 ); + assert( pExpr->x.pSelect==0 ); + pOrig = pEList->a[j].pExpr; + if( (pNC->ncFlags&NC_AllowAgg)==0 && ExprHasProperty(pOrig, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); + return WRC_Abort; + } + if( ExprHasProperty(pOrig, EP_Win) + && ((pNC->ncFlags&NC_AllowWin)==0 || pNC!=pTopNC ) + ){ + sqlite3ErrorMsg(pParse, "misuse of aliased window function %s",zAs); + return WRC_Abort; + } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } + resolveAlias(pParse, pEList, j, pExpr, nSubquery); + cnt = 1; + pMatch = 0; + assert( zTab==0 && zDb==0 ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + } + goto lookupname_end; + } + } + } + + /* Advance to the next name context. The loop will exit when either + ** we have a match (cnt>0) or when we run out of name contexts. + */ + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 ){ + assert( pExpr->op==TK_ID ); + if( ExprHasProperty(pExpr,EP_DblQuoted) + && areDoubleQuotedStringsEnabled(db, pTopNC) + && (db->init.bDropColumn==0 || sqlite3StrICmp(zCol, db->init.azInit[0])!=0) + ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + ** + ** 2021-03-15: ticket 1c24a659e6d7f3a1 + ** Do not do the ID-to-STRING conversion when doing the schema + ** sanity check following a DROP COLUMN if the identifer name matches + ** the name of the column being dropped. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); +#endif + pExpr->op = TK_STRING; + pExpr->y.pTab = 0; + return WRC_Prune; + } + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + } + + /* + ** cnt==0 means there was not match. cnt>1 means there were two or + ** more matches. Either way, we have an error. + */ + if( cnt!=1 ){ + const char *zErr; + zErr = cnt==0 ? "no such column" : "ambiguous column name"; + if( zDb ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); + }else if( zTab ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + } + pParse->checkSchema = 1; + pTopNC->nErr++; + } + + /* If a column from a table in pSrcList is referenced, then record + ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes + ** bit 0 to be set. Column 1 sets bit 1. And so forth. Bit 63 is + ** set if the 63rd or any subsequent column is used. + ** + ** The colUsed mask is an optimization used to help determine if an + ** index is a covering index. The correct answer is still obtained + ** if the mask contains extra set bits. However, it is important to + ** avoid setting bits beyond the maximum column number of the table. + ** (See ticket [b92e5e8ec2cdbaa1]). + ** + ** If a generated column is referenced, set bits for every column + ** of the table. + */ + if( pExpr->iColumn>=0 && pMatch!=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + } + + /* Clean up and return + */ + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + } + pExpr->op = eNewExprOp; + ExprSetProperty(pExpr, EP_Leaf); +lookupname_end: + if( cnt==1 ){ + assert( pNC!=0 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pParse->db->xAuth + && (pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER) + ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } +#endif + /* Increment the nRef value on all name contexts from TopNC up to + ** the point where the name matched. */ + for(;;){ + assert( pTopNC!=0 ); + pTopNC->nRef++; + if( pTopNC==pNC ) break; + pTopNC = pTopNC->pNext; + } + return WRC_Prune; + } else { + return WRC_Abort; + } +} + +/* +** Allocate and return a pointer to an expression to load the column iCol +** from datasource iSrc in SrcList pSrc. +*/ +Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ + Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); + if( p ){ + SrcItem *pItem = &pSrc->a[iSrc]; + Table *pTab = p->y.pTab = pItem->pTab; + p->iTable = pItem->iCursor; + if( p->y.pTab->iPKey==iCol ){ + p->iColumn = -1; + }else{ + p->iColumn = (ynVar)iCol; + if( (pTab->tabFlags & TF_HasGenerated)!=0 + && (pTab->aCol[iCol].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pTab->nCol==63 ); + testcase( pTab->nCol==64 ); + pItem->colUsed = pTab->nCol>=64 ? ALLBITS : MASKBIT(pTab->nCol)-1; + }else{ + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + return p; +} + +/* +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. +** +** static void notValid( +** Parse *pParse, // Leave error message here +** NameContext *pNC, // The name context +** const char *zMsg, // Type of error +** int validMask, // Set of contexts for which prohibited +** Expr *pExpr // Invalidate this expression on error +** ){...} +** +** As an optimization, since the conditional is almost always false +** (because errors are rare), the conditional is moved outside of the +** function call using a macro. +*/ +static void notValidImpl( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg, /* Type of error */ + Expr *pExpr /* Invalidate this expression on error */ +){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; +#ifndef SQLITE_OMIT_CHECK + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pNC->ncFlags & NC_GenCol ) zIn = "generated columns"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); + if( pExpr ) pExpr->op = TK_NULL; +} +#define sqlite3ResolveNotValid(P,N,M,X,E) \ + assert( ((X)&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); \ + if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E); + +/* +** Expression p should encode a floating point value between 1.0 and 0.0. +** Return 1024 times this value. Or return -1 if p is not a floating point +** value between 1.0 and 0.0. +*/ +static int exprProbability(Expr *p){ + double r = -1.0; + if( p->op!=TK_FLOAT ) return -1; + sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + assert( r>=0.0 ); + if( r>1.0 ) return -1; + return (int)(r*134217728.0); +} + +/* +** This routine is callback for sqlite3WalkExpr(). +** +** Resolve symbolic names into TK_COLUMN operators for the current +** node in the expression tree. Return 0 to continue the search down +** the tree or 2 to abort the tree walk. +** +** This routine also does error checking and name resolution for +** function names. The operator for aggregate functions is changed +** to TK_AGG_FUNCTION. +*/ +static int resolveExprStep(Walker *pWalker, Expr *pExpr){ + NameContext *pNC; + Parse *pParse; + + pNC = pWalker->u.pNC; + assert( pNC!=0 ); + pParse = pNC->pParse; + assert( pParse==pWalker->pParse ); + +#ifndef NDEBUG + if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ + SrcList *pSrcList = pNC->pSrcList; + int i; + for(i=0; ipSrcList->nSrc; i++){ + assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); + } + } +#endif + switch( pExpr->op ){ + + /* The special operator TK_ROW means use the rowid for the first + ** column in the FROM clause. This is used by the LIMIT and ORDER BY + ** clause processing on UPDATE and DELETE statements, and by + ** UPDATE ... FROM statement processing. + */ + case TK_ROW: { + SrcList *pSrcList = pNC->pSrcList; + SrcItem *pItem; + assert( pSrcList && pSrcList->nSrc>=1 ); + pItem = pSrcList->a; + pExpr->op = TK_COLUMN; + pExpr->y.pTab = pItem->pTab; + pExpr->iTable = pItem->iCursor; + pExpr->iColumn--; + pExpr->affExpr = SQLITE_AFF_INTEGER; + break; + } + + /* An optimization: Attempt to convert + ** + ** "expr IS NOT NULL" --> "TRUE" + ** "expr IS NULL" --> "FALSE" + ** + ** if we can prove that "expr" is never NULL. Call this the + ** "NOT NULL strength reduction optimization". + ** + ** If this optimization occurs, also restore the NameContext ref-counts + ** to the state they where in before the "column" LHS expression was + ** resolved. This prevents "column" from being counted as having been + ** referenced, which might prevent a SELECT from being erroneously + ** marked as correlated. + */ + case TK_NOTNULL: + case TK_ISNULL: { + int anRef[8]; + NameContext *p; + int i; + for(i=0, p=pNC; p && ipNext, i++){ + anRef[i] = p->nRef; + } + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ + if( pExpr->op==TK_NOTNULL ){ + pExpr->u.zToken = "true"; + ExprSetProperty(pExpr, EP_IsTrue); + }else{ + pExpr->u.zToken = "false"; + ExprSetProperty(pExpr, EP_IsFalse); + } + pExpr->op = TK_TRUEFALSE; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; + } + return WRC_Prune; + } + + /* A column name: ID + ** Or table name and column name: ID.ID + ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. + */ + case TK_ID: + case TK_DOT: { + const char *zColumn; + const char *zTable; + const char *zDb; + Expr *pRight; + + if( pExpr->op==TK_ID ){ + zDb = 0; + zTable = 0; + zColumn = pExpr->u.zToken; + }else{ + Expr *pLeft = pExpr->pLeft; + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "the \".\" operator", + NC_IdxExpr|NC_GenCol, 0); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pLeft->u.zToken; + pLeft = pRight->pLeft; + pRight = pRight->pRight; + } + zTable = pLeft->u.zToken; + zColumn = pRight->u.zToken; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); + sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); + } + } + return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + } + + /* Resolve function names + */ + case TK_FUNCTION: { + ExprList *pList = pExpr->x.pList; /* The argument list */ + int n = pList ? pList->nExpr : 0; /* Number of arguments */ + int no_such_func = 0; /* True if no such function exists */ + int wrong_num_args = 0; /* True if wrong number of arguments */ + int is_agg = 0; /* True if is an aggregate function */ + int nId; /* Number of characters in function name */ + const char *zId; /* The function name. */ + FuncDef *pDef; /* Information about the function */ + u8 enc = ENC(pParse->db); /* The database encoding */ + int savedAllowFlags = (pNC->ncFlags & (NC_AllowAgg | NC_AllowWin)); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); +#endif + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); + if( pDef==0 ){ + pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); + if( pDef==0 ){ + no_such_func = 1; + }else{ + wrong_num_args = 1; + } + }else{ + is_agg = pDef->xFinalize!=0; + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + ExprSetProperty(pExpr, EP_Unlikely); + if( n==2 ){ + pExpr->iTable = exprProbability(pList->a[1].pExpr); + if( pExpr->iTable<0 ){ + sqlite3ErrorMsg(pParse, + "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); + pNC->nErr++; + } + }else{ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is + ** equivalent to likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is + ** short-hand for likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand + ** for likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent + ** to likelihood(X,0.9375). */ + /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; + } + } +#endif + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query. + ** This allows them to be factored out of inner loops. */ + ExprSetProperty(pExpr,EP_ConstFunc); + } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Clearly non-deterministic functions like random(), but also + ** date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index or generated column. Curiously, they can be used + ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all + ** all this. */ + sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx|NC_GenCol, 0); + }else{ + assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ + pExpr->op2 = pNC->ncFlags & NC_SelfRef; + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); + } + if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && pParse->nested==0 + && (pParse->db->mDbFlags & DBFLAG_InternalFunc)==0 + ){ + /* Internal-use-only functions are disallowed unless the + ** SQL is being compiled using sqlite3NestedParse() or + ** the SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test-control has be + ** used to activate internal functionsn for testing purposes */ + no_such_func = 1; + pDef = 0; + }else + if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 + && !IN_RENAME_OBJECT + ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); + } + } + + if( 0==IN_RENAME_OBJECT ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX) + || (pDef->xValue==0 && pDef->xInverse==0) + || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) + ); + if( pDef && pDef->xValue==0 && pWin ){ + sqlite3ErrorMsg(pParse, + "%.*s() may not be used as a window function", nId, zId + ); + pNC->nErr++; + }else if( + (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) + || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) + || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) + ){ + const char *zType; + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pWin ){ + zType = "window"; + }else{ + zType = "aggregate"; + } + sqlite3ErrorMsg(pParse, "misuse of %s function %.*s()",zType,nId,zId); + pNC->nErr++; + is_agg = 0; + } +#else + if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ + sqlite3ErrorMsg(pParse,"misuse of aggregate function %.*s()",nId,zId); + pNC->nErr++; + is_agg = 0; + } +#endif + else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ErrorMsg(pParse, + "FILTER may not be used with non-aggregate %.*s()", + nId, zId + ); + pNC->nErr++; + } +#endif + if( is_agg ){ + /* Window functions may not be arguments of aggregate functions. + ** Or arguments of other window functions. But aggregate functions + ** may be arguments for window functions. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + pNC->ncFlags &= ~(NC_AllowWin | (!pWin ? NC_AllowAgg : 0)); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif + } + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + is_agg = 1; + } +#endif + sqlite3WalkExprList(pWalker, pList); + if( is_agg ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + Select *pSel = pNC->pWinSelect; + assert( pWin==pExpr->y.pWin ); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); + if( pParse->db->mallocFailed ) break; + } + sqlite3WalkExprList(pWalker, pWin->pPartition); + sqlite3WalkExprList(pWalker, pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pWin->pFilter); + sqlite3WindowLink(pSel, pWin); + pNC->ncFlags |= NC_HasWin; + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + NameContext *pNC2 = pNC; + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); + } +#endif + while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ + pExpr->op2++; + pNC2 = pNC2->pNext; + } + assert( pDef!=0 || IN_RENAME_OBJECT ); + if( pNC2 && pDef ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + + } + } + pNC->ncFlags |= savedAllowFlags; + } + /* FIX ME: Compute pExpr->affinity based on the expected return + ** type of the function + */ + return WRC_Prune; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: + case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); +#endif + case TK_IN: { + testcase( pExpr->op==TK_IN ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + int nRef = pNC->nRef; + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "subqueries", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); + sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + assert( pNC->nRef>=nRef ); + if( nRef!=pNC->nRef ){ + ExprSetProperty(pExpr, EP_VarSelect); + pNC->ncFlags |= NC_VarSelect; + } + } + break; + } + case TK_VARIABLE: { + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "parameters", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); + break; + } + case TK_IS: + case TK_ISNOT: { + Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); + assert( !ExprHasProperty(pExpr, EP_Reduced) ); + /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", + ** and "x IS NOT FALSE". */ + if( ALWAYS(pRight) && (pRight->op==TK_ID || pRight->op==TK_TRUEFALSE) ){ + int rc = resolveExprStep(pWalker, pRight); + if( rc==WRC_Abort ) return WRC_Abort; + if( pRight->op==TK_TRUEFALSE ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_TRUTH; + return WRC_Continue; + } + } + /* no break */ deliberate_fall_through + } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + } + break; + } + } + return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; +} + +/* +** pEList is a list of expressions which are really the result set of the +** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. +** This routine checks to see if pE is a simple identifier which corresponds +** to the AS-name of one of the terms of the expression list. If it is, +** this routine return an integer between 1 and N where N is the number of +** elements in pEList, corresponding to the matching entry. If there is +** no match, or if pE is not a simple identifier, then this routine +** return 0. +** +** pEList has been resolved. pE has not. +*/ +static int resolveAsName( + Parse *pParse, /* Parsing context for error messages */ + ExprList *pEList, /* List of expressions to scan */ + Expr *pE /* Expression we are trying to match */ +){ + int i; /* Loop counter */ + + UNUSED_PARAMETER(pParse); + + if( pE->op==TK_ID ){ + char *zCol = pE->u.zToken; + for(i=0; inExpr; i++){ + if( pEList->a[i].eEName==ENAME_NAME + && sqlite3_stricmp(pEList->a[i].zEName, zCol)==0 + ){ + return i+1; + } + } + } + return 0; +} + +/* +** pE is a pointer to an expression which is a single term in the +** ORDER BY of a compound SELECT. The expression has not been +** name resolved. +** +** At the point this routine is called, we already know that the +** ORDER BY term is not an integer index into the result set. That +** case is handled by the calling routine. +** +** Attempt to match pE against result set columns in the left-most +** SELECT statement. Return the index i of the matching column, +** as an indication to the caller that it should sort by the i-th column. +** The left-most column is 1. In other words, the value returned is the +** same integer value that would be used in the SQL statement to indicate +** the column. +** +** If there is no match, return 0. Return -1 if an error occurs. +*/ +static int resolveOrderByTermToExprList( + Parse *pParse, /* Parsing context for error messages */ + Select *pSelect, /* The SELECT statement with the ORDER BY clause */ + Expr *pE /* The specific ORDER BY term */ +){ + int i; /* Loop counter */ + ExprList *pEList; /* The columns of the result set */ + NameContext nc; /* Name context for resolving pE */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code from subprocedures */ + u8 savedSuppErr; /* Saved value of db->suppressErr */ + + assert( sqlite3ExprIsInteger(pE, &i)==0 ); + pEList = pSelect->pEList; + + /* Resolve all names in the ORDER BY term expression + */ + memset(&nc, 0, sizeof(nc)); + nc.pParse = pParse; + nc.pSrcList = pSelect->pSrc; + nc.uNC.pEList = pEList; + nc.ncFlags = NC_AllowAgg|NC_UEList; + nc.nErr = 0; + db = pParse->db; + savedSuppErr = db->suppressErr; + if( IN_RENAME_OBJECT==0 ) db->suppressErr = 1; + rc = sqlite3ResolveExprNames(&nc, pE); + db->suppressErr = savedSuppErr; + if( rc ) return 0; + + /* Try to match the ORDER BY expression against an expression + ** in the result set. Return an 1-based index of the matching + ** result-set entry. + */ + for(i=0; inExpr; i++){ + if( sqlite3ExprCompare(0, pEList->a[i].pExpr, pE, -1)<2 ){ + return i+1; + } + } + + /* If no match, return 0. */ + return 0; +} + +/* +** Generate an ORDER BY or GROUP BY term out-of-range error. +*/ +static void resolveOutOfRangeError( + Parse *pParse, /* The error context into which to write the error */ + const char *zType, /* "ORDER" or "GROUP" */ + int i, /* The index (1-based) of the term out of range */ + int mx /* Largest permissible value of i */ +){ + sqlite3ErrorMsg(pParse, + "%r %s BY term out of range - should be " + "between 1 and %d", i, zType, mx); +} + +/* +** Analyze the ORDER BY clause in a compound SELECT statement. Modify +** each term of the ORDER BY clause is a constant integer between 1 +** and N where N is the number of columns in the compound SELECT. +** +** ORDER BY terms that are already an integer between 1 and N are +** unmodified. ORDER BY terms that are integers outside the range of +** 1 through N generate an error. ORDER BY terms that are expressions +** are matched against result set expressions of compound SELECT +** beginning with the left-most SELECT and working toward the right. +** At the first match, the ORDER BY expression is transformed into +** the integer column number. +** +** Return the number of errors seen. +*/ +static int resolveCompoundOrderBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect /* The SELECT statement containing the ORDER BY */ +){ + int i; + ExprList *pOrderBy; + ExprList *pEList; + sqlite3 *db; + int moreToDo = 1; + + pOrderBy = pSelect->pOrderBy; + if( pOrderBy==0 ) return 0; + db = pParse->db; + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + return 1; + } + for(i=0; inExpr; i++){ + pOrderBy->a[i].done = 0; + } + pSelect->pNext = 0; + while( pSelect->pPrior ){ + pSelect->pPrior->pNext = pSelect; + pSelect = pSelect->pPrior; + } + while( pSelect && moreToDo ){ + struct ExprList_item *pItem; + moreToDo = 0; + pEList = pSelect->pEList; + assert( pEList!=0 ); + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + int iCol = -1; + Expr *pE, *pDup; + if( pItem->done ) continue; + pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); + if( NEVER(pE==0) ) continue; + if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( iCol<=0 || iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + return 1; + } + }else{ + iCol = resolveAsName(pParse, pEList, pE); + if( iCol==0 ){ + /* Now test if expression pE matches one of the values returned + ** by pSelect. In the usual case this is done by duplicating the + ** expression, resolving any symbols in it, and then comparing + ** it against each expression returned by the SELECT statement. + ** Once the comparisons are finished, the duplicate expression + ** is deleted. + ** + ** Or, if this is running as part of an ALTER TABLE operation, + ** resolve the symbols in the actual expression, not a duplicate. + ** And, if one of the comparisons is successful, leave the expression + ** as is instead of transforming it to an integer as in the usual + ** case. This allows the code in alter.c to modify column + ** refererences within the ORDER BY expression as required. */ + if( IN_RENAME_OBJECT ){ + pDup = pE; + }else{ + pDup = sqlite3ExprDup(db, pE, 0); + } + if( !db->mallocFailed ){ + assert(pDup); + iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + } + if( !IN_RENAME_OBJECT ){ + sqlite3ExprDelete(db, pDup); + } + } + } + if( iCol>0 ){ + /* Convert the ORDER BY term into an integer column number iCol, + ** taking care to preserve the COLLATE clause if it exists */ + if( !IN_RENAME_OBJECT ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; + } + sqlite3ExprDelete(db, pE); + pItem->u.x.iOrderByCol = (u16)iCol; + } + pItem->done = 1; + }else{ + moreToDo = 1; + } + } + pSelect = pSelect->pNext; + } + for(i=0; inExpr; i++){ + if( pOrderBy->a[i].done==0 ){ + sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " + "column in the result set", i+1); + return 1; + } + } + return 0; +} + +/* +** Check every term in the ORDER BY or GROUP BY clause pOrderBy of +** the SELECT statement pSelect. If any term is reference to a +** result set expression (as determined by the ExprList.a.u.x.iOrderByCol +** field) then convert that term into a copy of the corresponding result set +** column. +** +** If any errors are detected, add an error message to pParse and +** return non-zero. Return zero if no errors are seen. +*/ +int sqlite3ResolveOrderGroupBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect, /* The SELECT statement containing the clause */ + ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ + const char *zType /* "ORDER" or "GROUP" */ +){ + int i; + sqlite3 *db = pParse->db; + ExprList *pEList; + struct ExprList_item *pItem; + + if( pOrderBy==0 || pParse->db->mallocFailed || IN_RENAME_OBJECT ) return 0; + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); + return 1; + } + pEList = pSelect->pEList; + assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + if( pItem->u.x.iOrderByCol ){ + if( pItem->u.x.iOrderByCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + return 1; + } + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,0); + } + } + return 0; +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Walker callback for windowRemoveExprFromSelect(). +*/ +static int resolveRemoveWindowsCb(Walker *pWalker, Expr *pExpr){ + UNUSED_PARAMETER(pWalker); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + sqlite3WindowUnlinkFromSelect(pWin); + } + return WRC_Continue; +} + +/* +** Remove any Window objects owned by the expression pExpr from the +** Select.pWin list of Select object pSelect. +*/ +static void windowRemoveExprFromSelect(Select *pSelect, Expr *pExpr){ + if( pSelect->pWin ){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.xExprCallback = resolveRemoveWindowsCb; + sWalker.u.pSelect = pSelect; + sqlite3WalkExpr(&sWalker, pExpr); + } +} +#else +# define windowRemoveExprFromSelect(a, b) +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. +** The Name context of the SELECT statement is pNC. zType is either +** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. +** +** This routine resolves each term of the clause into an expression. +** If the order-by term is an integer I between 1 and N (where N is the +** number of columns in the result set of the SELECT) then the expression +** in the resolution is a copy of the I-th result-set expression. If +** the order-by term is an identifier that corresponds to the AS-name of +** a result-set expression, then the term resolves to a copy of the +** result-set expression. Otherwise, the expression is resolved in +** the usual way - using sqlite3ResolveExprNames(). +** +** This routine returns the number of errors. If errors occur, then +** an appropriate error message might be left in pParse. (OOM errors +** excepted.) +*/ +static int resolveOrderGroupBy( + NameContext *pNC, /* The name context of the SELECT statement */ + Select *pSelect, /* The SELECT statement holding pOrderBy */ + ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ + const char *zType /* Either "ORDER" or "GROUP", as appropriate */ +){ + int i, j; /* Loop counters */ + int iCol; /* Column number */ + struct ExprList_item *pItem; /* A term of the ORDER BY clause */ + Parse *pParse; /* Parsing context */ + int nResult; /* Number of terms in the result set */ + + if( pOrderBy==0 ) return 0; + nResult = pSelect->pEList->nExpr; + pParse = pNC->pParse; + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + Expr *pE = pItem->pExpr; + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE); + if( NEVER(pE2==0) ) continue; + if( zType[0]!='G' ){ + iCol = resolveAsName(pParse, pSelect->pEList, pE2); + if( iCol>0 ){ + /* If an AS-name match is found, mark this ORDER BY column as being + ** a copy of the iCol-th result-set column. The subsequent call to + ** sqlite3ResolveOrderGroupBy() will convert the expression to a + ** copy of the iCol-th result-set expression. */ + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + } + if( sqlite3ExprIsInteger(pE2, &iCol) ){ + /* The ORDER BY term is an integer constant. Again, set the column + ** number so that sqlite3ResolveOrderGroupBy() will convert the + ** order-by term to a copy of the result-set expression */ + if( iCol<1 || iCol>0xffff ){ + resolveOutOfRangeError(pParse, zType, i+1, nResult); + return 1; + } + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + + /* Otherwise, treat the ORDER BY term as an ordinary expression */ + pItem->u.x.iOrderByCol = 0; + if( sqlite3ResolveExprNames(pNC, pE) ){ + return 1; + } + for(j=0; jpEList->nExpr; j++){ + if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + /* Since this expresion is being changed into a reference + ** to an identical expression in the result set, remove all Window + ** objects belonging to the expression from the Select.pWin list. */ + windowRemoveExprFromSelect(pSelect, pE); + pItem->u.x.iOrderByCol = j+1; + } + } + } + return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); +} + +/* +** Resolve names in the SELECT statement p and all of its descendants. +*/ +static int resolveSelectStep(Walker *pWalker, Select *p){ + NameContext *pOuterNC; /* Context that contains this SELECT */ + NameContext sNC; /* Name context of this SELECT */ + int isCompound; /* True if p is a compound select */ + int nCompound; /* Number of compound terms processed so far */ + Parse *pParse; /* Parsing context */ + int i; /* Loop counter */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Select *pLeftmost; /* Left-most of SELECT of a compound */ + sqlite3 *db; /* Database connection */ + + + assert( p!=0 ); + if( p->selFlags & SF_Resolved ){ + return WRC_Prune; + } + pOuterNC = pWalker->u.pNC; + pParse = pWalker->pParse; + db = pParse->db; + + /* Normally sqlite3SelectExpand() will be called first and will have + ** already expanded this SELECT. However, if this is a subquery within + ** an expression, sqlite3ResolveExprNames() will be called without a + ** prior call to sqlite3SelectExpand(). When that happens, let + ** sqlite3SelectPrep() do all of the processing for this SELECT. + ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and + ** this routine in the correct order. + */ + if( (p->selFlags & SF_Expanded)==0 ){ + sqlite3SelectPrep(pParse, p, pOuterNC); + return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + } + + isCompound = p->pPrior!=0; + nCompound = 0; + pLeftmost = p; + while( p ){ + assert( (p->selFlags & SF_Expanded)!=0 ); + assert( (p->selFlags & SF_Resolved)==0 ); + p->selFlags |= SF_Resolved; + + /* Resolve the expressions in the LIMIT and OFFSET clauses. These + ** are not allowed to refer to any names, so pass an empty NameContext. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pWinSelect = p; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ + return WRC_Abort; + } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } + + /* Recursively resolve names in all subqueries + */ + for(i=0; ipSrc->nSrc; i++){ + SrcItem *pItem = &p->pSrc->a[i]; + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + int nRef = pOuterNC ? pOuterNC->nRef : 0; + const char *zSavedContext = pParse->zAuthContext; + + if( pItem->zName ) pParse->zAuthContext = pItem->zName; + sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + pParse->zAuthContext = zSavedContext; + if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + /* If the number of references to the outer context changed when + ** expressions in the sub-select were resolved, the sub-select + ** is correlated. It is not required to check the refcount on any + ** but the innermost outer context object, as lookupName() increments + ** the refcount on all contexts between the current one and the + ** context containing the column when it resolves a name. */ + if( pOuterNC ){ + assert( pItem->fg.isCorrelated==0 && pOuterNC->nRef>=nRef ); + pItem->fg.isCorrelated = (pOuterNC->nRef>nRef); + } + } + } + + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg|NC_AllowWin; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AllowWin; + + /* If there are no aggregate functions in the result-set, and no GROUP BY + ** expression, do not allow aggregates in any of the other expressions. + */ + assert( (p->selFlags & SF_Aggregate)==0 ); + pGroupBy = p->pGroupBy; + if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ + assert( NC_MinMaxAgg==SF_MinMaxAgg ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&NC_MinMaxAgg); + }else{ + sNC.ncFlags &= ~NC_AllowAgg; + } + + /* If a HAVING clause is present, then there must be a GROUP BY clause. + */ + if( p->pHaving && !pGroupBy ){ + sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + return WRC_Abort; + } + + /* Add the output column list to the name-context before parsing the + ** other expressions in the SELECT statement. This is so that + ** expressions in the WHERE clause (etc.) can refer to expressions by + ** aliases in the result set. + ** + ** Minor point: If this is the case, then the expression will be + ** re-evaluated for each reference to it. + */ + assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert|NC_UBaseReg))==0 ); + sNC.uNC.pEList = p->pEList; + sNC.ncFlags |= NC_UEList; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + SrcItem *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; + sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; + + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } + + /* Process the ORDER BY clause for singleton SELECT statements. + ** The ORDER BY clause for compounds SELECT statements is handled + ** below, after all of the result-sets for all of the elements of + ** the compound have been resolved. + ** + ** If there is an ORDER BY clause on a term of a compound-select other + ** than the right-most term, then that is a syntax error. But the error + ** is not detected until much later, and so we need to go ahead and + ** resolve those symbols on the incorrect ORDER BY for consistency. + */ + if( isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ + && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") + ){ + return WRC_Abort; + } + if( db->mallocFailed ){ + return WRC_Abort; + } + sNC.ncFlags &= ~NC_AllowWin; + + /* Resolve the GROUP BY clause. At the same time, make sure + ** the GROUP BY clause does not contain aggregate functions. + */ + if( pGroupBy ){ + struct ExprList_item *pItem; + + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ + return WRC_Abort; + } + for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ + if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " + "the GROUP BY clause"); + return WRC_Abort; + } + } + } + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( IN_RENAME_OBJECT ){ + Window *pWin; + for(pWin=p->pWinDefn; pWin; pWin=pWin->pNextWin){ + if( sqlite3ResolveExprListNames(&sNC, pWin->pOrderBy) + || sqlite3ResolveExprListNames(&sNC, pWin->pPartition) + ){ + return WRC_Abort; + } + } + } +#endif + + /* If this is part of a compound SELECT, check that it has the right + ** number of expressions in the select list. */ + if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pNext); + return WRC_Abort; + } + + /* Advance to the next term of the compound + */ + p = p->pPrior; + nCompound++; + } + + /* Resolve the ORDER BY on a compound SELECT after all terms of + ** the compound have been resolved. + */ + if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ + return WRC_Abort; + } + + return WRC_Prune; +} + +/* +** This routine walks an expression tree and resolves references to +** table columns and result-set columns. At the same time, do error +** checking on function usage and set a flag if any aggregate functions +** are seen. +** +** To resolve table columns references we look for nodes (or subtrees) of the +** form X.Y.Z or Y.Z or just Z where +** +** X: The name of a database. Ex: "main" or "temp" or +** the symbolic name assigned to an ATTACH-ed database. +** +** Y: The name of a table in a FROM clause. Or in a trigger +** one of the special names "old" or "new". +** +** Z: The name of a column in table Y. +** +** The node at the root of the subtree is modified as follows: +** +** Expr.op Changed to TK_COLUMN +** Expr.pTab Points to the Table object for X.Y +** Expr.iColumn The column index in X.Y. -1 for the rowid. +** Expr.iTable The VDBE cursor number for X.Y +** +** +** To resolve result-set references, look for expression nodes of the +** form Z (with no X and Y prefix) where the Z matches the right-hand +** size of an AS clause in the result-set of a SELECT. The Z expression +** is replaced by a copy of the left-hand side of the result-set expression. +** Table-name and function resolution occurs on the substituted expression +** tree. For example, in: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** +** The "x" term of the order by is replaced by "a+b" to render: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** +** Function calls are checked to make sure that the function is +** defined and that the correct number of arguments are specified. +** If the function is an aggregate function, then the NC_HasAgg flag is +** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. +** If an expression contains aggregate functions then the EP_Agg +** property on the expression is set. +** +** An error message is left in pParse if anything is amiss. The number +** if errors is returned. +*/ +int sqlite3ResolveExprNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + Expr *pExpr /* The expression to be analyzed. */ +){ + int savedHasAgg; + Walker w; + + if( pExpr==0 ) return SQLITE_OK; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return SQLITE_ERROR; + } +#endif + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + pNC->ncFlags |= savedHasAgg; + return pNC->nErr>0 || w.pParse->nErr>0; +} + +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + int savedHasAgg = 0; + Walker w; + if( pList==0 ) return WRC_Continue; + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr==0 ) continue; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return WRC_Abort; + } +#endif + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + if( pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin) ){ + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + savedHasAgg |= pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + } + if( pNC->nErr>0 || w.pParse->nErr>0 ) return WRC_Abort; + } + pNC->ncFlags |= savedHasAgg; + return WRC_Continue; +} + +/* +** Resolve all names in all expressions of a SELECT and in all +** decendents of the SELECT, including compounds off of p->pPrior, +** subqueries in expressions, and subqueries used as FROM clause +** terms. +** +** See sqlite3ResolveExprNames() for a description of the kinds of +** transformations that occur. +** +** All SELECT statements should have been expanded using +** sqlite3SelectExpand() prior to invoking this routine. +*/ +void sqlite3ResolveSelectNames( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for parent SELECT statement */ +){ + Walker w; + + assert( p!=0 ); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.pParse = pParse; + w.u.pNC = pOuterNC; + sqlite3WalkSelect(&w, p); +} + +/* +** Resolve names in expressions that can only reference a single table +** or which cannot reference any tables at all. Examples: +** +** "type" flag +** ------------ +** (1) CHECK constraints NC_IsCheck +** (2) WHERE clauses on partial indices NC_PartIdx +** (3) Expressions in indexes on expressions NC_IdxExpr +** (4) Expression arguments to VACUUM INTO. 0 +** (5) GENERATED ALWAYS as expressions NC_GenCol +** +** In all cases except (4), the Expr.iTable value for Expr.op==TK_COLUMN +** nodes of the expression is set to -1 and the Expr.iColumn value is +** set to the column number. In case (4), TK_COLUMN nodes cause an error. +** +** Any errors cause an error message to be set in pParse. +*/ +int sqlite3ResolveSelfReference( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced, or NULL */ + int type, /* NC_IsCheck, NC_PartIdx, NC_IdxExpr, NC_GenCol, or 0 */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NULL. */ +){ + SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + NameContext sNC; /* Name context for pParse->pNewTable */ + int rc; + + assert( type==0 || pTab!=0 ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr + || type==NC_GenCol || pTab==0 ); + memset(&sNC, 0, sizeof(sNC)); + memset(&sSrc, 0, sizeof(sSrc)); + if( pTab ){ + sSrc.nSrc = 1; + sSrc.a[0].zName = pTab->zName; + sSrc.a[0].pTab = pTab; + sSrc.a[0].iCursor = -1; + if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ + /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP + ** schema elements */ + type |= NC_FromDDL; + } + } + sNC.pParse = pParse; + sNC.pSrcList = &sSrc; + sNC.ncFlags = type | NC_IsDDL; + if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; + if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); + return rc; +} diff --git a/third_party/sqlite3/rowset.c b/third_party/sqlite3/rowset.c new file mode 100644 index 000000000..5f0a9b8ff --- /dev/null +++ b/third_party/sqlite3/rowset.c @@ -0,0 +1,503 @@ +/* +** 2008 December 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements an object we call a "RowSet". +** +** The RowSet object is a collection of rowids. Rowids +** are inserted into the RowSet in an arbitrary order. Inserts +** can be intermixed with tests to see if a given rowid has been +** previously inserted into the RowSet. +** +** After all inserts are finished, it is possible to extract the +** elements of the RowSet in sorted order. Once this extraction +** process has started, no new elements may be inserted. +** +** Hence, the primitive operations for a RowSet are: +** +** CREATE +** INSERT +** TEST +** SMALLEST +** DESTROY +** +** The CREATE and DESTROY primitives are the constructor and destructor, +** obviously. The INSERT primitive adds a new element to the RowSet. +** TEST checks to see if an element is already in the RowSet. SMALLEST +** extracts the least value from the RowSet. +** +** The INSERT primitive might allocate additional memory. Memory is +** allocated in chunks so most INSERTs do no allocation. There is an +** upper bound on the size of allocated memory. No memory is freed +** until DESTROY. +** +** The TEST primitive includes a "batch" number. The TEST primitive +** will only see elements that were inserted before the last change +** in the batch number. In other words, if an INSERT occurs between +** two TESTs where the TESTs have the same batch nubmer, then the +** value added by the INSERT will not be visible to the second TEST. +** The initial batch number is zero, so if the very first TEST contains +** a non-zero batch number, it will see all prior INSERTs. +** +** No INSERTs may occurs after a SMALLEST. An assertion will fail if +** that is attempted. +** +** The cost of an INSERT is roughly constant. (Sometimes new memory +** has to be allocated on an INSERT.) The cost of a TEST with a new +** batch number is O(NlogN) where N is the number of elements in the RowSet. +** The cost of a TEST using the same batch number is O(logN). The cost +** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST +** primitives are constant time. The cost of DESTROY is O(N). +** +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + + +/* +** Target size for allocation chunks. +*/ +#define ROWSET_ALLOCATION_SIZE 1024 + +/* +** The number of rowset entries per allocation chunk. +*/ +#define ROWSET_ENTRY_PER_CHUNK \ + ((ROWSET_ALLOCATION_SIZE-8)/sizeof(struct RowSetEntry)) + +/* +** Each entry in a RowSet is an instance of the following object. +** +** This same object is reused to store a linked list of trees of RowSetEntry +** objects. In that alternative use, pRight points to the next entry +** in the list, pLeft points to the tree, and v is unused. The +** RowSet.pForest value points to the head of this forest list. +*/ +struct RowSetEntry { + i64 v; /* ROWID value for this entry */ + struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ + struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ +}; + +/* +** RowSetEntry objects are allocated in large chunks (instances of the +** following structure) to reduce memory allocation overhead. The +** chunks are kept on a linked list so that they can be deallocated +** when the RowSet is destroyed. +*/ +struct RowSetChunk { + struct RowSetChunk *pNextChunk; /* Next chunk on list of them all */ + struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ +}; + +/* +** A RowSet in an instance of the following structure. +** +** A typedef of this structure if found in sqliteInt.h. +*/ +struct RowSet { + struct RowSetChunk *pChunk; /* List of all chunk allocations */ + sqlite3 *db; /* The database connection */ + struct RowSetEntry *pEntry; /* List of entries using pRight */ + struct RowSetEntry *pLast; /* Last entry on the pEntry list */ + struct RowSetEntry *pFresh; /* Source of new entry objects */ + struct RowSetEntry *pForest; /* List of binary trees of entries */ + u16 nFresh; /* Number of objects on pFresh */ + u16 rsFlags; /* Various flags */ + int iBatch; /* Current insert batch */ +}; + +/* +** Allowed values for RowSet.rsFlags +*/ +#define ROWSET_SORTED 0x01 /* True if RowSet.pEntry is sorted */ +#define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ + +/* +** Allocate a RowSet object. Return NULL if a memory allocation +** error occurs. +*/ +RowSet *sqlite3RowSetInit(sqlite3 *db){ + RowSet *p = sqlite3DbMallocRawNN(db, sizeof(*p)); + if( p ){ + int N = sqlite3DbMallocSize(db, p); + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + } + return p; +} + +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +void sqlite3RowSetClear(void *pArg){ + RowSet *p = (RowSet*)pArg; + struct RowSetChunk *pChunk, *pNextChunk; + for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ + pNextChunk = pChunk->pNextChunk; + sqlite3DbFree(p->db, pChunk); + } + p->pChunk = 0; + p->nFresh = 0; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->rsFlags = ROWSET_SORTED; +} + +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +void sqlite3RowSetDelete(void *pArg){ + sqlite3RowSetClear(pArg); + sqlite3DbFree(((RowSet*)pArg)->db, pArg); +} + +/* +** Allocate a new RowSetEntry object that is associated with the +** given RowSet. Return a pointer to the new and completely uninitialized +** object. +** +** In an OOM situation, the RowSet.db->mallocFailed flag is set and this +** routine returns NULL. +*/ +static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ + assert( p!=0 ); + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ + struct RowSetChunk *pNew; + pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); + if( pNew==0 ){ + return 0; + } + pNew->pNextChunk = p->pChunk; + p->pChunk = pNew; + p->pFresh = pNew->aEntry; + p->nFresh = ROWSET_ENTRY_PER_CHUNK; + } + p->nFresh--; + return p->pFresh++; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + + /* This routine is never called after sqlite3RowSetNext() */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + pEntry = rowSetEntryAlloc(p); + if( pEntry==0 ) return; + pEntry->v = rowid; + pEntry->pRight = 0; + pLast = p->pLast; + if( pLast ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ + p->rsFlags &= ~ROWSET_SORTED; + } + pLast->pRight = pEntry; + }else{ + p->pEntry = pEntry; + } + p->pLast = pEntry; +} + +/* +** Merge two lists of RowSetEntry objects. Remove duplicates. +** +** The input lists are connected via pRight pointers and are +** assumed to each already be in sorted order. +*/ +static struct RowSetEntry *rowSetEntryMerge( + struct RowSetEntry *pA, /* First sorted list to be merged */ + struct RowSetEntry *pB /* Second sorted list to be merged */ +){ + struct RowSetEntry head; + struct RowSetEntry *pTail; + + pTail = &head; + assert( pA!=0 && pB!=0 ); + for(;;){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + assert( pB->pRight==0 || pB->v<=pB->pRight->v ); + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; + pA = pA->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } + }else{ + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } + } + } + return head.pRight; +} + +/* +** Sort all elements on the list of RowSetEntry objects into order of +** increasing v. +*/ +static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ + unsigned int i; + struct RowSetEntry *pNext, *aBucket[40]; + + memset(aBucket, 0, sizeof(aBucket)); + while( pIn ){ + pNext = pIn->pRight; + pIn->pRight = 0; + for(i=0; aBucket[i]; i++){ + pIn = rowSetEntryMerge(aBucket[i], pIn); + aBucket[i] = 0; + } + aBucket[i] = pIn; + pIn = pNext; + } + pIn = aBucket[0]; + for(i=1; ipLeft ){ + struct RowSetEntry *p; + rowSetTreeToList(pIn->pLeft, ppFirst, &p); + p->pRight = pIn; + }else{ + *ppFirst = pIn; + } + if( pIn->pRight ){ + rowSetTreeToList(pIn->pRight, &pIn->pRight, ppLast); + }else{ + *ppLast = pIn; + } + assert( (*ppLast)->pRight==0 ); +} + + +/* +** Convert a sorted list of elements (connected by pRight) into a binary +** tree with depth of iDepth. A depth of 1 means the tree contains a single +** node taken from the head of *ppList. A depth of 2 means a tree with +** three nodes. And so forth. +** +** Use as many entries from the input list as required and update the +** *ppList to point to the unused elements of the list. If the input +** list contains too few elements, then construct an incomplete tree +** and leave *ppList set to NULL. +** +** Return a pointer to the root of the constructed binary tree. +*/ +static struct RowSetEntry *rowSetNDeepTree( + struct RowSetEntry **ppList, + int iDepth +){ + struct RowSetEntry *p; /* Root of the new tree */ + struct RowSetEntry *pLeft; /* Left subtree */ + if( *ppList==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Prevent unnecessary deep recursion when we run out of entries */ + return 0; + } + if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ + p = *ppList; + *ppList = p->pRight; + p->pLeft = p->pRight = 0; + } + return p; +} + +/* +** Convert a sorted list of elements into a binary tree. Make the tree +** as deep as it needs to be in order to contain the entire list. +*/ +static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ + int iDepth; /* Depth of the tree so far */ + struct RowSetEntry *p; /* Current tree root */ + struct RowSetEntry *pLeft; /* Left subtree */ + + assert( pList!=0 ); + p = pList; + pList = p->pRight; + p->pLeft = p->pRight = 0; + for(iDepth=1; pList; iDepth++){ + pLeft = p; + p = pList; + pList = p->pRight; + p->pLeft = pLeft; + p->pRight = rowSetNDeepTree(&pList, iDepth); + } + return p; +} + +/* +** Extract the smallest element from the RowSet. +** Write the element into *pRowid. Return 1 on success. Return +** 0 if the RowSet is already empty. +** +** After this routine has been called, the sqlite3RowSetInsert() +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. +*/ +int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ + assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ + + /* Merge the forest into a single sorted list on first call */ + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } + + /* Return the next entry on the list */ + if( p->pEntry ){ + *pRowid = p->pEntry->v; + p->pEntry = p->pEntry->pRight; + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ + sqlite3RowSetClear(p); + } + return 1; + }else{ + return 0; + } +} + +/* +** Check to see if element iRowid was inserted into the rowset as +** part of any insert batch prior to iBatch. Return 1 or 0. +** +** If this is the first test of a new batch and if there exist entries +** on pRowSet->pEntry, then sort those entries into the forest at +** pRowSet->pForest so that they can be tested. +*/ +int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ + struct RowSetEntry *p, *pTree; + + /* This routine is never called after sqlite3RowSetNext() */ + assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); + + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. + */ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ + p = pRowSet->pEntry; + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entries if they need it */ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + ppPrevTree = &pTree->pRight; + if( pTree->pLeft==0 ){ + pTree->pLeft = rowSetListToTree(p); + break; + }else{ + struct RowSetEntry *pAux, *pTail; + rowSetTreeToList(pTree->pLeft, &pAux, &pTail); + pTree->pLeft = 0; + p = rowSetEntryMerge(pAux, p); + } + } + if( pTree==0 ){ + *ppPrevTree = pTree = rowSetEntryAlloc(pRowSet); + if( pTree ){ + pTree->v = 0; + pTree->pRight = 0; + pTree->pLeft = rowSetListToTree(p); + } + } + pRowSet->pEntry = 0; + pRowSet->pLast = 0; + pRowSet->rsFlags |= ROWSET_SORTED; + } + pRowSet->iBatch = iBatch; + } + + /* Test to see if the iRowid value appears anywhere in the forest. + ** Return 1 if it does and 0 if not. + */ + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + p = pTree->pLeft; + while( p ){ + if( p->vpRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } + } + } + return 0; +} diff --git a/third_party/sqlite3/rtree.c b/third_party/sqlite3/rtree.c new file mode 100644 index 000000000..916d31739 --- /dev/null +++ b/third_party/sqlite3/rtree.c @@ -0,0 +1,4546 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code for implementations of the r-tree and r*-tree +** algorithms packaged as an SQLite virtual table module. +*/ +/* clang-format off */ + +/* +** Database Format of R-Tree Tables +** -------------------------------- +** +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. +** +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER, ...) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. If the r-tree contains auxiliary columns, those are stored +** on the end of the %_rowid table. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. +*/ + +#if !defined(SQLITE_CORE) \ + || (defined(SQLITE_ENABLE_RTREE) && !defined(SQLITE_OMIT_VIRTUALTABLE)) + +#ifndef SQLITE_CORE +#include "third_party/sqlite3/sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#else +#include "third_party/sqlite3/sqlite3.h" +#endif +int sqlite3GetToken(const unsigned char*,int*); /* In the SQLite core */ + +#ifndef SQLITE_AMALGAMATION +#include "third_party/sqlite3/sqlite3rtree.inc" +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif +#endif + +#include "libc/assert.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" + +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif + +typedef struct Rtree Rtree; +typedef struct RtreeCursor RtreeCursor; +typedef struct RtreeNode RtreeNode; +typedef struct RtreeCell RtreeCell; +typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; +typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; + +/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ +#define RTREE_MAX_DIMENSIONS 5 + +/* Maximum number of auxiliary columns */ +#define RTREE_MAX_AUX_COLUMN 100 + +/* Size of hash table Rtree.aHash. This hash table is not expected to +** ever contain very many entries, so a fixed number of buckets is +** used. +*/ +#define HASHSIZE 97 + +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 + +/* +** An rtree virtual-table object. +*/ +struct Rtree { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* Host database connection */ + int iNodeSize; /* Size in bytes of each node in the node table */ + u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ + u8 nAux; /* # of auxiliary columns in %_rowid */ + u8 nAuxNotNull; /* Number of initial not-null aux columns */ +#ifdef SQLITE_DEBUG + u8 bCorrupt; /* Shadow table corruption detected */ +#endif + int iDepth; /* Current depth of the r-tree structure */ + char *zDb; /* Name of database containing r-tree table */ + char *zName; /* Name of r-tree table */ + u32 nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ + u32 nNodeRef; /* Number RtreeNodes with positive nRef */ + char *zReadAuxSql; /* SQL for statement to read aux data */ + + /* List of nodes removed during a CondenseTree operation. List is + ** linked together via the pointer normally used for hash chains - + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** headed by the node (leaf nodes have RtreeNode.iNode==0). + */ + RtreeNode *pDeleted; + int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; + + /* Statements to read/write/delete a record from xxx_node */ + sqlite3_stmt *pWriteNode; + sqlite3_stmt *pDeleteNode; + + /* Statements to read/write/delete a record from xxx_rowid */ + sqlite3_stmt *pReadRowid; + sqlite3_stmt *pWriteRowid; + sqlite3_stmt *pDeleteRowid; + + /* Statements to read/write/delete a record from xxx_parent */ + sqlite3_stmt *pReadParent; + sqlite3_stmt *pWriteParent; + sqlite3_stmt *pDeleteParent; + + /* Statement for writing to the "aux:" fields, if there are any */ + sqlite3_stmt *pWriteAux; + + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ +}; + +/* Possible values for Rtree.eCoordType: */ +#define RTREE_COORD_REAL32 0 +#define RTREE_COORD_INT32 1 + +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif + +/* +** Set the Rtree.bCorrupt flag +*/ +#ifdef SQLITE_DEBUG +# define RTREE_IS_CORRUPT(X) ((X)->bCorrupt = 1) +#else +# define RTREE_IS_CORRUPT(X) +#endif + +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + +/* +** The minimum number of cells allowed for a node is a third of the +** maximum. In Gutman's notation: +** +** m = M/3 +** +** If an R*-tree "Reinsert" operation is required, the same number of +** cells are removed from the overfull node and reinserted into the tree. +*/ +#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) +#define RTREE_REINSERT(p) RTREE_MINCELLS(p) +#define RTREE_MAXCELLS 51 + +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 3^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + +/* +** An rtree cursor object. +*/ +struct RtreeCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ + u8 bAuxValid; /* True if pReadAux is valid */ + int iStrategy; /* Copy of idxNum search parameter */ + int nConstraint; /* Number of entries in aConstraint */ + RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + sqlite3_stmt *pReadAux; /* Statement to read aux-data */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ +}; + +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ +union RtreeCoord { + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ +}; + +/* +** The argument is an RtreeCoord. Return the value stored within the RtreeCoord +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. +*/ +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif + +/* +** A search constraint. +*/ +struct RtreeConstraint { + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ +}; + +/* Possible values for RtreeConstraint.op */ +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + +/* Special operators available only on cursors. Needs to be consecutive +** with the normal values above, but must be less than RTREE_MATCH. These +** are used in the cursor for contraints such as x=NULL (RTREE_FALSE) or +** x<'xyz' (RTREE_TRUE) */ +#define RTREE_TRUE 0x3f /* ? */ +#define RTREE_FALSE 0x40 /* @ */ + +/* +** An rtree structure node. +*/ +struct RtreeNode { + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ +}; + +/* Return the number of cells in a node */ +#define NCELL(pNode) readInt16(&(pNode)->zData[2]) + +/* +** A single cell from a node, deserialized +*/ +struct RtreeCell { + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; +}; + +/* +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. +*/ +struct RtreeMatchArg { + u32 iSize; /* Size of this object */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + sqlite3_value **apSqlParam; /* Original SQL parameter values */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ +}; + +#ifndef MAX +# define MAX(x,y) ((x) < (y) ? (y) : (x)) +#endif +#ifndef MIN +# define MIN(x,y) ((x) > (y) ? (y) : (x)) +#endif + +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGAMATION +# define testcase(X) +#endif + +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1400 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) +# else +# include +# endif +# endif +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +#elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +#else +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +#endif +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* +** Functions to deserialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The deserialized value is returned. +*/ +static int readInt16(u8 *p){ + return (p[0]<<8) + p[1]; +} +static void readCoord(u8 *p, RtreeCoord *pCoord){ + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; +#else + pCoord->u = ( + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + + (((u32)p[3]) << 0) + ); +#endif +} +static i64 readInt64(u8 *p){ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) + ); +#endif +} + +/* +** Functions to serialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The value returned is the number of bytes written +** to the argument buffer (always 2, 4 and 8 respectively). +*/ +static void writeInt16(u8 *p, int i){ + p[0] = (i>> 8)&0xFF; + p[1] = (i>> 0)&0xFF; +} +static int writeCoord(u8 *p, RtreeCoord *pCoord){ + u32 i; + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( sizeof(RtreeCoord)==4 ); + assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else + i = pCoord->u; + p[0] = (i>>24)&0xFF; + p[1] = (i>>16)&0xFF; + p[2] = (i>> 8)&0xFF; + p[3] = (i>> 0)&0xFF; +#endif + return 4; +} +static int writeInt64(u8 *p, i64 i){ +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else + p[0] = (i>>56)&0xFF; + p[1] = (i>>48)&0xFF; + p[2] = (i>>40)&0xFF; + p[3] = (i>>32)&0xFF; + p[4] = (i>>24)&0xFF; + p[5] = (i>>16)&0xFF; + p[6] = (i>> 8)&0xFF; + p[7] = (i>> 0)&0xFF; +#endif + return 8; +} + +/* +** Increment the reference count of node p. +*/ +static void nodeReference(RtreeNode *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } +} + +/* +** Clear the content of node p (set all bytes to 0x00). +*/ +static void nodeZero(Rtree *pRtree, RtreeNode *p){ + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; +} + +/* +** Given a node number iNode, return the corresponding key to use +** in the Rtree.aHash table. +*/ +static unsigned int nodeHash(i64 iNode){ + return ((unsigned)iNode) % HASHSIZE; +} + +/* +** Search the node hash table for node iNode. If found, return a pointer +** to it. Otherwise, return 0. +*/ +static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ + RtreeNode *p; + for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); + return p; +} + +/* +** Add node pNode to the node hash table. +*/ +static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; +} + +/* +** Remove node pNode from the node hash table. +*/ +static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode **pp; + if( pNode->iNode!=0 ){ + pp = &pRtree->aHash[nodeHash(pNode->iNode)]; + for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } + *pp = pNode->pNext; + pNode->pNext = 0; + } +} + +/* +** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), +** indicating that node has not yet been assigned a node number. It is +** assigned a node number when nodeWrite() is called to write the +** node contents out to the database. +*/ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ + RtreeNode *pNode; + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode) + pRtree->iNodeSize); + if( pNode ){ + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pRtree->nNodeRef++; + pNode->pParent = pParent; + pNode->isDirty = 1; + nodeReference(pParent); + } + return pNode; +} + +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); + } +} + +/* +** Check to see if pNode is the same as pParent or any of the parents +** of pParent. +*/ +static int nodeInParentChain(const RtreeNode *pNode, const RtreeNode *pParent){ + do{ + if( pNode==pParent ) return 1; + pParent = pParent->pParent; + }while( pParent ); + return 0; +} + +/* +** Obtain a reference to an r-tree node. +*/ +static int nodeAcquire( + Rtree *pRtree, /* R-tree structure */ + i64 iNode, /* Node number to load */ + RtreeNode *pParent, /* Either the parent node or NULL */ + RtreeNode **ppNode /* OUT: Acquired node */ +){ + int rc = SQLITE_OK; + RtreeNode *pNode = 0; + + /* Check if the requested node is already in the hash table. If so, + ** increase its reference count and return it. + */ + if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ + if( pParent && !pNode->pParent ){ + if( nodeInParentChain(pNode, pParent) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + pParent->nRef++; + pNode->pParent = pParent; + }else if( pParent && pNode->pParent && pParent!=pNode->pParent ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + pNode->nRef++; + *ppNode = pNode; + return SQLITE_OK; + } + + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + } + } + if( pRtree->pNodeBlob==0 ){ + char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); + if( zTab==0 ) return SQLITE_NOMEM; + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + &pRtree->pNodeBlob); + sqlite3_free(zTab); + } + if( rc ){ + nodeBlobReset(pRtree); + *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pRtree->nNodeRef++; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); + } + } + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && rc==SQLITE_OK && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT_VTAB. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeReference(pParent); + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + *ppNode = pNode; + }else{ + if( pNode ){ + pRtree->nNodeRef--; + sqlite3_free(pNode); + } + *ppNode = 0; + } + + return rc; +} + +/* +** Overwrite cell iCell of node pNode with the contents of pCell. +*/ +static void nodeOverwriteCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ +){ + int ii; + u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + p += writeInt64(p, pCell->iRowid); + for(ii=0; iinDim2; ii++){ + p += writeCoord(p, &pCell->aCoord[ii]); + } + pNode->isDirty = 1; +} + +/* +** Remove the cell with index iCell from node pNode. +*/ +static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ + u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + u8 *pSrc = &pDst[pRtree->nBytesPerCell]; + int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; + memmove(pDst, pSrc, nByte); + writeInt16(&pNode->zData[2], NCELL(pNode)-1); + pNode->isDirty = 1; +} + +/* +** Insert the contents of cell pCell into node pNode. If the insert +** is successful, return SQLITE_OK. +** +** If there is not enough free space in pNode, return SQLITE_FULL. +*/ +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ +){ + int nCell; /* Current number of cells in pNode */ + int nMaxCell; /* Maximum number of cells for pNode */ + + nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; + nCell = NCELL(pNode); + + assert( nCell<=nMaxCell ); + if( nCellzData[2], nCell+1); + pNode->isDirty = 1; + } + + return (nCell==nMaxCell); +} + +/* +** If the node is dirty, write it out to the database. +*/ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode->isDirty ){ + sqlite3_stmt *p = pRtree->pWriteNode; + if( pNode->iNode ){ + sqlite3_bind_int64(p, 1, pNode->iNode); + }else{ + sqlite3_bind_null(p, 1); + } + sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); + sqlite3_step(p); + pNode->isDirty = 0; + rc = sqlite3_reset(p); + sqlite3_bind_null(p, 2); + if( pNode->iNode==0 && rc==SQLITE_OK ){ + pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); + nodeHashInsert(pRtree, pNode); + } + } + return rc; +} + +/* +** Release a reference to a node. If the node is dirty and the reference +** count drops to zero, the node data is written to the database. +*/ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode ){ + assert( pNode->nRef>0 ); + assert( pRtree->nNodeRef>0 ); + pNode->nRef--; + if( pNode->nRef==0 ){ + pRtree->nNodeRef--; + if( pNode->iNode==1 ){ + pRtree->iDepth = -1; + } + if( pNode->pParent ){ + rc = nodeRelease(pRtree, pNode->pParent); + } + if( rc==SQLITE_OK ){ + rc = nodeWrite(pRtree, pNode); + } + nodeHashDelete(pRtree, pNode); + sqlite3_free(pNode); + } + } + return rc; +} + +/* +** Return the 64-bit integer value associated with cell iCell of +** node pNode. If pNode is a leaf node, this is a rowid. If it is +** an internal node, then the 64-bit integer is a child page number. +*/ +static i64 nodeGetRowid( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ +){ + assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); +} + +/* +** Return coordinate iCoord from cell iCell in node pNode. +*/ +static void nodeGetCoord( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ +){ + readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); +} + +/* +** Deserialize cell iCell of node pNode. Populate the structure pointed +** to by pCell with the results. +*/ +static void nodeGetCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + RtreeCoord *pCoord; + int ii = 0; + pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pCoord = pCell->aCoord; + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim2 ); +} + + +/* Forward declaration for the function that does the work of +** the virtual table module xCreate() and xConnect() methods. +*/ +static int rtreeInit( + sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int +); + +/* +** Rtree virtual table module xCreate method. +*/ +static int rtreeCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** Rtree virtual table module xConnect method. +*/ +static int rtreeConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + +/* +** Increment the r-tree reference count. +*/ +static void rtreeReference(Rtree *pRtree){ + pRtree->nBusy++; +} + +/* +** Decrement the r-tree reference count. When the reference count reaches +** zero the structure is deleted. +*/ +static void rtreeRelease(Rtree *pRtree){ + pRtree->nBusy--; + if( pRtree->nBusy==0 ){ + pRtree->inWrTrans = 0; + assert( pRtree->nCursor==0 ); + nodeBlobReset(pRtree); + assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + sqlite3_finalize(pRtree->pWriteNode); + sqlite3_finalize(pRtree->pDeleteNode); + sqlite3_finalize(pRtree->pReadRowid); + sqlite3_finalize(pRtree->pWriteRowid); + sqlite3_finalize(pRtree->pDeleteRowid); + sqlite3_finalize(pRtree->pReadParent); + sqlite3_finalize(pRtree->pWriteParent); + sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_finalize(pRtree->pWriteAux); + sqlite3_free(pRtree->zReadAuxSql); + sqlite3_free(pRtree); + } +} + +/* +** Rtree virtual table module xDisconnect method. +*/ +static int rtreeDisconnect(sqlite3_vtab *pVtab){ + rtreeRelease((Rtree *)pVtab); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xDestroy method. +*/ +static int rtreeDestroy(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + int rc; + char *zCreate = sqlite3_mprintf( + "DROP TABLE '%q'.'%q_node';" + "DROP TABLE '%q'.'%q_rowid';" + "DROP TABLE '%q'.'%q_parent';", + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName + ); + if( !zCreate ){ + rc = SQLITE_NOMEM; + }else{ + nodeBlobReset(pRtree); + rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + } + if( rc==SQLITE_OK ){ + rtreeRelease(pRtree); + } + + return rc; +} + +/* +** Rtree virtual table module xOpen method. +*/ +static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; + RtreeCursor *pCsr; + + pCsr = (RtreeCursor *)sqlite3_malloc64(sizeof(RtreeCursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = pVTab; + rc = SQLITE_OK; + pRtree->nCursor++; + } + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + + return rc; +} + + +/* +** Reset a cursor back to its initial state. +*/ +static void resetCursor(RtreeCursor *pCsr){ + Rtree *pRtree = (Rtree *)(pCsr->base.pVtab); + int ii; + sqlite3_stmt *pStmt; + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr->aPoint); + pStmt = pCsr->pReadAux; + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->pReadAux = pStmt; + +} + +/* +** Rtree virtual table module xClose method. +*/ +static int rtreeClose(sqlite3_vtab_cursor *cur){ + Rtree *pRtree = (Rtree *)(cur->pVtab); + RtreeCursor *pCsr = (RtreeCursor *)cur; + assert( pRtree->nCursor>0 ); + resetCursor(pCsr); + sqlite3_finalize(pCsr->pReadAux); + sqlite3_free(pCsr); + pRtree->nCursor--; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int rtreeEof(sqlite3_vtab_cursor *cur){ + RtreeCursor *pCsr = (RtreeCursor *)cur; + return pCsr->atEOF; +} + +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. +*/ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif + +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. +*/ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ +){ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ + + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); + + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; +#ifndef SQLITE_RTREE_INT_ONLY + if( eInt==0 ){ + switch( nCoord ){ + case 10: readCoord(pCellData+36, &c); aCoord[9] = c.f; + readCoord(pCellData+32, &c); aCoord[8] = c.f; + case 8: readCoord(pCellData+28, &c); aCoord[7] = c.f; + readCoord(pCellData+24, &c); aCoord[6] = c.f; + case 6: readCoord(pCellData+20, &c); aCoord[5] = c.f; + readCoord(pCellData+16, &c); aCoord[4] = c.f; + case 4: readCoord(pCellData+12, &c); aCoord[3] = c.f; + readCoord(pCellData+8, &c); aCoord[2] = c.f; + default: readCoord(pCellData+4, &c); aCoord[1] = c.f; + readCoord(pCellData, &c); aCoord[0] = c.f; + } + }else +#endif + { + switch( nCoord ){ + case 10: readCoord(pCellData+36, &c); aCoord[9] = c.i; + readCoord(pCellData+32, &c); aCoord[8] = c.i; + case 8: readCoord(pCellData+28, &c); aCoord[7] = c.i; + readCoord(pCellData+24, &c); aCoord[6] = c.i; + case 6: readCoord(pCellData+20, &c); aCoord[5] = c.i; + readCoord(pCellData+16, &c); aCoord[4] = c.i; + case 4: readCoord(pCellData+12, &c); aCoord[3] = c.i; + readCoord(pCellData+8, &c); aCoord[2] = c.i; + default: readCoord(pCellData+4, &c); aCoord[1] = c.i; + readCoord(pCellData, &c); aCoord[0] = c.i; + } + } + if( pConstraint->op==RTREE_MATCH ){ + int eWithin = 0; + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; + } + } + return rc; +} + +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + switch( p->op ){ + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); + pCellData += 8 + p->iCoord*4; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + +/* +** One of the cells in node pNode is guaranteed to have a 64-bit +** integer value equal to iRowid. Return the index of this cell. +*/ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ + int ii; + int nCell = NCELL(pNode); + assert( nCell<200 ); + for(ii=0; iipParent; + if( pParent ){ + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + } + *piIndex = -1; + return SQLITE_OK; +} + +/* +** Compare two search points. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second. +** +** The rScore is the primary key. Smaller rScore values come first. +** If the rScore is a tie, then use iLevel as the tie breaker with smaller +** iLevel values coming first. In this way, if rScore is the same for all +** SearchPoints, then iLevel becomes the deciding factor and the result +** is a depth-first search, which is the desired default behavior. +*/ +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB +){ + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange two search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc64(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + u8 *pCellData; + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); + while( p->iCellaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ){ + p->iCell++; + pCellData += pRtree->nBytesPerCell; + break; + } + } + if( eWithin==NOT_WITHIN ) continue; + p->iCell++; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + for(ii=0; iinPoint; ii++){ + if( pCur->aPoint[ii].id==x.id ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + } + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = (u8)eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xNext method. +*/ +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + if( pCsr->bAuxValid ){ + pCsr->bAuxValid = 0; + sqlite3_reset(pCsr->pReadAux); + } + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); + return rc; +} + +/* +** Rtree virtual table module xRowid method. +*/ +static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; +} + +/* +** Rtree virtual table module xColumn method. +*/ +static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 ){ + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); + }else if( i<=pRtree->nDim2 ){ + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + sqlite3_result_double(ctx, c.f); + }else +#endif + { + assert( pRtree->eCoordType==RTREE_COORD_INT32 ); + sqlite3_result_int(ctx, c.i); + } + }else{ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, + sqlite3_column_value(pCsr->pReadAux, i - pRtree->nDim2 + 1)); + } + return SQLITE_OK; +} + +/* +** Use nodeAcquire() to obtain the leaf node containing the record with +** rowid iRowid. If successful, set *ppLeaf to point to the node and +** return SQLITE_OK. If there is no such record in the table, set +** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf +** to zero and return an SQLite error code. +*/ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ + int rc; + *ppLeaf = 0; + sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); + if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ + i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; + rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); + sqlite3_reset(pRtree->pReadRowid); + }else{ + rc = sqlite3_reset(pRtree->pReadRowid); + } + return rc; +} + +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *pBlob, *pSrc; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + + pSrc = sqlite3_value_pointer(pValue, "RtreeMatchArg"); + if( pSrc==0 ) return SQLITE_ERROR; + pInfo = (sqlite3_rtree_query_info*) + sqlite3_malloc64( sizeof(*pInfo)+pSrc->iSize ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; + memcpy(pBlob, pSrc, pSrc->iSize); + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; + pInfo->apSqlParam = pBlob->apSqlParam; + + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xFilter method. +*/ +static int rtreeFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int ii; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + int eType = sqlite3_value_numeric_type(argv[0]); + if( eType==SQLITE_INTEGER + || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + ){ + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + }else{ + rc = SQLITE_OK; + pLeaf = 0; + } + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ + pCsr->aConstraint = sqlite3_malloc64(sizeof(RtreeConstraint)*argc); + pCsr->nConstraint = argc; + if( !pCsr->aConstraint ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); + for(ii=0; iiaConstraint[ii]; + int eType = sqlite3_value_numeric_type(argv[ii]); + p->op = idxStr[ii*2]; + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + p->pInfo->nCoord = pRtree->nDim2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; + }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif + }else{ + p->u.rValue = RTREE_ZERO; + if( eType==SQLITE_NULL ){ + p->op = RTREE_FALSE; + }else if( p->op==RTREE_LT || p->op==RTREE_LE ){ + p->op = RTREE_TRUE; + }else{ + p->op = RTREE_FALSE; + } + } + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 Unused Direct lookup by rowid. +** 2 See below R-tree query or full-table scan. +** ------------------------------------------------ +** +** If strategy 1 is used, then idxStr is not meaningful. If strategy +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to +** the constraint in sqlite3_index_info.aConstraintUsage[] with +** (argvIndex==1) etc. +** +** The first of each pair of bytes in idxStr identifies the constraint +** operator as follows: +** +** Operator Byte Value +** ---------------------- +** = 0x41 ('A') +** <= 0x42 ('B') +** < 0x43 ('C') +** >= 0x44 ('D') +** > 0x45 ('E') +** MATCH 0x46 ('F') +** ---------------------- +** +** The second of each pair of bytes identifies the coordinate column +** to which the constraint applies. The leftmost coordinate column +** is 'a', the second from the left 'b' etc. +*/ +static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; + int rc = SQLITE_OK; + int ii; + int bMatch = 0; /* True if there exists a MATCH constraint */ + i64 nRow; /* Estimated rows returned by this scan */ + + int iIdx = 0; + char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; + memset(zIdxStr, 0, sizeof(zIdxStr)); + + /* Check if there exists a MATCH constraint - even an unusable one. If there + ** is, do not consider the lookup-by-rowid plan as using such a plan would + ** require the VDBE to evaluate the MATCH constraint, which is not currently + ** possible. */ + for(ii=0; iinConstraint; ii++){ + if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + bMatch = 1; + } + } + + assert( pIdxInfo->idxStr==0 ); + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + + if( bMatch==0 && p->usable + && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + /* We have an equality constraint on the rowid. Use strategy 1. */ + int jj; + for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; + pIdxInfo->aConstraintUsage[jj].omit = 0; + } + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[ii].argvIndex = 1; + pIdxInfo->aConstraintUsage[jj].omit = 1; + + /* This strategy involves a two rowid lookups on an B-Tree structures + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. + */ + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + + if( p->usable + && ((p->iColumn>0 && p->iColumn<=pRtree->nDim2) + || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) + ){ + u8 op; + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; + default: op = 0; break; + } + if( op ){ + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; + } + } + } + + pIdxInfo->idxNum = 2; + pIdxInfo->needToFreeIdxStr = 1; + if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ + return SQLITE_NOMEM; + } + + nRow = pRtree->nRowEst >> (iIdx/2); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + pIdxInfo->estimatedRows = nRow; + + return rc; +} + +/* +** Return the N-dimensional volumn of the cell stored in *p. +*/ +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = (i64)p->aCoord[9].i - (i64)p->aCoord[8].i; + case 4: area *= (i64)p->aCoord[7].i - (i64)p->aCoord[6].i; + case 3: area *= (i64)p->aCoord[5].i - (i64)p->aCoord[4].i; + case 2: area *= (i64)p->aCoord[3].i - (i64)p->aCoord[2].i; + default: area *= (i64)p->aCoord[1].i - (i64)p->aCoord[0].i; + } + } + return area; +} + +/* +** Return the margin length of cell p. The margin length is the sum +** of the objects size in each dimension. +*/ +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + ii -= 2; + }while( ii>=0 ); + return margin; +} + +/* +** Store the union of cells p1 and p2 in p1. +*/ +static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii = 0; + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + do{ + p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); + p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); + ii += 2; + }while( iinDim2 ); + }else{ + do{ + p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); + p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); + ii += 2; + }while( iinDim2 ); + } +} + +/* +** Return true if the area covered by p2 is a subset of the area covered +** by p1. False otherwise. +*/ +static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( (!isInt && (a2[0].fa1[1].f)) + || ( isInt && (a2[0].ia1[1].i)) + ){ + return 0; + } + } + return 1; +} + +/* +** Return the amount cell p would grow by if it were unioned with pCell. +*/ +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; + RtreeCell cell; + memcpy(&cell, p, sizeof(RtreeCell)); + area = cellArea(pRtree, &cell); + cellUnion(pRtree, &cell, pCell); + return (cellArea(pRtree, &cell)-area); +} + +static RtreeDValue cellOverlap( + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, + int nCell +){ + int ii; + RtreeDValue overlap = RTREE_ZERO; + for(ii=0; iinDim2; jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-iHeight); ii++){ + int iCell; + sqlite3_int64 iBest = 0; + + RtreeDValue fMinGrowth = RTREE_ZERO; + RtreeDValue fMinArea = RTREE_ZERO; + + int nCell = NCELL(pNode); + RtreeCell cell; + RtreeNode *pChild; + + RtreeCell *aCell = 0; + + /* Select the child node which will be enlarged the least if pCell + ** is inserted into it. Resolve ties by choosing the entry with + ** the smallest area. + */ + for(iCell=0; iCellpParent ){ + RtreeNode *pParent = p->pParent; + RtreeCell cell; + int iCell; + + if( (++cnt)>1000 || nodeParentIndex(pRtree, p, &iCell) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + + nodeGetCell(pRtree, pParent, iCell, &cell); + if( !cellContains(pRtree, &cell, pCell) ){ + cellUnion(pRtree, &cell, pCell); + nodeOverwriteCell(pRtree, pParent, &cell, iCell); + } + + p = pParent; + } + return SQLITE_OK; +} + +/* +** Write mapping (iRowid->iNode) to the _rowid table. +*/ +static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ + sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); + sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); + sqlite3_step(pRtree->pWriteRowid); + return sqlite3_reset(pRtree->pWriteRowid); +} + +/* +** Write mapping (iNode->iPar) to the _parent table. +*/ +static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ + sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); + sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); + sqlite3_step(pRtree->pWriteParent); + return sqlite3_reset(pRtree->pWriteParent); +} + +static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); + + +/* +** Arguments aIdx, aDistance and aSpare all point to arrays of size +** nIdx. The aIdx array contains the set of integers from 0 to +** (nIdx-1) in no particular order. This function sorts the values +** in aIdx according to the indexed values in aDistance. For +** example, assuming the inputs: +** +** aIdx = { 0, 1, 2, 3 } +** aDistance = { 5.0, 2.0, 7.0, 6.0 } +** +** this function sets the aIdx array to contain: +** +** aIdx = { 0, 1, 2, 3 } +** +** The aSpare array is used as temporary working space by the +** sorting algorithm. +*/ +static void SortByDistance( + int *aIdx, + int nIdx, + RtreeDValue *aDistance, + int *aSpare +){ + if( nIdx>1 ){ + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDistance(aLeft, nLeft, aDistance, aSpare); + SortByDistance(aRight, nRight, aDistance, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + + while( iLeft1 ){ + + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); + SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + while( iLeftnDim+1)*(sizeof(int*)+nCell*sizeof(int)); + + aaSorted = (int **)sqlite3_malloc64(nByte); + if( !aaSorted ){ + return SQLITE_NOMEM; + } + + aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; + memset(aaSorted, 0, nByte); + for(ii=0; iinDim; ii++){ + int jj; + aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; + for(jj=0; jjnDim; ii++){ + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; + int iBestLeft = 0; + int nLeft; + + for( + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft++ + ){ + RtreeCell left; + RtreeCell right; + int kk; + RtreeDValue overlap; + RtreeDValue area; + + memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); + memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); + for(kk=1; kk<(nCell-1); kk++){ + if( kk0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + return xSetMapping(pRtree, iRowid, pNode->iNode); +} + +static int SplitNode( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int i; + int newCellIsRight = 0; + + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + RtreeCell *aCell; + int *aiUsed; + + RtreeNode *pLeft = 0; + RtreeNode *pRight = 0; + + RtreeCell leftbbox; + RtreeCell rightbbox; + + /* Allocate an array and populate it with a copy of pCell and + ** all cells from node pLeft. Then zero the original node. + */ + aCell = sqlite3_malloc64((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + if( !aCell ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + aiUsed = (int *)&aCell[nCell+1]; + memset(aiUsed, 0, sizeof(int)*(nCell+1)); + for(i=0; iiNode==1 ){ + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); + pRtree->iDepth++; + pNode->isDirty = 1; + writeInt16(pNode->zData, pRtree->iDepth); + }else{ + pLeft = pNode; + pRight = nodeNew(pRtree, pLeft->pParent); + pLeft->nRef++; + } + + if( !pLeft || !pRight ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + + memset(pLeft->zData, 0, pRtree->iNodeSize); + memset(pRight->zData, 0, pRtree->iNodeSize); + + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) + || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) + ){ + goto splitnode_out; + } + + rightbbox.iRowid = pRight->iNode; + leftbbox.iRowid = pLeft->iNode; + + if( pNode->iNode==1 ){ + rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + }else{ + RtreeNode *pParent = pLeft->pParent; + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ + goto splitnode_out; + } + + for(i=0; iiRowid ){ + newCellIsRight = 1; + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( pNode->iNode==1 ){ + for(i=0; iiRowid, pLeft, iHeight); + } + + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRight); + pRight = 0; + } + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pLeft); + pLeft = 0; + } + +splitnode_out: + nodeRelease(pRtree, pRight); + nodeRelease(pRtree, pLeft); + sqlite3_free(aCell); + return rc; +} + +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ +static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ + int rc = SQLITE_OK; + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ){ + RTREE_IS_CORRUPT(pRtree); + rc = SQLITE_CORRUPT_VTAB; + } + pChild = pChild->pParent; + } + return rc; +} + +static int deleteCell(Rtree *, RtreeNode *, int, int); + +static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ + int rc; + int rc2; + RtreeNode *pParent = 0; + int iCell; + + assert( pNode->nRef==1 ); + + /* Remove the entry in the parent cell. */ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Remove the xxx_node entry. */ + sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteNode); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ + return rc; + } + + /* Remove the xxx_parent entry. */ + sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteParent); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ + return rc; + } + + /* Remove the node from the in-memory hash table and link it into + ** the Rtree.pDeleted list. Its contents will be re-inserted later on. + */ + nodeHashDelete(pRtree, pNode); + pNode->iNode = iHeight; + pNode->pNext = pRtree->pDeleted; + pNode->nRef++; + pRtree->pDeleted = pNode; + + return SQLITE_OK; +} + +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; + if( pParent ){ + int ii; + int nCell = NCELL(pNode); + RtreeCell box; /* Bounding box for pNode */ + nodeGetCell(pRtree, pNode, 0, &box); + for(ii=1; iiiNode; + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } + } + return rc; +} + +/* +** Delete the cell at index iCell of node pNode. After removing the +** cell, adjust the r-tree data structure if required. +*/ +static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; + int rc; + + if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ + return rc; + } + + /* Remove the cell from the node. This call just moves bytes around + ** the in-memory node image, so it cannot fail. + */ + nodeDeleteCell(pRtree, pNode, iCell); + + /* If the node is not the tree root and now has less than the minimum + ** number of cells, remove it from the tree. Otherwise, update the + ** cell in the parent node so that it tightly contains the updated + ** node. + */ + pParent = pNode->pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)nDim; iDim++){ + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + } + } + for(iDim=0; iDimnDim; iDim++){ + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); + } + + for(ii=0; iinDim; iDim++){ + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); + aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); + } + } + + SortByDistance(aOrder, nCell, aDistance, aSpare); + nodeZero(pRtree, pNode); + + for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ + RtreeCell *p = &aCell[aOrder[ii]]; + nodeInsertCell(pRtree, pNode, p); + if( p->iRowid==pCell->iRowid ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, p->iRowid, pNode->iNode); + } + } + } + if( rc==SQLITE_OK ){ + rc = fixBoundingBox(pRtree, pNode); + } + for(; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + RtreeNode *pInsert; + RtreeCell *p = &aCell[aOrder[ii]]; + rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + sqlite3_free(aCell); + return rc; +} + +/* +** Insert cell pCell into node pNode. Node pNode is the head of a +** subtree iHeight high (leaf nodes have iHeight==0). +*/ +static int rtreeInsertCell( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int rc = SQLITE_OK; + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + if( nodeInsertCell(pRtree, pNode, pCell) ){ + if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ + rc = SplitNode(pRtree, pNode, pCell, iHeight); + }else{ + pRtree->iReinsertHeight = iHeight; + rc = Reinsert(pRtree, pNode, pCell, iHeight); + } + }else{ + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } + } + } + return rc; +} + +static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ + int ii; + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + + for(ii=0; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + return rc; +} + +/* +** Select a currently unused rowid for a new r-tree record. +*/ +static int rtreeNewRowid(Rtree *pRtree, i64 *piRowid){ + int rc; + sqlite3_bind_null(pRtree->pWriteRowid, 1); + sqlite3_bind_null(pRtree->pWriteRowid, 2); + sqlite3_step(pRtree->pWriteRowid); + rc = sqlite3_reset(pRtree->pWriteRowid); + *piRowid = sqlite3_last_insert_rowid(pRtree->db); + return rc; +} + +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot = 0; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialize Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); + } + +#ifdef CORRUPT_DB + assert( pLeaf!=0 || rc!=SQLITE_OK || CORRUPT_DB ); +#endif + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK && pLeaf ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild = 0; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + pRtree->nNodeRef--; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + +/* +** Rounding constants for float->double conversion. +*/ +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ + +#if !defined(SQLITE_RTREE_INT_ONLY) +/* +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. +*/ +static RtreeValue rtreeValueDown(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f>d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); + } + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( fbase.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + + +/* +** The xUpdate method for rtree module virtual tables. +*/ +static int rtreeUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + cell.iRowid = 0; /* Used only to suppress a compiler warning */ + + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; + int nn = nData - 4; + + if( nn > pRtree->nDim2 ) nn = pRtree->nDim2; + /* Populate the cell.aCoord[] array. The first coordinate is aData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; iicell.aCoord[ii+1].f ){ + rc = rtreeConstraintError(pRtree, ii+1); + goto constraint; + } + } + }else +#endif + { + for(ii=0; iicell.aCoord[ii+1].i ){ + rc = rtreeConstraintError(pRtree, ii+1); + goto constraint; + } + } + } + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(aData[2])!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(aData[2]); + if( sqlite3_value_type(aData[0])==SQLITE_NULL + || sqlite3_value_int64(aData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + goto constraint; + } + } + } + bHaveRowid = 1; + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(aData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(aData[0])); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + if( rc==SQLITE_OK && pRtree->nAux ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + sqlite3_bind_int64(pUp, 1, *pRowid); + for(jj=0; jjnAux; jj++){ + sqlite3_bind_value(pUp, jj+2, aData[pRtree->nDim2+3+jj]); + } + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +constraint: + rtreeRelease(pRtree); + return rc; +} + +/* +** Called when a transaction starts. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans++; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + +/* +** The xRename method for rtree module virtual tables. +*/ +static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" + "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" + "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + ); + if( zSql ){ + nodeBlobReset(pRtree); + rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + return rc; +} + +/* +** The xSavepoint method. +** +** This module does not need to do anything to support savepoints. However, +** it uses this hook to close any open blob handle. This is done because a +** DROP TABLE command - which fortunately always opens a savepoint - cannot +** succeed if there are any open blob handles. i.e. if the blob handle were +** not closed here, the following would fail: +** +** BEGIN; +** INSERT INTO rtree... +** DROP TABLE ; -- Would fail with SQLITE_LOCKED +** COMMIT; +*/ +static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){ + Rtree *pRtree = (Rtree *)pVtab; + u8 iwt = pRtree->inWrTrans; + UNUSED_PARAMETER(iSavepoint); + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + pRtree->inWrTrans = iwt; + return SQLITE_OK; +} + +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; + + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); + } + + return rc; +} + + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int rtreeShadowName(const char *zName){ + static const char *azName[] = { + "node", "parent", "rowid" + }; + unsigned int i; + for(i=0; idb = db; + + if( isCreate ){ + char *zCreate; + sqlite3_str *p = sqlite3_str_new(db); + int ii; + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY,nodeno", + zDb, zPrefix); + for(ii=0; iinAux; ii++){ + sqlite3_str_appendf(p,",a%d",ii); + } + sqlite3_str_appendf(p, + ");CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY,data);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY,parentnode);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "INSERT INTO \"%w\".\"%w_node\"VALUES(1,zeroblob(%d))", + zDb, zPrefix, pRtree->iNodeSize); + zCreate = sqlite3_str_finish(p); + if( !zCreate ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc!=SQLITE_OK ){ + return rc; + } + } + + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; + + rc = rtreeQueryStat1(db, pRtree); + for(i=0; inAux==0 ){ + zFormat = azSql[i]; + }else { + /* An UPSERT is very slightly slower than REPLACE, but it is needed + ** if there are auxiliary columns */ + zFormat = "INSERT INTO\"%w\".\"%w_rowid\"(rowid,nodeno)VALUES(?1,?2)" + "ON CONFLICT(rowid)DO UPDATE SET nodeno=excluded.nodeno"; + } + zSql = sqlite3_mprintf(zFormat, zDb, zPrefix); + if( zSql ){ + rc = sqlite3_prepare_v3(db, zSql, -1, f, appStmt[i], 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + } + if( pRtree->nAux ){ + pRtree->zReadAuxSql = sqlite3_mprintf( + "SELECT * FROM \"%w\".\"%w_rowid\" WHERE rowid=?1", + zDb, zPrefix); + if( pRtree->zReadAuxSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_str *p = sqlite3_str_new(db); + int ii; + char *zSql; + sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix); + for(ii=0; iinAux; ii++){ + if( ii ) sqlite3_str_append(p, ",", 1); + if( iinAuxNotNull ){ + sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii); + }else{ + sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + } + } + sqlite3_str_appendf(p, " WHERE rowid=?1"); + zSql = sqlite3_str_finish(p); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(db, zSql, -1, f, &pRtree->pWriteAux, 0); + sqlite3_free(zSql); + } + } + } + + return rc; +} + +/* +** The second argument to this function contains the text of an SQL statement +** that returns a single integer value. The statement is compiled and executed +** using database connection db. If successful, the integer value returned +** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error +** code is returned and the value of *piVal after returning is not defined. +*/ +static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ + int rc = SQLITE_NOMEM; + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *piVal = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + } + } + return rc; +} + +/* +** This function is called from within the xConnect() or xCreate() method to +** determine the node-size used by the rtree table being created or connected +** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. +** +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. +** +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. +*/ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ +){ + int rc; + char *zSql; + if( isCreate ){ + int iPageSize = 0; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + }else{ + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else if( pRtree->iNodeSize<(512-64) ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"", + pRtree->zName); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** Return the length of a token +*/ +static int rtreeTokenLength(const char *z){ + int dummy = 0; + return sqlite3GetToken((const unsigned char*)z,&dummy); +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int rtreeInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); + sqlite3_str *pSql; + char *zSql; + int ii = 4; + int iErr; + + const char *aErrMsg[] = { + 0, /* 0 */ + "Wrong number of columns for an rtree table", /* 1 */ + "Too few columns for an rtree table", /* 2 */ + "Too many columns for an rtree table", /* 3 */ + "Auxiliary rtree columns must be last" /* 4 */ + }; + + assert( RTREE_MAX_AUX_COLUMN<256 ); /* Aux columns counted by a u8 */ + if( argc<6 || argc>RTREE_MAX_AUX_COLUMN+3 ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[2 + (argc>=6)]); + return SQLITE_ERROR; + } + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = (u8)eCoordType; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(%.*s INT", + rtreeTokenLength(argv[3]), argv[3]); + for(ii=4; iinAux++; + sqlite3_str_appendf(pSql, ",%.*s", rtreeTokenLength(zArg+1), zArg+1); + }else if( pRtree->nAux>0 ){ + break; + }else{ + static const char *azFormat[] = {",%.*s REAL", ",%.*s INT"}; + pRtree->nDim2++; + sqlite3_str_appendf(pSql, azFormat[eCoordType], + rtreeTokenLength(zArg), zArg); + } + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( iinDim = pRtree->nDim2/2; + if( pRtree->nDim<1 ){ + iErr = 2; + }else if( pRtree->nDim2>RTREE_MAX_DIMENSIONS*2 ){ + iErr = 3; + }else if( pRtree->nDim2 % 2 ){ + iErr = 1; + }else{ + iErr = 0; + } + if( iErr ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + goto rtreeInit_fail; + } + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto rtreeInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto rtreeInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +rtreeInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** Implementation of a scalar function that decodes r-tree nodes to +** human readable strings. This can be used for debugging and analysis. +** +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: +** +** SELECT rtreenode(2, data) FROM rt_node; +** +** The human readable string takes the form of a Tcl list with one +** entry for each cell in the r-tree node. Each entry is itself a +** list, containing the 8-byte rowid/pageno followed by the +** *2 coordinates. +*/ +static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + RtreeNode node; + Rtree tree; + int ii; + int nData; + int errCode; + sqlite3_str *pOut; + + UNUSED_PARAMETER(nArg); + memset(&node, 0, sizeof(RtreeNode)); + memset(&tree, 0, sizeof(Rtree)); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); + if( tree.nDim<1 || tree.nDim>5 ) return; + tree.nDim2 = tree.nDim*2; + tree.nBytesPerCell = 8 + 8 * tree.nDim; + node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + nData = sqlite3_value_bytes(apArg[1]); + if( nData<4 ) return; + if( nData0 ) sqlite3_str_append(pOut, " ", 1); + sqlite3_str_appendf(pOut, "{%lld", cell.iRowid); + for(jj=0; jjrc==SQLITE_OK ) pCheck->rc = rc; +} + +/* +** The second and subsequent arguments to this function are a format string +** and printf style arguments. This function formats the string and attempts +** to compile it as an SQL statement. +** +** If successful, a pointer to the new SQL statement is returned. Otherwise, +** NULL is returned and an error code left in RtreeCheck.rc. +*/ +static sqlite3_stmt *rtreeCheckPrepare( + RtreeCheck *pCheck, /* RtreeCheck object */ + const char *zFmt, ... /* Format string and trailing args */ +){ + va_list ap; + char *z; + sqlite3_stmt *pRet = 0; + + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + + if( pCheck->rc==SQLITE_OK ){ + if( z==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + } + } + + sqlite3_free(z); + va_end(ap); + return pRet; +} + +/* +** The second and subsequent arguments to this function are a printf() +** style format string and arguments. This function formats the string and +** appends it to the report being accumuated in pCheck. +*/ +static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pCheck->rc==SQLITE_OK && pCheck->nErrrc = SQLITE_NOMEM; + }else{ + pCheck->zReport = sqlite3_mprintf("%z%s%z", + pCheck->zReport, (pCheck->zReport ? "\n" : ""), z + ); + if( pCheck->zReport==0 ){ + pCheck->rc = SQLITE_NOMEM; + } + } + pCheck->nErr++; + } + va_end(ap); +} + +/* +** This function is a no-op if there is already an error code stored +** in the RtreeCheck object indicated by the first argument. NULL is +** returned in this case. +** +** Otherwise, the contents of rtree table node iNode are loaded from +** the database and copied into a buffer obtained from sqlite3_malloc(). +** If no error occurs, a pointer to the buffer is returned and (*pnNode) +** is set to the size of the buffer in bytes. +** +** Or, if an error does occur, NULL is returned and an error code left +** in the RtreeCheck object. The final value of *pnNode is undefined in +** this case. +*/ +static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){ + u8 *pRet = 0; /* Return value */ + + if( pCheck->rc==SQLITE_OK && pCheck->pGetNode==0 ){ + pCheck->pGetNode = rtreeCheckPrepare(pCheck, + "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", + pCheck->zDb, pCheck->zTab + ); + } + + if( pCheck->rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck->pGetNode, 1, iNode); + if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){ + int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0); + const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0); + pRet = sqlite3_malloc64(nNode); + if( pRet==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + memcpy(pRet, pNode, nNode); + *pnNode = nNode; + } + } + rtreeCheckReset(pCheck, pCheck->pGetNode); + if( pCheck->rc==SQLITE_OK && pRet==0 ){ + rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode); + } + } + + return pRet; +} + +/* +** This function is used to check that the %_parent (if bLeaf==0) or %_rowid +** (if bLeaf==1) table contains a specified entry. The schemas of the +** two tables are: +** +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER, ...) +** +** In both cases, this function checks that there exists an entry with +** IPK value iKey and the second column set to iVal. +** +*/ +static void rtreeCheckMapping( + RtreeCheck *pCheck, /* RtreeCheck object */ + int bLeaf, /* True for a leaf cell, false for interior */ + i64 iKey, /* Key for mapping */ + i64 iVal /* Expected value for mapping */ +){ + int rc; + sqlite3_stmt *pStmt; + const char *azSql[2] = { + "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?1", + "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?1" + }; + + assert( bLeaf==0 || bLeaf==1 ); + if( pCheck->aCheckMapping[bLeaf]==0 ){ + pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck, + azSql[bLeaf], pCheck->zDb, pCheck->zTab + ); + } + if( pCheck->rc!=SQLITE_OK ) return; + + pStmt = pCheck->aCheckMapping[bLeaf]; + sqlite3_bind_int64(pStmt, 1, iKey); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_DONE ){ + rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table", + iKey, iVal, (bLeaf ? "%_rowid" : "%_parent") + ); + }else if( rc==SQLITE_ROW ){ + i64 ii = sqlite3_column_int64(pStmt, 0); + if( ii!=iVal ){ + rtreeCheckAppendMsg(pCheck, + "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)", + iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal + ); + } + } + rtreeCheckReset(pCheck, pStmt); +} + +/* +** Argument pCell points to an array of coordinates stored on an rtree page. +** This function checks that the coordinates are internally consistent (no +** x1>x2 conditions) and adds an error message to the RtreeCheck object +** if they are not. +** +** Additionally, if pParent is not NULL, then it is assumed to point to +** the array of coordinates on the parent page that bound the page +** containing pCell. In this case it is also verified that the two +** sets of coordinates are mutually consistent and an error message added +** to the RtreeCheck object if they are not. +*/ +static void rtreeCheckCellCoord( + RtreeCheck *pCheck, + i64 iNode, /* Node id to use in error messages */ + int iCell, /* Cell number to use in error messages */ + u8 *pCell, /* Pointer to cell coordinates */ + u8 *pParent /* Pointer to parent coordinates */ +){ + RtreeCoord c1, c2; + RtreeCoord p1, p2; + int i; + + for(i=0; inDim; i++){ + readCoord(&pCell[4*2*i], &c1); + readCoord(&pCell[4*(2*i + 1)], &c2); + + /* printf("%e, %e\n", c1.u.f, c2.u.f); */ + if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode + ); + } + + if( pParent ){ + readCoord(&pParent[4*2*i], &p1); + readCoord(&pParent[4*(2*i + 1)], &p2); + + if( (pCheck->bInt ? c1.ibInt ? c2.i>p2.i : c2.f>p2.f) + ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt relative to parent" + , i, iCell, iNode + ); + } + } + } +} + +/* +** Run rtreecheck() checks on node iNode, which is at depth iDepth within +** the r-tree structure. Argument aParent points to the array of coordinates +** that bound node iNode on the parent node. +** +** If any problems are discovered, an error message is appended to the +** report accumulated in the RtreeCheck object. +*/ +static void rtreeCheckNode( + RtreeCheck *pCheck, + int iDepth, /* Depth of iNode (0==leaf) */ + u8 *aParent, /* Buffer containing parent coords */ + i64 iNode /* Node to check */ +){ + u8 *aNode = 0; + int nNode = 0; + + assert( iNode==1 || aParent!=0 ); + assert( pCheck->nDim>0 ); + + aNode = rtreeCheckGetNode(pCheck, iNode, &nNode); + if( aNode ){ + if( nNode<4 ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small (%d bytes)", iNode, nNode + ); + }else{ + int nCell; /* Number of cells on page */ + int i; /* Used to iterate through cells */ + if( aParent==0 ){ + iDepth = readInt16(aNode); + if( iDepth>RTREE_MAX_DEPTH ){ + rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth); + sqlite3_free(aNode); + return; + } + } + nCell = readInt16(&aNode[2]); + if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small for cell count of %d (%d bytes)", + iNode, nCell, nNode + ); + }else{ + for(i=0; inDim*2*4)]; + i64 iVal = readInt64(pCell); + rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); + + if( iDepth>0 ){ + rtreeCheckMapping(pCheck, 0, iVal, iNode); + rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); + pCheck->nNonLeaf++; + }else{ + rtreeCheckMapping(pCheck, 1, iVal, iNode); + pCheck->nLeaf++; + } + } + } + } + sqlite3_free(aNode); + } +} + +/* +** The second argument to this function must be either "_rowid" or +** "_parent". This function checks that the number of entries in the +** %_rowid or %_parent table is exactly nExpect. If not, it adds +** an error message to the report in the RtreeCheck object indicated +** by the first argument. +*/ +static void rtreeCheckCount(RtreeCheck *pCheck, const char *zTbl, i64 nExpect){ + if( pCheck->rc==SQLITE_OK ){ + sqlite3_stmt *pCount; + pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'", + pCheck->zDb, pCheck->zTab, zTbl + ); + if( pCount ){ + if( sqlite3_step(pCount)==SQLITE_ROW ){ + i64 nActual = sqlite3_column_int64(pCount, 0); + if( nActual!=nExpect ){ + rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table" + " - expected %lld, actual %lld" , zTbl, nExpect, nActual + ); + } + } + pCheck->rc = sqlite3_finalize(pCount); + } + } +} + +/* +** This function does the bulk of the work for the rtree integrity-check. +** It is called by rtreecheck(), which is the SQL function implementation. +*/ +static int rtreeCheckTable( + sqlite3 *db, /* Database handle to access db through */ + const char *zDb, /* Name of db ("main", "temp" etc.) */ + const char *zTab, /* Name of rtree table to check */ + char **pzReport /* OUT: sqlite3_malloc'd report text */ +){ + RtreeCheck check; /* Common context for various routines */ + sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ + int bEnd = 0; /* True if transaction should be closed */ + int nAux = 0; /* Number of extra columns. */ + + /* Initialize the context object */ + memset(&check, 0, sizeof(check)); + check.db = db; + check.zDb = zDb; + check.zTab = zTab; + + /* If there is not already an open transaction, open one now. This is + ** to ensure that the queries run as part of this integrity-check operate + ** on a consistent snapshot. */ + if( sqlite3_get_autocommit(db) ){ + check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + bEnd = 1; + } + + /* Find the number of auxiliary columns */ + if( check.rc==SQLITE_OK ){ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); + if( pStmt ){ + nAux = sqlite3_column_count(pStmt) - 2; + sqlite3_finalize(pStmt); + } + check.rc = SQLITE_OK; + } + + /* Find number of dimensions in the rtree table. */ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.%Q", zDb, zTab); + if( pStmt ){ + int rc; + check.nDim = (sqlite3_column_count(pStmt) - 1 - nAux) / 2; + if( check.nDim<1 ){ + rtreeCheckAppendMsg(&check, "Schema corrupt or not an rtree"); + }else if( SQLITE_ROW==sqlite3_step(pStmt) ){ + check.bInt = (sqlite3_column_type(pStmt, 1)==SQLITE_INTEGER); + } + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_CORRUPT ) check.rc = rc; + } + + /* Do the actual integrity-check */ + if( check.nDim>=1 ){ + if( check.rc==SQLITE_OK ){ + rtreeCheckNode(&check, 0, 0, 1); + } + rtreeCheckCount(&check, "_rowid", check.nLeaf); + rtreeCheckCount(&check, "_parent", check.nNonLeaf); + } + + /* Finalize SQL statements used by the integrity-check */ + sqlite3_finalize(check.pGetNode); + sqlite3_finalize(check.aCheckMapping[0]); + sqlite3_finalize(check.aCheckMapping[1]); + + /* If one was opened, close the transaction */ + if( bEnd ){ + int rc = sqlite3_exec(db, "END", 0, 0, 0); + if( check.rc==SQLITE_OK ) check.rc = rc; + } + *pzReport = check.zReport; + return check.rc; +} + +/* +** Usage: +** +** rtreecheck(); +** rtreecheck(, ); +** +** Invoking this SQL function runs an integrity-check on the named rtree +** table. The integrity-check verifies the following: +** +** 1. For each cell in the r-tree structure (%_node table), that: +** +** a) for each dimension, (coord1 <= coord2). +** +** b) unless the cell is on the root node, that the cell is bounded +** by the parent cell on the parent node. +** +** c) for leaf nodes, that there is an entry in the %_rowid +** table corresponding to the cell's rowid value that +** points to the correct node. +** +** d) for cells on non-leaf nodes, that there is an entry in the +** %_parent table mapping from the cell's child node to the +** node that it resides on. +** +** 2. That there are the same number of entries in the %_rowid table +** as there are leaf cells in the r-tree structure, and that there +** is a leaf cell that corresponds to each entry in the %_rowid table. +** +** 3. That there are the same number of entries in the %_parent table +** as there are non-leaf cells in the r-tree structure, and that +** there is a non-leaf cell that corresponds to each entry in the +** %_parent table. +*/ +static void rtreecheck( + sqlite3_context *ctx, + int nArg, + sqlite3_value **apArg +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(ctx, + "wrong number of arguments to function rtreecheck()", -1 + ); + }else{ + int rc; + char *zReport = 0; + const char *zDb = (const char*)sqlite3_value_text(apArg[0]); + const char *zTab; + if( nArg==1 ){ + zTab = zDb; + zDb = "main"; + }else{ + zTab = (const char*)sqlite3_value_text(apArg[1]); + } + rc = rtreeCheckTable(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); + if( rc==SQLITE_OK ){ + sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(ctx, rc); + } + sqlite3_free(zReport); + } +} + +/* Conditionally include the geopoly code */ +#ifdef SQLITE_ENABLE_GEOPOLY +#include "third_party/sqlite3/geopoly.inc" +#endif + +/* +** Register the r-tree module with database handle db. This creates the +** virtual table module "rtree" and the debugging/analysis scalar +** function "rtreenode". +*/ +int sqlite3RtreeInit(sqlite3 *db){ + const int utf8 = SQLITE_UTF8; + int rc; + + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtreecheck", -1, utf8, 0,rtreecheck, 0,0); + } + if( rc==SQLITE_OK ){ +#ifdef SQLITE_RTREE_INT_ONLY + void *c = (void *)RTREE_COORD_INT32; +#else + void *c = (void *)RTREE_COORD_REAL32; +#endif + rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); + } + if( rc==SQLITE_OK ){ + void *c = (void *)RTREE_COORD_INT32; + rc = sqlite3_create_module_v2(db, "rtree_i32", &rtreeModule, c, 0); + } +#ifdef SQLITE_ENABLE_GEOPOLY + if( rc==SQLITE_OK ){ + rc = sqlite3_geopoly_init(db); + } +#endif + + return rc; +} + +/* +** This routine deletes the RtreeGeomCallback object that was attached +** one of the SQL functions create by sqlite3_rtree_geometry_callback() +** or sqlite3_rtree_query_callback(). In other words, this routine is the +** destructor for an RtreeGeomCallback objecct. This routine is called when +** the corresponding SQL function is deleted. +*/ +static void rtreeFreeCallback(void *p){ + RtreeGeomCallback *pInfo = (RtreeGeomCallback*)p; + if( pInfo->xDestructor ) pInfo->xDestructor(pInfo->pContext); + sqlite3_free(p); +} + +/* +** This routine frees the BLOB that is returned by geomCallback(). +*/ +static void rtreeMatchArgFree(void *pArg){ + int i; + RtreeMatchArg *p = (RtreeMatchArg*)pArg; + for(i=0; inParam; i++){ + sqlite3_value_free(p->apSqlParam[i]); + } + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. +** +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + sqlite3_int64 nBlob; + int memErr = 0; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) + + nArg*sizeof(sqlite3_value*); + pBlob = (RtreeMatchArg *)sqlite3_malloc64(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->iSize = nBlob; + pBlob->cb = pGeomCtx[0]; + pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; + pBlob->nParam = nArg; + for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); + if( pBlob->apSqlParam[i]==0 ) memErr = 1; +#ifdef SQLITE_RTREE_INT_ONLY + pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#else + pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif + } + if( memErr ){ + sqlite3_result_error_nomem(ctx); + rtreeMatchArgFree(pBlob); + }else{ + sqlite3_result_pointer(ctx, pBlob, "RtreeMatchArg", rtreeMatchArgFree); + } + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +int sqlite3_rtree_geometry_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_rtree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3RtreeInit(db); +} +#endif + +#endif diff --git a/third_party/sqlite3/rtree.inc b/third_party/sqlite3/rtree.inc new file mode 100644 index 000000000..cb9cdd097 --- /dev/null +++ b/third_party/sqlite3/rtree.inc @@ -0,0 +1,31 @@ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ +#include "third_party/sqlite3/sqlite3.h" +/* clang-format off */ + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_RTREE +#endif + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +int sqlite3RtreeInit(sqlite3 *db); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ diff --git a/third_party/sqlite3/select.c b/third_party/sqlite3/select.c new file mode 100644 index 000000000..91f51f954 --- /dev/null +++ b/third_party/sqlite3/select.c @@ -0,0 +1,7161 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle SELECT statements in SQLite. +*/ +#include "third_party/sqlite3/sqliteInt.inc" +/* clang-format off */ + +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +** +** The aDefer[] array is used by the sorter-references optimization. For +** example, assuming there is no index that can be used for the ORDER BY, +** for the query: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10; +** +** it may be more efficient to add just the "a" values to the sorter, and +** retrieve the associated "bigblob" values directly from table t1 as the +** 10 smallest "a" values are extracted from the sorter. +** +** When the sorter-reference optimization is used, there is one entry in the +** aDefer[] array for each database table that may be read as values are +** extracted from the sorter. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ + int labelOBLopt; /* Jump here when sorter is full */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + u8 nDefer; /* Number of valid entries in aDefer[] */ + struct DeferredCsr { + Table *pTab; /* Table definition */ + int iCsr; /* Cursor number for table */ + int nKey; /* Number of PK columns for table pTab (>=1) */ + } aDefer[4]; +#endif + struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ + +/* +** Delete all the content of a Select structure. Deallocate the structure +** itself depending on the value of bFree +** +** If bFree==1, call sqlite3DbFree() on the p object. +** If bFree==0, Leave the first Select object unfreed +*/ +static void clearSelect(sqlite3 *db, Select *p, int bFree){ + while( p ){ + Select *pPrior = p->pPrior; + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pLimit); + if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ + sqlite3WindowListDelete(db, p->pWinDefn); + } + while( p->pWin ){ + assert( p->pWin->ppThis==&p->pWin ); + sqlite3WindowUnlinkFromSelect(p->pWin); + } +#endif + if( bFree ) sqlite3DbFreeNN(db, p); + p = pPrior; + bFree = 1; + } +} + +/* +** Initialize a SelectDest structure. +*/ +void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ + pDest->eDest = (u8)eDest; + pDest->iSDParm = iParm; + pDest->iSDParm2 = 0; + pDest->zAffSdst = 0; + pDest->iSdst = 0; + pDest->nSdst = 0; +} + + +/* +** Allocate a new Select structure and return a pointer to that +** structure. +*/ +Select *sqlite3SelectNew( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* which columns to include in the result */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* the WHERE clause */ + ExprList *pGroupBy, /* the GROUP BY clause */ + Expr *pHaving, /* the HAVING clause */ + ExprList *pOrderBy, /* the ORDER BY clause */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ + Expr *pLimit /* LIMIT value. NULL means not used */ +){ + Select *pNew, *pAllocated; + Select standin; + pAllocated = pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) ); + if( pNew==0 ){ + assert( pParse->db->mallocFailed ); + pNew = &standin; + } + if( pEList==0 ){ + pEList = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(pParse->db,TK_ASTERISK,0)); + } + pNew->pEList = pEList; + pNew->op = TK_SELECT; + pNew->selFlags = selFlags; + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selId = ++pParse->nSelect; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = 0; + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + pNew->pSrc = pSrc; + pNew->pWhere = pWhere; + pNew->pGroupBy = pGroupBy; + pNew->pHaving = pHaving; + pNew->pOrderBy = pOrderBy; + pNew->pPrior = 0; + pNew->pNext = 0; + pNew->pLimit = pLimit; + pNew->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = 0; +#endif + if( pParse->db->mallocFailed ) { + clearSelect(pParse->db, pNew, pNew!=&standin); + pAllocated = 0; + }else{ + assert( pNew->pSrc!=0 || pParse->nErr>0 ); + } + return pAllocated; +} + + +/* +** Delete the given Select structure and all of its substructures. +*/ +void sqlite3SelectDelete(sqlite3 *db, Select *p){ + if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); +} + +/* +** Return a pointer to the right-most SELECT statement in a compound. +*/ +static Select *findRightmost(Select *p){ + while( p->pNext ) p = p->pNext; + return p; +} + +/* +** Given 1 to 3 identifiers preceding the JOIN keyword, determine the +** type of join. Return an integer constant that expresses that type +** in terms of the following bit values: +** +** JT_INNER +** JT_CROSS +** JT_OUTER +** JT_NATURAL +** JT_LEFT +** JT_RIGHT +** +** A full outer join is the combination of JT_LEFT and JT_RIGHT. +** +** If an illegal or unsupported join type is seen, then still return +** a join type, but put an error in the pParse structure. +*/ +int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ + int jointype = 0; + Token *apAll[3]; + Token *p; + /* 0123456789 123456789 123456789 123 */ + static const char zKeyText[] = "naturaleftouterightfullinnercross"; + static const struct { + u8 i; /* Beginning of keyword text in zKeyText[] */ + u8 nChar; /* Length of the keyword in characters */ + u8 code; /* Join type mask */ + } aKeyword[] = { + /* natural */ { 0, 7, JT_NATURAL }, + /* left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* outer */ { 10, 5, JT_OUTER }, + /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* inner */ { 23, 5, JT_INNER }, + /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + }; + int i, j; + apAll[0] = pA; + apAll[1] = pB; + apAll[2] = pC; + for(i=0; i<3 && apAll[i]; i++){ + p = apAll[i]; + for(j=0; jn==aKeyword[j].nChar + && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + jointype |= aKeyword[j].code; + break; + } + } + testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); + if( j>=ArraySize(aKeyword) ){ + jointype |= JT_ERROR; + break; + } + } + if( + (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || + (jointype & JT_ERROR)!=0 + ){ + const char *zSp = " "; + assert( pB!=0 ); + if( pC==0 ){ zSp++; } + sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " + "%T %T%s%T", pA, pB, zSp, pC); + jointype = JT_INNER; + }else if( (jointype & JT_OUTER)!=0 + && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ + sqlite3ErrorMsg(pParse, + "RIGHT and FULL OUTER JOINs are not currently supported"); + jointype = JT_INNER; + } + return jointype; +} + +/* +** Return the index of a column in a table. Return -1 if the column +** is not contained in the table. +*/ +int sqlite3ColumnIndex(Table *pTab, const char *zCol){ + int i; + u8 h = sqlite3StrIHash(zCol); + Column *pCol; + for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ + if( pCol->hName==h && sqlite3StrICmp(pCol->zName, zCol)==0 ) return i; + } + return -1; +} + +/* +** Search the first N tables in pSrc, from left to right, looking for a +** table that has a column named zCol. +** +** When found, set *piTab and *piCol to the table index and column index +** of the matching column and return TRUE. +** +** If not found, return FALSE. +*/ +static int tableAndColumnIndex( + SrcList *pSrc, /* Array of tables to search */ + int N, /* Number of tables in pSrc->a[] to search */ + const char *zCol, /* Name of the column we are looking for */ + int *piTab, /* Write index of pSrc->a[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int bIgnoreHidden /* True to ignore hidden columns */ +){ + int i; /* For looping over tables in pSrc */ + int iCol; /* Index of column matching zCol */ + + assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ + for(i=0; ia[i].pTab, zCol); + if( iCol>=0 + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + ){ + if( piTab ){ + *piTab = i; + *piCol = iCol; + } + return 1; + } + } + return 0; +} + +/* +** This function is used to add terms implied by JOIN syntax to the +** WHERE clause expression of a SELECT statement. The new term, which +** is ANDed with the existing WHERE clause, is of the form: +** +** (tab1.col1 = tab2.col2) +** +** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the +** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is +** column iColRight of tab2. +*/ +static void addWhereTerm( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* List of tables in FROM clause */ + int iLeft, /* Index of first table to join in pSrc */ + int iColLeft, /* Index of column in first table */ + int iRight, /* Index of second table in pSrc */ + int iColRight, /* Index of column in second table */ + int isOuterJoin, /* True if this is an OUTER join */ + Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ +){ + sqlite3 *db = pParse->db; + Expr *pE1; + Expr *pE2; + Expr *pEq; + + assert( iLeftnSrc>iRight ); + assert( pSrc->a[iLeft].pTab ); + assert( pSrc->a[iRight].pTab ); + + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); + pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); + + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); + if( pEq && isOuterJoin ){ + ExprSetProperty(pEq, EP_FromJoin); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); + pEq->iRightJoinTable = pE2->iTable; + } + *ppWhere = sqlite3ExprAnd(pParse, *ppWhere, pEq); +} + +/* +** Set the EP_FromJoin property on all terms of the given expression. +** And set the Expr.iRightJoinTable to iTable for every term in the +** expression. +** +** The EP_FromJoin property is used on terms of an expression to tell +** the LEFT OUTER JOIN processing logic that this term is part of the +** join restriction specified in the ON or USING clause and not a part +** of the more general WHERE clause. These terms are moved over to the +** WHERE clause during join processing but we need to remember that they +** originated in the ON or USING clause. +** +** The Expr.iRightJoinTable tells the WHERE clause processing that the +** expression depends on table iRightJoinTable even if that table is not +** explicitly mentioned in the expression. That information is needed +** for cases like this: +** +** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5 +** +** The where clause needs to defer the handling of the t1.x=5 +** term until after the t2 loop of the join. In that way, a +** NULL t2 row will be inserted whenever t1.x!=5. If we do not +** defer the handling of t1.x=5, it will be processed immediately +** after the t1 loop and rows with t1.x!=5 will never appear in +** the output, which is incorrect. +*/ +void sqlite3SetJoinExpr(Expr *p, int iTable){ + while( p ){ + ExprSetProperty(p, EP_FromJoin); + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(p, EP_NoReduce); + p->iRightJoinTable = iTable; + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } + sqlite3SetJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + +/* Undo the work of sqlite3SetJoinExpr(). In the expression p, convert every +** term that is marked with EP_FromJoin and iRightJoinTable==iTable into +** an ordinary term that omits the EP_FromJoin mark. +** +** This happens when a LEFT JOIN is simplified into an ordinary JOIN. +*/ +static void unsetJoinExpr(Expr *p, int iTable){ + while( p ){ + if( ExprHasProperty(p, EP_FromJoin) + && (iTable<0 || p->iRightJoinTable==iTable) ){ + ExprClearProperty(p, EP_FromJoin); + } + if( p->op==TK_COLUMN && p->iTable==iTable ){ + ExprClearProperty(p, EP_CanBeNull); + } + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + unsetJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } + unsetJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + +/* +** This routine processes the join information for a SELECT statement. +** ON and USING clauses are converted into extra terms of the WHERE clause. +** NATURAL joins also create extra WHERE clause terms. +** +** The terms of a FROM clause are contained in the Select.pSrc structure. +** The left most table is the first entry in Select.pSrc. The right-most +** table is the last entry. The join operator is held in the entry to +** the left. Thus entry 0 contains the join operator for the join between +** entries 0 and 1. Any ON or USING clauses associated with the join are +** also attached to the left entry. +** +** This routine returns the number of errors encountered. +*/ +static int sqliteProcessJoin(Parse *pParse, Select *p){ + SrcList *pSrc; /* All tables in the FROM clause */ + int i, j; /* Loop counters */ + SrcItem *pLeft; /* Left table being joined */ + SrcItem *pRight; /* Right table being joined */ + + pSrc = p->pSrc; + pLeft = &pSrc->a[0]; + pRight = &pLeft[1]; + for(i=0; inSrc-1; i++, pRight++, pLeft++){ + Table *pRightTab = pRight->pTab; + int isOuter; + + if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; + + /* When the NATURAL keyword is present, add WHERE clause terms for + ** every column that the two tables have in common. + */ + if( pRight->fg.jointype & JT_NATURAL ){ + if( pRight->pOn || pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "a NATURAL join may not have " + "an ON or USING clause", 0); + return 1; + } + for(j=0; jnCol; j++){ + char *zName; /* Name of column in the right table */ + int iLeft; /* Matching left table */ + int iLeftCol; /* Matching column in the left table */ + + if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue; + zName = pRightTab->aCol[j].zName; + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 1) ){ + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, + isOuter, &p->pWhere); + } + } + } + + /* Disallow both ON and USING clauses in the same join + */ + if( pRight->pOn && pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "cannot have both ON and USING " + "clauses in the same join"); + return 1; + } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + if( pRight->pOn ){ + if( isOuter ) sqlite3SetJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->pOn); + pRight->pOn = 0; + } + + /* Create extra terms on the WHERE clause for each column named + ** in the USING clause. Example: If the two tables to be joined are + ** A and B and the USING clause names X, Y, and Z, then add this + ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z + ** Report an error if any column mentioned in the USING clause is + ** not contained in both tables to be joined. + */ + if( pRight->pUsing ){ + IdList *pList = pRight->pUsing; + for(j=0; jnId; j++){ + char *zName; /* Name of the term in the USING clause */ + int iLeft; /* Table on the left with matching column name */ + int iLeftCol; /* Column number of matching column on the left */ + int iRightCol; /* Column number of matching column on the right */ + + zName = pList->a[j].zName; + iRightCol = sqlite3ColumnIndex(pRightTab, zName); + if( iRightCol<0 + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 0) + ){ + sqlite3ErrorMsg(pParse, "cannot join using column %s - column " + "not present in both tables", zName); + return 1; + } + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, + isOuter, &p->pWhere); + } + } + } + return 0; +} + +/* +** An instance of this object holds information (beyond pParse and pSelect) +** needed to load the next result row that is to be added to the sorter. +*/ +typedef struct RowLoadInfo RowLoadInfo; +struct RowLoadInfo { + int regResult; /* Store results in array of registers here */ + u8 ecelFlags; /* Flag argument to ExprCodeExprList() */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra; /* Extra columns needed by sorter refs */ + int regExtraResult; /* Where to load the extra columns */ +#endif +}; + +/* +** This routine does the work of loading query data into an array of +** registers so that it can be added to the sorter. +*/ +static void innerLoopLoadRow( + Parse *pParse, /* Statement under construction */ + Select *pSelect, /* The query being coded */ + RowLoadInfo *pInfo /* Info needed to complete the row load */ +){ + sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult, + 0, pInfo->ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pInfo->pExtra ){ + sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0); + sqlite3ExprListDelete(pParse->db, pInfo->pExtra); + } +#endif +} + +/* +** Code the OP_MakeRecord instruction that generates the entry to be +** added into the sorter. +** +** Return the register in which the result is stored. +*/ +static int makeSorterRecord( + Parse *pParse, + SortCtx *pSort, + Select *pSelect, + int regBase, + int nBase +){ + int nOBSat = pSort->nOBSat; + Vdbe *v = pParse->pVdbe; + int regOut = ++pParse->nMem; + if( pSort->pDeferredRowLoad ){ + innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut); + return regOut; +} + +/* +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. +*/ +static void pushOntoSorter( + Parse *pParse, /* Parser context */ + SortCtx *pSort, /* Information about the ORDER BY clause */ + Select *pSelect, /* The whole SELECT statement */ + int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ + int nData, /* Number of elements in the regData data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ +){ + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ + int regBase; /* Regs for sorter record */ + int regRecord = 0; /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ + int iSkip = 0; /* End of the sorter insert loop */ + + assert( bSeq==0 || bSeq==1 ); + + /* Three cases: + ** (1) The data to be sorted has already been packed into a Record + ** by a prior OP_MakeRecord. In this case nData==1 and regData + ** will be completely unrelated to regOrigData. + ** (2) All output columns are included in the sort record. In that + ** case regData==regOrigData. + ** (3) Some output columns are omitted from the sort record due to + ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** SQLITE_ECEL_OMITREF optimization, or due to the + ** SortCtx.pDeferredRowLoad optimiation. In any of these cases + ** regOrigData is 0 to prevent this routine from trying to copy + ** values that might not yet exist. + */ + assert( nData==1 || regData==regOrigData || regOrigData==0 ); + + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nPrefixReg; + }else{ + regBase = pParse->nMem + 1; + pParse->nMem += nBase; + } + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + } + if( nPrefixReg==0 && nData>0 ){ + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); + } + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; + pOp->p2 = nKey + nData; + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); + testcase( pKI->nAllField > pKI->nKeyField+2 ); + pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, + pKI->nAllField-pKI->nKeyField-1); + pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( iLimit ){ + /* At this point the values for the new sorter entry are stored + ** in an array of registers. They need to be composed into a record + ** and inserted into the sorter if either (a) there are currently + ** less than LIMIT+OFFSET items or (b) the new record is smaller than + ** the largest record currently in the sorter. If (b) is true and there + ** are already LIMIT+OFFSET items in the sorter, delete the largest + ** entry before inserting the new one. This way there are never more + ** than LIMIT+OFFSET items in the sorter. + ** + ** If the new record does not need to be inserted into the sorter, + ** jump to the next iteration of the loop. If the pSort->labelOBLopt + ** value is not zero, then it is a label of where to jump. Otherwise, + ** just bypass the row insert logic. See the header comment on the + ** sqlite3WhereOrderByLimitOptLabel() function for additional info. + */ + int iCsr = pSort->iECursor; + sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0); + iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE, + iCsr, 0, regBase+nOBSat, nExpr-nOBSat); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Delete, iCsr); + } + if( regRecord==0 ){ + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, + regBase+nOBSat, nBase-nOBSat); + if( iSkip ){ + sqlite3VdbeChangeP2(v, iSkip, + pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); + } +} + +/* +** Add code to implement the OFFSET +*/ +static void codeOffset( + Vdbe *v, /* Generate code into this VM */ + int iOffset, /* Register holding the offset counter */ + int iContinue /* Jump here to skip the current record */ +){ + if( iOffset>0 ){ + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); + } +} + +/* +** Add code that will check to make sure the N registers starting at iMem +** form a distinct entry. iTab is a sorting index that holds previously +** seen combinations of the N values. A new entry is made in iTab +** if the current N values are new. +** +** A jump to addrRepeat is made and the N+1 values are popped from the +** stack if the top N elements are not distinct. +*/ +static void codeDistinct( + Parse *pParse, /* Parsing and code generating context */ + int iTab, /* A sorting index used to test for distinctness */ + int addrRepeat, /* Jump to here if not distinct */ + int N, /* Number of elements */ + int iMem /* First element */ +){ + Vdbe *v; + int r1; + + v = pParse->pVdbe; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, r1); +} + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES +/* +** This function is called as part of inner-loop generation for a SELECT +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference +** optimization should be used for. The sorter-reference optimization +** is used for SELECT queries like: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10 +** +** If the optimization is used for expression "bigblob", then instead of +** storing values read from that column in the sorter records, the PK of +** the row from table t1 is stored instead. Then, as records are extracted from +** the sorter to return to the user, the required value of bigblob is +** retrieved directly from table t1. If the values are very large, this +** can be more efficient than storing them directly in the sorter records. +** +** The ExprList_item.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. +** Additionally, the pSort->aDefer[] array is populated with entries +** for all cursors required to evaluate all selected expressions. Finally. +** output variable (*ppExtra) is set to an expression list containing +** expressions for all extra PK values that should be stored in the +** sorter records. +*/ +static void selectExprDefer( + Parse *pParse, /* Leave any error here */ + SortCtx *pSort, /* Sorter context */ + ExprList *pEList, /* Expressions destined for sorter */ + ExprList **ppExtra /* Expressions to append to sorter record */ +){ + int i; + int nDefer = 0; + ExprList *pExtra = 0; + for(i=0; inExpr; i++){ + struct ExprList_item *pItem = &pEList->a[i]; + if( pItem->u.x.iOrderByCol==0 ){ + Expr *pExpr = pItem->pExpr; + Table *pTab = pExpr->y.pTab; + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 && pTab && !IsVirtual(pTab) + && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) + ){ + int j; + for(j=0; jaDefer[j].iCsr==pExpr->iTable ) break; + } + if( j==nDefer ){ + if( nDefer==ArraySize(pSort->aDefer) ){ + continue; + }else{ + int nKey = 1; + int k; + Index *pPk = 0; + if( !HasRowid(pTab) ){ + pPk = sqlite3PrimaryKeyIndex(pTab); + nKey = pPk->nKeyCol; + } + for(k=0; kiTable = pExpr->iTable; + pNew->y.pTab = pExpr->y.pTab; + pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; + pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); + } + } + pSort->aDefer[nDefer].pTab = pExpr->y.pTab; + pSort->aDefer[nDefer].iCsr = pExpr->iTable; + pSort->aDefer[nDefer].nKey = nKey; + nDefer++; + } + } + pItem->bSorterRef = 1; + } + } + } + pSort->nDefer = (u8)nDefer; + *ppExtra = pExtra; +} +#endif + +/* +** This routine generates the code for the inside of the inner loop +** of a SELECT. +** +** If srcTab is negative, then the p->pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and p->pEList is used only +** to get the number of columns and the collation sequence for each column. +*/ +static void selectInnerLoop( + Parse *pParse, /* The parser context */ + Select *p, /* The complete select statement being coded */ + int srcTab, /* Pull data from this table if non-negative */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ + SelectDest *pDest, /* How to dispose of the results */ + int iContinue, /* Jump here to continue with next row */ + int iBreak /* Jump here to break out of the inner loop */ +){ + Vdbe *v = pParse->pVdbe; + int i; + int hasDistinct; /* True if the DISTINCT keyword is present */ + int eDest = pDest->eDest; /* How to dispose of results */ + int iParm = pDest->iSDParm; /* First argument to disposal method */ + int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ + RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */ + + /* Usually, regResult is the first cell in an array of memory cells + ** containing the current result row. In this case regOrig is set to the + ** same value. However, if the results are being sent to the sorter, the + ** values for any expressions that are also part of the sort-key are omitted + ** from this array. In this case regOrig is set to zero. */ + int regResult; /* Start of memory holding current results */ + int regOrig; /* Start of memory holding full result (or 0) */ + + assert( v ); + assert( p->pEList!=0 ); + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); + codeOffset(v, p->iOffset, iContinue); + } + + /* Pull the requested columns. + */ + nResultCol = p->pEList->nExpr; + + if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; + pParse->nMem += nPrefixReg; + } + pDest->iSdst = pParse->nMem+1; + pParse->nMem += nResultCol; + }else if( pDest->iSdst+nResultCol > pParse->nMem ){ + /* This is an error condition that can result, for example, when a SELECT + ** on the right-hand side of an INSERT contains more result columns than + ** there are columns in the table on the left. The error will be caught + ** and reported later. But we need to make sure enough memory is allocated + ** to avoid other spurious errors in the meantime. */ + pParse->nMem += nResultCol; + } + pDest->nSdst = nResultCol; + regOrig = regResult = pDest->iSdst; + if( srcTab>=0 ){ + for(i=0; ipEList->a[i].zEName)); + } + }else if( eDest!=SRT_Exists ){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra = 0; +#endif + /* If the destination is an EXISTS(...) expression, the actual + ** values returned by the SELECT are not required. + */ + u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */ + ExprList *pEList; + if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ + ecelFlags = SQLITE_ECEL_DUP; + }else{ + ecelFlags = 0; + } + if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){ + /* For each expression in p->pEList that is a copy of an expression in + ** the ORDER BY clause (pSort->pOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY + ** expression within the sort-key that pushOntoSorter() will generate. + ** This allows the p->pEList field to be omitted from the sorted record, + ** saving space and CPU cycles. */ + ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ + int j; + if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ + p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; + } + } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + selectExprDefer(pParse, pSort, p->pEList, &pExtra); + if( pExtra && pParse->db->mallocFailed==0 ){ + /* If there are any extra PK columns to add to the sorter records, + ** allocate extra memory cells and adjust the OpenEphemeral + ** instruction to account for the larger records. This is only + ** required if there are one or more WITHOUT ROWID tables with + ** composite primary keys in the SortCtx.aDefer[] array. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->p2 += (pExtra->nExpr - pSort->nDefer); + pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer); + pParse->nMem += pExtra->nExpr; + } +#endif + + /* Adjust nResultCol to account for columns that are omitted + ** from the sorter by the optimizations in this branch */ + pEList = p->pEList; + for(i=0; inExpr; i++){ + if( pEList->a[i].u.x.iOrderByCol>0 +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + || pEList->a[i].bSorterRef +#endif + ){ + nResultCol--; + regOrig = 0; + } + } + + testcase( regOrig ); + testcase( eDest==SRT_Set ); + testcase( eDest==SRT_Mem ); + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + assert( eDest==SRT_Set || eDest==SRT_Mem + || eDest==SRT_Coroutine || eDest==SRT_Output + || eDest==SRT_Upfrom ); + } + sRowLoadInfo.regResult = regResult; + sRowLoadInfo.ecelFlags = ecelFlags; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + sRowLoadInfo.pExtra = pExtra; + sRowLoadInfo.regExtraResult = regResult + nResultCol; + if( pExtra ) nResultCol += pExtra->nExpr; +#endif + if( p->iLimit + && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 + && nPrefixReg>0 + ){ + assert( pSort!=0 ); + assert( hasDistinct==0 ); + pSort->pDeferredRowLoad = &sRowLoadInfo; + regOrig = 0; + }else{ + innerLoopLoadRow(pParse, p, &sRowLoadInfo); + } + } + + /* If the DISTINCT keyword was present on the SELECT statement + ** and this row has been seen before, then do not make this row + ** part of the result. + */ + if( hasDistinct ){ + switch( pDistinct->eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Null + ** sets the MEM_Cleared bit on the first register of the + ** previous value. This will cause the OP_Ne below to always + ** fail on the first iteration of the loop even if the first + ** row is all NULLs. + */ + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = regPrev; + pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ipEList->a[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); + break; + } + + default: { + assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, + regResult); + break; + } + } + if( pSort==0 ){ + codeOffset(v, p->iOffset, iContinue); + } + } + + switch( eDest ){ + /* In this mode, write each query result to the key of the temporary + ** table iParm. + */ +#ifndef SQLITE_OMIT_COMPOUND_SELECT + case SRT_Union: { + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* Construct a record from the query result, but instead of + ** saving that record, use it as a key to delete elements from + ** the temporary table iParm. + */ + case SRT_Except: { + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); + break; + } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + + /* Store the result as data using a unique key. + */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + testcase( eDest==SRT_Fifo ); + testcase( eDest==SRT_DistFifo ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol); + assert( pSort==0 ); + } +#endif + if( pSort ){ + assert( regResult==regOrig ); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg); + }else{ + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + } + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); + break; + } + + case SRT_Upfrom: { + if( pSort ){ + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + + /* If the UPDATE FROM join is an aggregate that matches no rows, it + ** might still be trying to return one row, because that is what + ** aggregates do. Don't record that empty row in the output table. */ + sqlite3VdbeAddOp2(v, OP_IsNull, regResult, iBreak); VdbeCoverage(v); + + sqlite3VdbeAddOp3(v, OP_MakeRecord, + regResult+(i2<0), nResultCol-(i2<0), r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regResult); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, i2); + } + } + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + if( pSort ){ + /* At first glance you would think we could optimize out the + ** ORDER BY in this case since the order of entries in the set + ** does not matter. But there might be a LIMIT clause, in which + ** case the order does matter */ + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + int r1 = sqlite3GetTempReg(pParse); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + sqlite3ReleaseTempReg(pParse, r1); + } + break; + } + + + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. + */ + case SRT_Mem: { + if( pSort ){ + assert( nResultCol<=pDest->nSdst ); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + assert( nResultCol==pDest->nSdst ); + assert( regResult==iParm ); + /* The LIMIT clause will jump out of the loop for us */ + } + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + case SRT_Coroutine: /* Send data to a co-routine */ + case SRT_Output: { /* Return the results */ + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol, + nPrefixReg); + }else if( eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + }else{ + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + } + break; + } + +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + +#if !defined(SQLITE_OMIT_TRIGGER) + /* Discard the results. This is used for SELECT statements inside + ** the body of a TRIGGER. The purpose of such selects is to call + ** user-defined functions that have side effects. We do not care + ** about the actual results of the select. + */ + default: { + assert( eDest==SRT_Discard ); + break; + } +#endif + } + + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. + */ + if( pSort==0 && p->iLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); + } +} + +/* +** Allocate a KeyInfo object sufficient for an index of N key columns and +** X extra columns. +*/ +KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ + int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + if( p ){ + p->aSortFlags = (u8*)&p->aColl[N+X]; + p->nKeyField = (u16)N; + p->nAllField = (u16)(N+X); + p->enc = ENC(db); + p->db = db; + p->nRef = 1; + memset(&p[1], 0, nExtra); + }else{ + sqlite3OomFault(db); + } + return p; +} + +/* +** Deallocate a KeyInfo object +*/ +void sqlite3KeyInfoUnref(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + } +} + +/* +** Make a new pointer to a KeyInfo object +*/ +KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } + return p; +} + +#ifdef SQLITE_DEBUG +/* +** Return TRUE if a KeyInfo object can be change. The KeyInfo object +** can only be changed if this is just a single reference to the object. +** +** This routine is used only inside of assert() statements. +*/ +int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } +#endif /* SQLITE_DEBUG */ + +/* +** Given an expression list, generate a KeyInfo structure that records +** the collating sequence for each expression in that expression list. +** +** If the ExprList is an ORDER BY or GROUP BY clause then the resulting +** KeyInfo structure is appropriate for initializing a virtual index to +** implement that clause. If the ExprList is the result set of a SELECT +** then the KeyInfo structure is appropriate for initializing a virtual +** index to implement a DISTINCT test. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for seeing that this structure is eventually +** freed. +*/ +KeyInfo *sqlite3KeyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ + int nExpr; + KeyInfo *pInfo; + struct ExprList_item *pItem; + sqlite3 *db = pParse->db; + int i; + + nExpr = pList->nExpr; + pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1); + if( pInfo ){ + assert( sqlite3KeyInfoIsWriteable(pInfo) ); + for(i=iStart, pItem=pList->a+iStart; iaColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); + pInfo->aSortFlags[i-iStart] = pItem->sortFlags; + } + } + return pInfo; +} + +/* +** Name of the connection operator, used for error messages. +*/ +const char *sqlite3SelectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; + } + return z; +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage)); +} + +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b + +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainSetInteger(y,z) +#endif + + +/* +** If the inner loop was generated using a non-null pOrderBy argument, +** then the results were placed in a sorter. After the loop is terminated +** we need to run the sorter and output the results. The following +** routine generates the code needed to do that. +*/ +static void generateSortTail( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SortCtx *pSort, /* Information on the ORDER BY clause */ + int nColumn, /* Number of columns of data */ + SelectDest *pDest /* Write the sorted results here */ +){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ + int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */ + int addr; /* Top of output loop. Jump for Next. */ + int addrOnce = 0; + int iTab; + ExprList *pOrderBy = pSort->pOrderBy; + int eDest = pDest->eDest; + int iParm = pDest->iSDParm; + int regRow; + int regRowid; + int iCol; + int nKey; /* Number of key columns in sorter record */ + int iSortTab; /* Sorter cursor to read from */ + int i; + int bSeq; /* True if sorter record includes seq. no. */ + int nRefKey = 0; + struct ExprList_item *aOutEx = p->pEList->a; + + assert( addrBreak<0 ); + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeGoto(v, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + /* Open any cursors needed for sorter-reference expressions */ + for(i=0; inDefer; i++){ + Table *pTab = pSort->aDefer[i].pTab; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); + } +#endif + + iTab = pSort->iECursor; + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ + regRowid = 0; + regRow = pDest->iSdst; + }else{ + regRowid = sqlite3GetTempReg(pParse); + if( eDest==SRT_EphemTab || eDest==SRT_Table ){ + regRow = sqlite3GetTempReg(pParse); + nColumn = 0; + }else{ + regRow = sqlite3GetTempRange(pParse, nColumn); + } + } + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + int regSortOut = ++pParse->nMem; + iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + nKey+1+nColumn+nRefKey); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + iSortTab = iTab; + bSeq = 1; + } + for(i=0, iCol=nKey+bSeq-1; inDefer ){ + int iKey = iCol+1; + int regKey = sqlite3GetTempRange(pParse, nRefKey); + + for(i=0; inDefer; i++){ + int iCsr = pSort->aDefer[i].iCsr; + Table *pTab = pSort->aDefer[i].pTab; + int nKey = pSort->aDefer[i].nKey; + + sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeCurrentAddr(v)+1, regKey); + }else{ + int k; + int iJmp; + assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey ); + for(k=0; k=0; i--){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( aOutEx[i].bSorterRef ){ + sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); + }else +#endif + { + int iRead; + if( aOutEx[i].u.x.iOrderByCol ){ + iRead = aOutEx[i].u.x.iOrderByCol-1; + }else{ + iRead = iCol--; + } + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); + VdbeComment((v, "%s", aOutEx[i].zEName)); + } + } + switch( eDest ){ + case SRT_Table: + case SRT_EphemTab: { + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case SRT_Set: { + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); + break; + } + case SRT_Mem: { + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + case SRT_Upfrom: { + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord,regRow+(i2<0),nColumn-(i2<0),r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regRow); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regRow, i2); + } + break; + } + default: { + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + testcase( eDest==SRT_Coroutine ); + if( eDest==SRT_Output ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); + }else{ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + } + break; + } + } + if( regRowid ){ + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } + sqlite3ReleaseTempReg(pParse, regRowid); + } + /* The bottom of the loop + */ + sqlite3VdbeResolveLabel(v, addrContinue); + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); + } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); + sqlite3VdbeResolveLabel(v, addrBreak); +} + +/* +** Return a pointer to a string containing the 'declaration type' of the +** expression pExpr. The string may be treated as static by the caller. +** +** Also try to estimate the size of the returned value and return that +** result in *pEstWidth. +** +** The declaration type is the exact datatype definition extracted from the +** original CREATE TABLE statement if the expression is a column. The +** declaration type for a ROWID field is INTEGER. Exactly when an expression +** is considered a column can be complex in the presence of subqueries. The +** result-set expression in all of the following SELECT statements is +** considered a column by this function. +** +** SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl); +** SELECT abc FROM (SELECT col AS abc FROM tbl); +** +** The declaration type for any expression other than a column is NULL. +** +** This routine has either 3 or 6 parameters depending on whether or not +** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used. +*/ +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E) +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E) columnTypeImpl(A,B) +#endif +static const char *columnTypeImpl( + NameContext *pNC, +#ifndef SQLITE_ENABLE_COLUMN_METADATA + Expr *pExpr +#else + Expr *pExpr, + const char **pzOrigDb, + const char **pzOrigTab, + const char **pzOrigCol +#endif +){ + char const *zType = 0; + int j; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#endif + + assert( pExpr!=0 ); + assert( pNC->pSrcList!=0 ); + switch( pExpr->op ){ + case TK_COLUMN: { + /* The expression is a column. Locate the table the column is being + ** extracted from in NameContext.pSrcList. This table may be real + ** database table or a subquery. + */ + Table *pTab = 0; /* Table structure column is extracted from */ + Select *pS = 0; /* Select the column is extracted from */ + int iCol = pExpr->iColumn; /* Index of column in pTab */ + while( pNC && !pTab ){ + SrcList *pTabList = pNC->pSrcList; + for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); + if( jnSrc ){ + pTab = pTabList->a[j].pTab; + pS = pTabList->a[j].pSelect; + }else{ + pNC = pNC->pNext; + } + } + + if( pTab==0 ){ + /* At one time, code such as "SELECT new.x" within a trigger would + ** cause this condition to run. Since then, we have restructured how + ** trigger code is generated and so this condition is no longer + ** possible. However, it can still be true for statements like + ** the following: + ** + ** CREATE TABLE t1(col INTEGER); + ** SELECT (SELECT t1.col) FROM FROM t1; + ** + ** when columnType() is called on the expression "t1.col" in the + ** sub-select. In this case, set the column type to NULL, even + ** though it should really be "INTEGER". + ** + ** This is not a problem, as the column type of "t1.col" is never + ** used. When columnType() is called on the expression + ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT + ** branch below. */ + break; + } + + assert( pTab && pExpr->y.pTab==pTab ); + if( pS ){ + /* The "table" is actually a sub-select or a view in the FROM clause + ** of the SELECT statement. Return the declaration type and origin + ** data for the result-set column of the sub-select. + */ + if( iCol>=0 && iColpEList->nExpr ){ + /* If iCol is less than zero, then the expression requests the + ** rowid of the sub-select or view. This expression is legal (see + ** test case misc2.2.2) - it always evaluates to NULL. + */ + NameContext sNC; + Expr *p = pS->pEList->a[iCol].pExpr; + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); + } + }else{ + /* A real table or a CTE table */ + assert( !pS ); +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zType = "INTEGER"; + zOrigCol = "rowid"; + }else{ + zOrigCol = pTab->aCol[iCol].zName; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); + } + zOrigTab = pTab->zName; + if( pNC->pParse && pTab->pSchema ){ + int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; + } +#else + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zType = "INTEGER"; + }else{ + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); + } +#endif + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: { + /* The expression is a sub-select. Return the declaration type and + ** origin info for the single column in the result set of the SELECT + ** statement. + */ + NameContext sNC; + Select *pS = pExpr->x.pSelect; + Expr *p = pS->pEList->a[0].pExpr; + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + break; + } +#endif + } + +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( pzOrigDb ){ + assert( pzOrigTab && pzOrigCol ); + *pzOrigDb = zOrigDb; + *pzOrigTab = zOrigTab; + *pzOrigCol = zOrigCol; + } +#endif + return zType; +} + +/* +** Generate code that will tell the VDBE the declaration types of columns +** in the result set. +*/ +static void generateColumnTypes( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ +#ifndef SQLITE_OMIT_DECLTYPE + Vdbe *v = pParse->pVdbe; + int i; + NameContext sNC; + sNC.pSrcList = pTabList; + sNC.pParse = pParse; + sNC.pNext = 0; + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + const char *zType; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + const char *zOrigDb = 0; + const char *zOrigTab = 0; + const char *zOrigCol = 0; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + + /* The vdbe must make its own copy of the column-type and other + ** column specific strings, in case the schema is reset before this + ** virtual machine is deleted. + */ + sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); +#else + zType = columnType(&sNC, p, 0, 0, 0); +#endif + sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); + } +#endif /* !defined(SQLITE_OMIT_DECLTYPE) */ +} + + +/* +** Compute the column names for a SELECT statement. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: sqlite3ColumnsFromExprList() +** +** The PRAGMA short_column_names and PRAGMA full_column_names settings are +** deprecated. The default setting is short=ON, full=OFF. 99.9% of all +** applications should operate this way. Nevertheless, we need to support the +** other modes for legacy: +** +** short=OFF, full=OFF: Column name is the text of the expression has it +** originally appears in the SELECT statement. In +** other words, the zSpan of the result expression. +** +** short=ON, full=OFF: (This is the default setting). If the result +** refers directly to a table column, then the +** result column name is just the table column +** name: COLUMN. Otherwise use zSpan. +** +** full=ON, short=ANY: If the result refers directly to a table column, +** then the result column name with the table name +** prefix, ex: TABLE.COLUMN. Otherwise use zSpan. +*/ +static void generateColumnNames( + Parse *pParse, /* Parser context */ + Select *pSelect /* Generate column names for this SELECT statement */ +){ + Vdbe *v = pParse->pVdbe; + int i; + Table *pTab; + SrcList *pTabList; + ExprList *pEList; + sqlite3 *db = pParse->db; + int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ + int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ + +#ifndef SQLITE_OMIT_EXPLAIN + /* If this is an EXPLAIN, skip this step */ + if( pParse->explain ){ + return; + } +#endif + + if( pParse->colNamesSet ) return; + /* Column names are determined by the left-most term of a compound select */ + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + pTabList = pSelect->pSrc; + pEList = pSelect->pEList; + assert( v!=0 ); + assert( pTabList!=0 ); + pParse->colNamesSet = 1; + fullName = (db->flags & SQLITE_FullColNames)!=0; + srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName; + sqlite3VdbeSetNumCols(v, pEList->nExpr); + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + + assert( p!=0 ); + assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ + assert( p->op!=TK_COLUMN || p->y.pTab!=0 ); /* Covering idx not yet coded */ + if( pEList->a[i].zEName && pEList->a[i].eEName==ENAME_NAME ){ + /* An AS clause always takes first priority */ + char *zName = pEList->a[i].zEName; + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); + }else if( srcName && p->op==TK_COLUMN ){ + char *zCol; + int iCol = p->iColumn; + pTab = p->y.pTab; + assert( pTab!=0 ); + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = pTab->aCol[iCol].zName; + } + if( fullName ){ + char *zName = 0; + zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); + }else{ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); + } + }else{ + const char *z = pEList->a[i].zEName; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); + } + } + generateColumnTypes(pParse, pTabList, pEList); +} + +/* +** Given an expression list (which is really the list of expressions +** that form the result set of a SELECT statement) compute appropriate +** column names for a table that would hold the expression list. +** +** All column names will be unique. +** +** Only the column names are computed. Column.zType, Column.zColl, +** and other fields of Column are zeroed. +** +** Return SQLITE_OK on success. If a memory allocation error occurs, +** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: generateColumnNames() +*/ +int sqlite3ColumnsFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* Expr list from which to derive column names */ + i16 *pnCol, /* Write the number of columns here */ + Column **paCol /* Write the new column list here */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + int i, j; /* Loop counters */ + u32 cnt; /* Index added to make the name unique */ + Column *aCol, *pCol; /* For looping over result columns */ + int nCol; /* Number of columns in the result set */ + char *zName; /* Column name */ + int nName; /* Size of name in zName[] */ + Hash ht; /* Hash table of column names */ + Table *pTab; + + sqlite3HashInit(&ht); + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + if( NEVER(nCol>32767) ) nCol = 32767; + }else{ + nCol = 0; + aCol = 0; + } + assert( nCol==(i16)nCol ); + *pnCol = nCol; + *paCol = aCol; + + for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + /* Get an appropriate name for the column + */ + if( (zName = pEList->a[i].zEName)!=0 && pEList->a[i].eEName==ENAME_NAME ){ + /* If the column contains an "AS " phrase, use as the name */ + }else{ + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr); + while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){ + pColExpr = pColExpr->pRight; + assert( pColExpr!=0 ); + } + if( pColExpr->op==TK_COLUMN && (pTab = pColExpr->y.pTab)!=0 ){ + /* For columns use the column name name */ + int iCol = pColExpr->iColumn; + if( iCol<0 ) iCol = pTab->iPKey; + zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; + }else if( pColExpr->op==TK_ID ){ + assert( !ExprHasProperty(pColExpr, EP_IntValue) ); + zName = pColExpr->u.zToken; + }else{ + /* Use the original text of the column expression as its name */ + zName = pEList->a[i].zEName; + } + } + if( zName && !sqlite3IsTrueOrFalse(zName) ){ + zName = sqlite3DbStrDup(db, zName); + }else{ + zName = sqlite3MPrintf(db,"column%d",i+1); + } + + /* Make sure the column name is unique. If the name is not unique, + ** append an integer to the name so that it becomes unique. + */ + cnt = 0; + while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + nName = sqlite3Strlen30(zName); + if( nName>0 ){ + for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} + if( zName[j]==':' ) nName = j; + } + zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); + if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + } + pCol->zName = zName; + pCol->hName = sqlite3StrIHash(zName); + sqlite3ColumnPropertiesFromName(0, pCol); + if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + sqlite3OomFault(db); + } + } + sqlite3HashClear(&ht); + if( db->mallocFailed ){ + for(j=0; jdb; + NameContext sNC; + Column *pCol; + CollSeq *pColl; + int i; + Expr *p; + struct ExprList_item *a; + + assert( pSelect!=0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); + assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); + if( db->mallocFailed ) return; + memset(&sNC, 0, sizeof(sNC)); + sNC.pSrcList = pSelect->pSrc; + a = pSelect->pEList->a; + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + const char *zType; + int n, m; + pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); + p = a[i].pExpr; + zType = columnType(&sNC, p, 0, 0, 0); + /* pCol->szEst = ... // Column size est for SELECT tables never used */ + pCol->affinity = sqlite3ExprAffinity(p); + if( zType ){ + m = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zName); + pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); + if( pCol->zName ){ + memcpy(&pCol->zName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } + if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; + pColl = sqlite3ExprCollSeq(pParse, p); + if( pColl && pCol->zColl==0 ){ + pCol->zColl = sqlite3DbStrDup(db, pColl->zName); + } + } + pTab->szTabRow = 1; /* Any non-zero value works */ +} + +/* +** Given a SELECT statement, generate a Table structure that describes +** the result set of that SELECT. +*/ +Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){ + Table *pTab; + sqlite3 *db = pParse->db; + u64 savedFlags; + + savedFlags = db->flags; + db->flags &= ~(u64)SQLITE_FullColNames; + db->flags |= SQLITE_ShortColNames; + sqlite3SelectPrep(pParse, pSelect, 0); + db->flags = savedFlags; + if( pParse->nErr ) return 0; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + pTab = sqlite3DbMallocZero(db, sizeof(Table) ); + if( pTab==0 ){ + return 0; + } + pTab->nTabRef = 1; + pTab->zName = 0; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); + pTab->iPKey = -1; + if( db->mallocFailed ){ + sqlite3DeleteTable(db, pTab); + return 0; + } + return pTab; +} + +/* +** Get a VDBE for the given parser context. Create a new one if necessary. +** If an error occurs, return NULL and leave a message in pParse. +*/ +Vdbe *sqlite3GetVdbe(Parse *pParse){ + if( pParse->pVdbe ){ + return pParse->pVdbe; + } + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; + } + return sqlite3VdbeCreate(pParse); +} + + +/* +** Compute the iLimit and iOffset fields of the SELECT based on the +** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions +** that appear in the original SQL statement after the LIMIT and OFFSET +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then +** iLimit and iOffset are negative. +** +** This routine changes the values of iLimit and iOffset only if +** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit +** and iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** +** Only if pLimit->pLeft!=0 do the limit registers get +** redefined. The UNION ALL operator uses this property to force +** the reuse of the same limit and offset registers across multiple +** SELECT statements. +*/ +static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ + Vdbe *v = 0; + int iLimit = 0; + int iOffset; + int n; + Expr *pLimit = p->pLimit; + + if( p->iLimit ) return; + + /* + ** "LIMIT -1" always shows all rows. There is some + ** controversy about what the correct behavior should be. + ** The current implementation interprets "LIMIT 0" to mean + ** no rows. + */ + if( pLimit ){ + assert( pLimit->op==TK_LIMIT ); + assert( pLimit->pLeft!=0 ); + p->iLimit = iLimit = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); + VdbeComment((v, "LIMIT counter")); + if( n==0 ){ + sqlite3VdbeGoto(v, iBreak); + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; + } + }else{ + sqlite3ExprCode(pParse, pLimit->pLeft, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); + } + if( pLimit->pRight ){ + p->iOffset = iOffset = ++pParse->nMem; + pParse->nMem++; /* Allocate an extra register for limit+offset */ + sqlite3ExprCode(pParse, pLimit->pRight, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); + VdbeComment((v, "OFFSET counter")); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); + VdbeComment((v, "LIMIT+OFFSET")); + } + } +} + +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Return the appropriate collating sequence for the iCol-th column of +** the result set for the compound-select statement "p". Return NULL if +** the column has no default collating sequence. +** +** The collating sequence for the compound select is taken from the +** left-most term of the select that has a collating sequence. +*/ +static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ + CollSeq *pRet; + if( p->pPrior ){ + pRet = multiSelectCollSeq(pParse, p->pPrior, iCol); + }else{ + pRet = 0; + } + assert( iCol>=0 ); + /* iCol must be less than p->pEList->nExpr. Otherwise an error would + ** have been thrown during name resolution and we would not have gotten + ** this far */ + if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ + pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); + } + return pRet; +} + +/* +** The select statement passed as the second parameter is a compound SELECT +** with an ORDER BY clause. This function allocates and returns a KeyInfo +** structure suitable for implementing the ORDER BY. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for ensuring that this structure is eventually +** freed. +*/ +static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ + ExprList *pOrderBy = p->pOrderBy; + int nOrderBy = p->pOrderBy->nExpr; + sqlite3 *db = pParse->db; + KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); + if( pRet ){ + int i; + for(i=0; ia[i]; + Expr *pTerm = pItem->pExpr; + CollSeq *pColl; + + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); + }else{ + pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1); + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); + } + assert( sqlite3KeyInfoIsWriteable(pRet) ); + pRet->aColl[i] = pColl; + pRet->aSortFlags[i] = pOrderBy->a[i].sortFlags; + } + } + + return pRet; +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + Select *pFirstRec; /* Left-most recursive term */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent = 0; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Fifo; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries"); + return; + } +#endif + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(pParse); + p->nSelectRow = 320; /* 4 billion rows */ + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = 0; + p->iLimit = p->iOffset = 0; + pOrderBy = p->pOrderBy; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].fg.isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Detach the ORDER BY clause from the compound SELECT */ + p->pOrderBy = 0; + + /* Figure out how many elements of the compound SELECT are part of the + ** recursive query. Make sure no recursive elements use aggregate + ** functions. Mark the recursive elements as UNION ALL even if they + ** are really UNION because the distinctness will be enforced by the + ** iDistinct table. pFirstRec is left pointing to the left-most + ** recursive term of the CTE. + */ + pFirstRec = p; + for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){ + if( pFirstRec->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + goto end_of_recursive_query; + } + pFirstRec->op = TK_ALL; + if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break; + } + + /* Store the results of the setup-query in Queue. */ + pSetup = pFirstRec->pPrior; + pSetup->pNext = 0; + ExplainQueryPlan((pParse, 1, "SETUP")); + rc = sqlite3Select(pParse, pSetup, &destQueue); + pSetup->pNext = p; + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(pParse); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak); + VdbeCoverage(v); + } + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + pFirstRec->pPrior = 0; + ExplainQueryPlan((pParse, 1, "RECURSIVE STEP")); + sqlite3Select(pParse, p, &destQueue); + assert( pFirstRec->pPrior==0 ); + pFirstRec->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeGoto(v, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + return; +} +#endif /* SQLITE_OMIT_CTE */ + +/* Forward references */ +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +); + +/* +** Handle the special case of a compound-select that originates from a +** VALUES clause. By handling this as a special case, we avoid deep +** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT +** on a VALUES clause. +** +** Because the Select object originates from a VALUES clause: +** (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1 +** (2) All terms are UNION ALL +** (3) There is no ORDER BY clause +** +** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES +** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))"). +** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case. +** Since the limit is exactly 1, we only need to evalutes the left-most VALUES. +*/ +static int multiSelectValues( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int nRow = 1; + int rc = 0; + int bShowAll = p->pLimit==0; + assert( p->selFlags & SF_MultiValue ); + do{ + assert( p->selFlags & SF_Values ); + assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); + assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) return -1; +#endif + if( p->pPrior==0 ) break; + assert( p->pPrior->pNext==p ); + p = p->pPrior; + nRow += bShowAll; + }while(1); + ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow, + nRow==1 ? "" : "S")); + while( p ){ + selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1); + if( !bShowAll ) break; + p->nSelectRow = nRow; + p = p->pNext; + } + return rc; +} + +/* +** Return true if the SELECT statement which is known to be the recursive +** part of a recursive CTE still has its anchor terms attached. If the +** anchor terms have already been removed, then return false. +*/ +static int hasAnchor(Select *p){ + while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; } + return p!=0; +} + +/* +** This routine is called to process a compound query form from +** two or more separate queries using UNION, UNION ALL, EXCEPT, or +** INTERSECT +** +** "p" points to the right-most of the two queries. the query on the +** left is p->pPrior. The left query could also be a compound query +** in which case this routine will be called recursively. +** +** The results of the total query are to be written into a destination +** of type eDest with parameter iParm. +** +** Example 1: Consider a three-way compound SQL statement. +** +** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3 +** +** This statement is parsed up as follows: +** +** SELECT c FROM t3 +** | +** `-----> SELECT b FROM t2 +** | +** `------> SELECT a FROM t1 +** +** The arrows in the diagram above represent the Select.pPrior pointer. +** So if this routine is called with p equal to the t3 query, then +** pPrior will be the t2 query. p->op will be TK_UNION in this case. +** +** Notice that because of the way SQLite parses compound SELECTs, the +** individual selects always group from left to right. +*/ +static int multiSelect( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int rc = SQLITE_OK; /* Success code from a subroutine */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest dest; /* Alternative data destination */ + Select *pDelete = 0; /* Chain of simple selects to delete */ + sqlite3 *db; /* Database connection */ + + /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only + ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. + */ + assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( p->selFlags & SF_Compound ); + db = pParse->db; + pPrior = p->pPrior; + dest = *pDest; + assert( pPrior->pOrderBy==0 ); + assert( pPrior->pLimit==0 ); + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); /* The VDBE already created by calling function */ + + /* Create the destination temporary table if necessary + */ + if( dest.eDest==SRT_EphemTab ){ + assert( p->pEList ); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); + dest.eDest = SRT_Table; + } + + /* Special handling for a compound-select that originates as a VALUES clause. + */ + if( p->selFlags & SF_MultiValue ){ + rc = multiSelectValues(pParse, p, &dest); + if( rc>=0 ) goto multi_select_end; + rc = SQLITE_OK; + } + + /* Make sure all SELECTs in the statement have the same number of elements + ** in their result sets. + */ + assert( p->pEList && pPrior->pEList ); + assert( p->pEList->nExpr==pPrior->pEList->nExpr ); + +#ifndef SQLITE_OMIT_CTE + if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){ + generateWithRecursiveQuery(pParse, p, &dest); + }else +#endif + + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); + }else{ + +#ifndef SQLITE_OMIT_EXPLAIN + if( pPrior->pPrior==0 ){ + ExplainQueryPlan((pParse, 1, "COMPOUND QUERY")); + ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); + } +#endif + + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit; + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = p->pLimit; + rc = sqlite3Select(pParse, pPrior, &dest); + pPrior->pLimit = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temp table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit; /* Saved values of p->nLimit */ + int addr; + SelectDest uniondest; + + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iSDParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + uniondest.eDest = op; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); + assert( p->pOrderBy==0 ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->iLimit = 0; + p->iOffset = 0; + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); + assert( p->pEList || db->mallocFailed ); + if( dest.eDest!=priorOp && db->mallocFailed==0 ){ + int iCont, iBreak, iStart; + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, unionTab, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit; + int addr; + SelectDest intersectdest; + int r1; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + intersectdest.iSDParm = tab2; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ){ + p->nSelectRow = pPrior->nSelectRow; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + if( rc ) break; + assert( p->pEList ); + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, tab1, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; + } + } + + #ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); + } + #endif + } + if( pParse->nErr ) goto multi_select_end; + + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. + ** + ** This section is run by the right-most SELECT statement only. + ** SELECT statements to the left always skip this part. The right-most + ** SELECT might also skip this part if it has no ORDER BY clause and + ** no temp tables are required. + */ + if( p->selFlags & SF_UsesEphemeral ){ + int i; /* Loop counter */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ + + assert( p->pNext==0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); + if( !pKeyInfo ){ + rc = SQLITE_NOMEM_BKPT; + goto multi_select_end; + } + for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; + } + } + + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenEphm[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenEphm[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), + P4_KEYINFO); + pLoop->addrOpenEphm[i] = -1; + } + } + sqlite3KeyInfoUnref(pKeyInfo); + } + +multi_select_end: + pDest->iSdst = dest.iSdst; + pDest->nSdst = dest.nSdst; + sqlite3SelectDelete(db, pDelete); + return rc; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +/* +** Error message for when two or more terms of a compound select have different +** size result sets. +*/ +void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", + sqlite3SelectOpName(p->op)); + } +} + +/* +** Code an output subroutine for a coroutine implementation of a +** SELECT statment. +** +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should +** be sent. +** +** regReturn is the number of the register holding the subroutine +** return address. +** +** If regPrev>0 then it is the first register in a vector that +** records the previous output. mem[regPrev] is a flag that is false +** if there has been no previous output. If regPrev>0 then code is +** generated to suppress duplicates. pKeyInfo is used for comparing +** keys. +** +** If the LIMIT found in p->iLimit is reached, jump immediately to +** iBreak. +*/ +static int generateOutputSubroutine( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SelectDest *pIn, /* Coroutine supplying data */ + SelectDest *pDest, /* Where to send the data */ + int regReturn, /* The return address register */ + int regPrev, /* Previous result register. No uniqueness if 0 */ + KeyInfo *pKeyInfo, /* For comparing with previous entry */ + int iBreak /* Jump here if we hit the LIMIT */ +){ + Vdbe *v = pParse->pVdbe; + int iContinue; + int addr; + + addr = sqlite3VdbeCurrentAddr(v); + iContinue = sqlite3VdbeMakeLabel(pParse); + + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + */ + if( regPrev ){ + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); + } + if( pParse->db->mallocFailed ) return 0; + + /* Suppress the first OFFSET entries if there is an OFFSET clause + */ + codeOffset(v, p->iOffset, iContinue); + + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); + switch( pDest->eDest ){ + /* Store the result as data using a unique key. + */ + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)". + */ + case SRT_Set: { + int r1; + testcase( pIn->nSdst>1 ); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, + pIn->iSdst, pIn->nSdst); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. Note that the select might return multiple columns + ** if it is the RHS of a row-value IN operator. + */ + case SRT_Mem: { + testcase( pIn->nSdst>1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst); + /* The LIMIT clause will jump out of the loop for us */ + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + /* The results are stored in a sequence of registers + ** starting at pDest->iSdst. Then the co-routine yields. + */ + case SRT_Coroutine: { + if( pDest->iSdst==0 ){ + pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); + pDest->nSdst = pIn->nSdst; + } + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + break; + } + + /* If none of the above, then the result destination must be + ** SRT_Output. This routine is never called with any other + ** destination other than the ones handled above or SRT_Output. + ** + ** For SRT_Output, results are stored in a sequence of registers. + ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to + ** return the next row of result. + */ + default: { + assert( pDest->eDest==SRT_Output ); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); + break; + } + } + + /* Jump to the end of the loop if the LIMIT is reached. + */ + if( p->iLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); + } + + /* Generate the subroutine return + */ + sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp1(v, OP_Return, regReturn); + + return addr; +} + +/* +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. +** +** We assume a query of the following form: +** +** ORDER BY +** +** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea +** is to code both and with the ORDER BY clause as +** co-routines. Then run the co-routines in parallel and merge the results +** into the output. In addition to the two coroutines (called selectA and +** selectB) there are 7 subroutines: +** +** outA: Move the output of the selectA coroutine into the output +** of the compound query. +** +** outB: Move the output of the selectB coroutine into the output +** of the compound query. (Only generated for UNION and +** UNION ALL. EXCEPT and INSERTSECT never output a row that +** appears only in B.) +** +** AltB: Called when there is data from both coroutines and AB. +** +** EofA: Called when data is exhausted from selectA. +** +** EofB: Called when data is exhausted from selectB. +** +** The implementation of the latter five subroutines depend on which +** is used: +** +** +** UNION ALL UNION EXCEPT INTERSECT +** ------------- ----------------- -------------- ----------------- +** AltB: outA, nextA outA, nextA outA, nextA nextA +** +** AeqB: outA, nextA nextA nextA outA, nextA +** +** AgtB: outB, nextB outB, nextB nextB nextB +** +** EofA: outB, nextB outB, nextB halt halt +** +** EofB: outA, nextA outA, nextA outA, nextA halt +** +** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA +** causes an immediate jump to EofA and an EOF on B following nextB causes +** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or +** following nextX causes a jump to the end of the select processing. +** +** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled +** within the output subroutine. The regPrev register set holds the previously +** output value. A comparison is made against this value and the output +** is skipped if the next results would be the same as the previous. +** +** The implementation plan is to implement the two coroutines and seven +** subroutines first, then put the control logic at the bottom. Like this: +** +** goto Init +** coA: coroutine for left query (A) +** coB: coroutine for right query (B) +** outA: output one row of A +** outB: output one row of B (UNION and UNION ALL only) +** EofA: ... +** EofB: ... +** AltB: ... +** AeqB: ... +** AgtB: ... +** Init: initialize coroutine registers +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB +** Cmpr: Compare A, B +** Jump AltB, AeqB, AgtB +** End: ... +** +** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not +** actually called using Gosub and they do not Return. EofA and EofB loop +** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** and AgtB jump to either L2 or to one of EofA or EofB. +*/ +#ifndef SQLITE_OMIT_COMPOUND_SELECT +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int i, j; /* Loop counters */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest destA; /* Destination for coroutine A */ + SelectDest destB; /* Destination for coroutine B */ + int regAddrA; /* Address register for select-A coroutine */ + int regAddrB; /* Address register for select-B coroutine */ + int addrSelectA; /* Address of the select-A coroutine */ + int addrSelectB; /* Address of the select-B coroutine */ + int regOutA; /* Address register for the output-A subroutine */ + int regOutB; /* Address register for the output-B subroutine */ + int addrOutA; /* Address of the output-A subroutine */ + int addrOutB = 0; /* Address of the output-B subroutine */ + int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */ + int addrEofB; /* Address of the select-B-exhausted subroutine */ + int addrAltB; /* Address of the AB subroutine */ + int regLimitA; /* Limit register for select-A */ + int regLimitB; /* Limit register for select-A */ + int regPrev; /* A range of registers to hold previous output */ + int savedLimit; /* Saved value of p->iLimit */ + int savedOffset; /* Saved value of p->iOffset */ + int labelCmpr; /* Label for the start of the merge algorithm */ + int labelEnd; /* Label for the end of the overall SELECT stmt */ + int addr1; /* Jump instructions that get retargetted */ + int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ + KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ + KeyInfo *pKeyMerge; /* Comparison information for merging rows */ + sqlite3 *db; /* Database connection */ + ExprList *pOrderBy; /* The ORDER BY clause */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + u32 *aPermute; /* Mapping from ORDER BY terms to result set columns */ + + assert( p->pOrderBy!=0 ); + assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ + labelEnd = sqlite3VdbeMakeLabel(pParse); + labelCmpr = sqlite3VdbeMakeLabel(pParse); + + + /* Patch up the ORDER BY clause + */ + op = p->op; + pPrior = p->pPrior; + assert( pPrior->pOrderBy==0 ); + pOrderBy = p->pOrderBy; + assert( pOrderBy ); + nOrderBy = pOrderBy->nExpr; + + /* For operators other than UNION ALL we have to make sure that + ** the ORDER BY clause covers every term of the result set. Add + ** terms to the ORDER BY clause as necessary. + */ + if( op!=TK_ALL ){ + for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ + struct ExprList_item *pItem; + for(j=0, pItem=pOrderBy->a; ju.x.iOrderByCol>0 ); + if( pItem->u.x.iOrderByCol==i ) break; + } + if( j==nOrderBy ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return SQLITE_NOMEM_BKPT; + pNew->flags |= EP_IntValue; + pNew->u.iValue = i; + p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); + if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; + } + } + } + + /* Compute the comparison permutation and keyinfo that is used with + ** the permutation used to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. + */ + aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); + if( aPermute ){ + struct ExprList_item *pItem; + aPermute[0] = nOrderBy; + for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ + assert( pItem->u.x.iOrderByCol>0 ); + assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + aPermute[i] = pItem->u.x.iOrderByCol - 1; + } + pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); + }else{ + pKeyMerge = 0; + } + + /* Reattach the ORDER BY clause to the query. + */ + p->pOrderBy = pOrderBy; + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); + + /* Allocate a range of temporary registers and the KeyInfo needed + ** for the logic that removes duplicate result rows when the + ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). + */ + if( op==TK_ALL ){ + regPrev = 0; + }else{ + int nExpr = p->pEList->nExpr; + assert( nOrderBy>=nExpr || db->mallocFailed ); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); + pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1); + if( pKeyDup ){ + assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); + for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); + pKeyDup->aSortFlags[i] = 0; + } + } + } + + /* Separate the left and the right query from one another + */ + p->pPrior = 0; + pPrior->pNext = 0; + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + if( pPrior->pPrior==0 ){ + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); + } + + /* Compute the limit registers */ + computeLimitRegisters(pParse, p, labelEnd); + if( p->iLimit && op==TK_ALL ){ + regLimitA = ++pParse->nMem; + regLimitB = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, + regLimitA); + sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); + }else{ + regLimitA = regLimitB = 0; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = 0; + + regAddrA = ++pParse->nMem; + regAddrB = ++pParse->nMem; + regOutA = ++pParse->nMem; + regOutB = ++pParse->nMem; + sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); + sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); + + ExplainQueryPlan((pParse, 1, "MERGE (%s)", sqlite3SelectOpName(p->op))); + + /* Generate a coroutine to evaluate the SELECT statement to the + ** left of the compound operator - the "A" select. + */ + addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + VdbeComment((v, "left SELECT")); + pPrior->iLimit = regLimitA; + ExplainQueryPlan((pParse, 1, "LEFT")); + sqlite3Select(pParse, pPrior, &destA); + sqlite3VdbeEndCoroutine(v, regAddrA); + sqlite3VdbeJumpHere(v, addr1); + + /* Generate a coroutine to evaluate the SELECT statement on + ** the right - the "B" select + */ + addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + VdbeComment((v, "right SELECT")); + savedLimit = p->iLimit; + savedOffset = p->iOffset; + p->iLimit = regLimitB; + p->iOffset = 0; + ExplainQueryPlan((pParse, 1, "RIGHT")); + sqlite3Select(pParse, p, &destB); + p->iLimit = savedLimit; + p->iOffset = savedOffset; + sqlite3VdbeEndCoroutine(v, regAddrB); + + /* Generate a subroutine that outputs the current row of the A + ** select as the next output row of the compound select. + */ + VdbeNoopComment((v, "Output routine for A")); + addrOutA = generateOutputSubroutine(pParse, + p, &destA, pDest, regOutA, + regPrev, pKeyDup, labelEnd); + + /* Generate a subroutine that outputs the current row of the B + ** select as the next output row of the compound select. + */ + if( op==TK_ALL || op==TK_UNION ){ + VdbeNoopComment((v, "Output routine for B")); + addrOutB = generateOutputSubroutine(pParse, + p, &destB, pDest, regOutB, + regPrev, pKeyDup, labelEnd); + } + sqlite3KeyInfoUnref(pKeyDup); + + /* Generate a subroutine to run when the results from select A + ** are exhausted and only data in select B remains. + */ + if( op==TK_EXCEPT || op==TK_INTERSECT ){ + addrEofA_noB = addrEofA = labelEnd; + }else{ + VdbeNoopComment((v, "eof-A subroutine")); + addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); + VdbeCoverage(v); + sqlite3VdbeGoto(v, addrEofA); + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + + /* Generate a subroutine to run when the results from select B + ** are exhausted and only data in select A remains. + */ + if( op==TK_INTERSECT ){ + addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + }else{ + VdbeNoopComment((v, "eof-B subroutine")); + addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + sqlite3VdbeGoto(v, addrEofB); + } + + /* Generate code to handle the case of AB + */ + VdbeNoopComment((v, "A-gt-B subroutine")); + addrAgtB = sqlite3VdbeCurrentAddr(v); + if( op==TK_ALL || op==TK_UNION ){ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + } + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + sqlite3VdbeGoto(v, labelCmpr); + + /* This code runs once to initialize everything. + */ + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + + /* Implement the main merge loop + */ + sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + + /* Jump to the this point in order to terminate the query. + */ + sqlite3VdbeResolveLabel(v, labelEnd); + + /* Reassembly the compound query so that it will be freed correctly + ** by the calling function */ + if( p->pPrior ){ + sqlite3SelectDelete(db, p->pPrior); + } + p->pPrior = pPrior; + pPrior->pNext = p; + + /*** TBD: Insert subroutine calls to close cursors on incomplete + **** subqueries ****/ + ExplainQueryPlanPop(pParse); + return pParse->nErr!=0; +} +#endif + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + +/* An instance of the SubstContext object describes an substitution edit +** to be performed on a parse tree. +** +** All references to columns in table iTable are to be replaced by corresponding +** expressions in pEList. +*/ +typedef struct SubstContext { + Parse *pParse; /* The parsing context */ + int iTable; /* Replace references to this table */ + int iNewTable; /* New table number */ + int isLeftJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + ExprList *pEList; /* Replacement expressions */ +} SubstContext; + +/* Forward Declarations */ +static void substExprList(SubstContext*, ExprList*); +static void substSelect(SubstContext*, Select*, int); + +/* +** Scan through the expression pExpr. Replace every reference to +** a column in table number iTable with a copy of the iColumn-th +** entry in pEList. (But leave references to the ROWID column +** unchanged.) +** +** This routine is part of the flattening procedure. A subquery +** whose result set is defined by pEList appears as entry in the +** FROM clause of a SELECT such that the VDBE cursor assigned to that +** FORM clause entry is iTable. This routine makes the necessary +** changes to pExpr so that it refers directly to the source table +** of the subquery rather the result set of the subquery. +*/ +static Expr *substExpr( + SubstContext *pSubst, /* Description of the substitution */ + Expr *pExpr /* Expr in which substitution occurs */ +){ + if( pExpr==0 ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) + && pExpr->iRightJoinTable==pSubst->iTable + ){ + pExpr->iRightJoinTable = pSubst->iNewTable; + } + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pSubst->iTable + && !ExprHasProperty(pExpr, EP_FixedCol) + ){ + if( pExpr->iColumn<0 ){ + pExpr->op = TK_NULL; + }else{ + Expr *pNew; + Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + Expr ifNullRow; + assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + assert( pExpr->pRight==0 ); + if( sqlite3ExprIsVector(pCopy) ){ + sqlite3VectorErrorMsg(pSubst->pParse, pCopy); + }else{ + sqlite3 *db = pSubst->pParse->db; + if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){ + memset(&ifNullRow, 0, sizeof(ifNullRow)); + ifNullRow.op = TK_IF_NULL_ROW; + ifNullRow.pLeft = pCopy; + ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.flags = EP_IfNullRow; + pCopy = &ifNullRow; + } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); + pNew = sqlite3ExprDup(db, pCopy, 0); + if( pNew && pSubst->isLeftJoin ){ + ExprSetProperty(pNew, EP_CanBeNull); + } + if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){ + sqlite3SetJoinExpr(pNew, pExpr->iRightJoinTable); + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; + + /* Ensure that the expression now has an implicit collation sequence, + ** just as it did when it was a column of a view or sub-query. */ + if( pExpr ){ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } + ExprClearProperty(pExpr, EP_Collate); + } + } + } + }else{ + if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ + pExpr->iTable = pSubst->iNewTable; + } + pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); + pExpr->pRight = substExpr(pSubst, pExpr->pRight); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + substSelect(pSubst, pExpr->x.pSelect, 1); + }else{ + substExprList(pSubst, pExpr->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + pWin->pFilter = substExpr(pSubst, pWin->pFilter); + substExprList(pSubst, pWin->pPartition); + substExprList(pSubst, pWin->pOrderBy); + } +#endif + } + return pExpr; +} +static void substExprList( + SubstContext *pSubst, /* Description of the substitution */ + ExprList *pList /* List to scan and in which to make substitutes */ +){ + int i; + if( pList==0 ) return; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr); + } +} +static void substSelect( + SubstContext *pSubst, /* Description of the substitution */ + Select *p, /* SELECT statement in which to make substitutions */ + int doPrior /* Do substitutes on p->pPrior too */ +){ + SrcList *pSrc; + SrcItem *pItem; + int i; + if( !p ) return; + do{ + substExprList(pSubst, p->pEList); + substExprList(pSubst, p->pGroupBy); + substExprList(pSubst, p->pOrderBy); + p->pHaving = substExpr(pSubst, p->pHaving); + p->pWhere = substExpr(pSubst, p->pWhere); + pSrc = p->pSrc; + assert( pSrc!=0 ); + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(pSubst, pItem->pSelect, 1); + if( pItem->fg.isTabFunc ){ + substExprList(pSubst, pItem->u1.pFuncArg); + } + } + }while( doPrior && (p = p->pPrior)!=0 ); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** pSelect is a SELECT statement and pSrcItem is one item in the FROM +** clause of that SELECT. +** +** This routine scans the entire SELECT statement and recomputes the +** pSrcItem->colUsed mask. +*/ +static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){ + SrcItem *pItem; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + pItem = pWalker->u.pSrcItem; + if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue; + if( pExpr->iColumn<0 ) return WRC_Continue; + pItem->colUsed |= sqlite3ExprColUsed(pExpr); + return WRC_Continue; +} +static void recomputeColumnsUsed( + Select *pSelect, /* The complete SELECT statement */ + SrcItem *pSrcItem /* Which FROM clause item to recompute */ +){ + Walker w; + if( NEVER(pSrcItem->pTab==0) ) return; + memset(&w, 0, sizeof(w)); + w.xExprCallback = recomputeColumnsUsedExpr; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pSrcItem = pSrcItem; + pSrcItem->colUsed = 0; + sqlite3WalkSelect(&w, pSelect); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Assign new cursor numbers to each of the items in pSrc. For each +** new cursor number assigned, set an entry in the aCsrMap[] array +** to map the old cursor number to the new: +** +** aCsrMap[iOld] = iNew; +** +** The array is guaranteed by the caller to be large enough for all +** existing cursor numbers in pSrc. +** +** If pSrc contains any sub-selects, call this routine recursively +** on the FROM clause of each such sub-select, with iExcept set to -1. +*/ +static void srclistRenumberCursors( + Parse *pParse, /* Parse context */ + int *aCsrMap, /* Array to store cursor mappings in */ + SrcList *pSrc, /* FROM clause to renumber */ + int iExcept /* FROM clause item to skip */ +){ + int i; + SrcItem *pItem; + for(i=0, pItem=pSrc->a; inSrc; i++, pItem++){ + if( i!=iExcept ){ + Select *p; + pItem->iCursor = aCsrMap[pItem->iCursor] = pParse->nTab++; + for(p=pItem->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } + } + } +} + +/* +** Expression walker callback used by renumberCursors() to update +** Expr objects to match newly assigned cursor numbers. +*/ +static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ + int *aCsrMap = pWalker->u.aiCol; + int op = pExpr->op; + if( (op==TK_COLUMN || op==TK_IF_NULL_ROW) && aCsrMap[pExpr->iTable] ){ + pExpr->iTable = aCsrMap[pExpr->iTable]; + } + if( ExprHasProperty(pExpr, EP_FromJoin) && aCsrMap[pExpr->iRightJoinTable] ){ + pExpr->iRightJoinTable = aCsrMap[pExpr->iRightJoinTable]; + } + return WRC_Continue; +} + +/* +** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc) +** of the SELECT statement passed as the second argument, and to each +** cursor in the FROM clause of any FROM clause sub-selects, recursively. +** Except, do not assign a new cursor number to the iExcept'th element in +** the FROM clause of (*p). Update all expressions and other references +** to refer to the new cursor numbers. +** +** Argument aCsrMap is an array that may be used for temporary working +** space. Two guarantees are made by the caller: +** +** * the array is larger than the largest cursor number used within the +** select statement passed as an argument, and +** +** * the array entries for all cursor numbers that do *not* appear in +** FROM clauses of the select statement as described above are +** initialized to zero. +*/ +static void renumberCursors( + Parse *pParse, /* Parse context */ + Select *p, /* Select to renumber cursors within */ + int iExcept, /* FROM clause item to skip */ + int *aCsrMap /* Working space */ +){ + Walker w; + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, iExcept); + memset(&w, 0, sizeof(w)); + w.u.aiCol = aCsrMap; + w.xExprCallback = renumberCursorsCb; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkSelect(&w, p); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** This routine attempts to flatten subqueries as a performance optimization. +** This routine returns 1 if it makes changes and 0 if no flattening occurs. +** +** To understand the concept of flattening, consider the following +** query: +** +** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5 +** +** The default way of implementing this query is to execute the +** subquery first and store the results in a temporary table, then +** run the outer query on that temporary table. This requires two +** passes over the data. Furthermore, because the temporary table +** has no indices, the WHERE clause on the outer query cannot be +** optimized. +** +** This routine attempts to rewrite queries such as the above into +** a single flat select, like this: +** +** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 +** +** The code generated for this simplification gives the same result +** but only has to scan the data once. And because indices might +** exist on the table t1, a complete scan of the data might be +** avoided. +** +** Flattening is subject to the following constraints: +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery and the outer query cannot both be aggregates. +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** (2) If the subquery is an aggregate then +** (2a) the outer query must not be a join and +** (2b) the outer query must not use subqueries +** other than the one FROM-clause subquery that is a candidate +** for flattening. (This is due to ticket [2f7170d73bf9abf80] +** from 2015-02-09.) +** +** (3) If the subquery is the right operand of a LEFT JOIN then +** (3a) the subquery may not be a join and +** (3b) the FROM clause of the subquery may not contain a virtual +** table and +** (3c) the outer query may not be an aggregate. +** (3d) the outer query may not be DISTINCT. +** +** (4) The subquery can not be DISTINCT. +** +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** If the subquery is aggregate, the outer query may not be DISTINCT. +** +** (7) The subquery must have a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM clause with the special +** table sqlite_once that consists of a single row containing a +** single NULL. +** +** (8) If the subquery uses LIMIT then the outer query may not be a join. +** +** (9) If the subquery uses LIMIT then the outer query may not be aggregate. +** +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** constraint: "If the subquery is aggregate then the outer query +** may not use LIMIT." +** +** (11) The subquery and the outer query may not both have ORDER BY clauses. +** +** (**) Not implemented. Subsumed into restriction (3). Was previously +** a separate restriction deriving from ticket #350. +** +** (13) The subquery and outer query may not both use LIMIT. +** +** (14) The subquery may not use OFFSET. +** +** (15) If the outer query is part of a compound select, then the +** subquery may not use LIMIT. +** (See ticket #2339 and ticket [02a8e81d44]). +** +** (16) If the outer query is aggregate, then the subquery may not +** use ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. +** +** (17) If the subquery is a compound select, then +** (17a) all compound operators must be a UNION ALL, and +** (17b) no terms within the subquery compound may be aggregate +** or DISTINCT, and +** (17c) every term within the subquery compound must have a FROM clause +** (17d) the outer query may not be +** (17d1) aggregate, or +** (17d2) DISTINCT +** (17e) the subquery may not contain window functions, and +** (17f) the subquery must not be the RHS of a LEFT JOIN. +** +** The parent and sub-query may contain WHERE clauses. Subject to +** rules (11), (13) and (14), they may also contain ORDER BY, +** LIMIT and OFFSET clauses. The subquery cannot use any compound +** operator other than UNION ALL because all the other compound +** operators have an implied DISTINCT which is disallowed by +** restriction (4). +** +** Also, each component of the sub-query must return the same number +** of result columns. This is actually a requirement for any compound +** SELECT statement, but all the code here does is make sure that no +** such (illegal) sub-query is flattened. The caller will detect the +** syntax error and return a detailed message. +** +** (18) If the sub-query is a compound select, then all terms of the +** ORDER BY clause of the parent must be copies of a term returned +** by the parent query. +** +** (19) If the subquery uses LIMIT then the outer query may not +** have a WHERE clause. +** +** (20) If the sub-query is a compound select, then it must not use +** an ORDER BY clause. Ticket #3773. We could relax this constraint +** somewhat by saying that the terms of the ORDER BY clause must +** appear as unmodified result columns in the outer query. But we +** have other optimizations in mind to deal with that case. +** +** (21) If the subquery uses LIMIT then the outer query may not be +** DISTINCT. (See ticket [752e1646fc]). +** +** (22) The subquery may not be a recursive CTE. +** +** (23) If the outer query is a recursive CTE, then the sub-query may not be +** a compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery may not be an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** +** (25) If either the subquery or the parent query contains a window +** function in the select list or ORDER BY clause, flattening +** is not attempted. +** +** +** In this routine, the "p" parameter is a pointer to the outer query. +** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query +** uses aggregates. +** +** If flattening is not attempted, this routine is a no-op and returns 0. +** If flattening is attempted this routine returns 1. +** +** All of the expression analysis must occur on both the outer query and +** the subquery before this routine runs. +*/ +static int flattenSubquery( + Parse *pParse, /* Parsing context */ + Select *p, /* The parent or outer SELECT statement */ + int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ + int isAgg /* True if outer SELECT uses aggregate functions */ +){ + const char *zSavedAuthContext = pParse->zAuthContext; + Select *pParent; /* Current UNION ALL term of the other query */ + Select *pSub; /* The inner query or "subquery" */ + Select *pSub1; /* Pointer to the rightmost select in sub-query */ + SrcList *pSrc; /* The FROM clause of the outer query */ + SrcList *pSubSrc; /* The FROM clause of the subquery */ + int iParent; /* VDBE cursor number of the pSub result set temp table */ + int iNewParent = -1;/* Replacement table for iParent */ + int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ + int i; /* Loop counter */ + Expr *pWhere; /* The WHERE clause */ + SrcItem *pSubitem; /* The subquery */ + sqlite3 *db = pParse->db; + Walker w; /* Walker to persist agginfo data */ + int *aCsrMap = 0; + + /* Check to see if flattening is permitted. Return 0 if not. + */ + assert( p!=0 ); + assert( p->pPrior==0 ); + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; + pSrc = p->pSrc; + assert( pSrc && iFrom>=0 && iFromnSrc ); + pSubitem = &pSrc->a[iFrom]; + iParent = pSubitem->iCursor; + pSub = pSubitem->pSelect; + assert( pSub!=0 ); + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */ +#endif + + pSubSrc = pSub->pSrc; + assert( pSubSrc ); + /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET + ** because they could be computed at compile-time. But when LIMIT and OFFSET + ** became arbitrary expressions, we were forced to add restrictions (13) + ** and (14). */ + if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ + if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */ + if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ + return 0; /* Restriction (15) */ + } + if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ + } + if( p->pOrderBy && pSub->pOrderBy ){ + return 0; /* Restriction (11) */ + } + if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ + if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ + if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ + return 0; /* Restriction (21) */ + } + if( pSub->selFlags & (SF_Recursive) ){ + return 0; /* Restrictions (22) */ + } + + /* + ** If the subquery is the right operand of a LEFT JOIN, then the + ** subquery may not be a join itself (3a). Example of why this is not + ** allowed: + ** + ** t1 LEFT OUTER JOIN (t2 JOIN t3) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) JOIN t3 + ** + ** which is not at all the same thing. + ** + ** If the subquery is the right operand of a LEFT JOIN, then the outer + ** query cannot be an aggregate. (3c) This is an artifact of the way + ** aggregates are processed - there is no mechanism to determine if + ** the LEFT JOIN table should be all-NULL. + ** + ** See also tickets #306, #350, and #3300. + */ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ + isLeftJoin = 1; + if( pSubSrc->nSrc>1 /* (3a) */ + || isAgg /* (3b) */ + || IsVirtual(pSubSrc->a[0].pTab) /* (3c) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + ){ + return 0; + } + } +#ifdef SQLITE_EXTRA_IFNULLROW + else if( iFrom>0 && !isAgg ){ + /* Setting isLeftJoin to -1 causes OP_IfNullRow opcodes to be generated for + ** every reference to any result column from subquery in a join, even + ** though they are not necessary. This will stress-test the OP_IfNullRow + ** opcode. */ + isLeftJoin = -1; + } +#endif + + /* Restriction (17): If the sub-query is a compound SELECT, then it must + ** use only the UNION ALL operator. And none of the simple select queries + ** that make up the compound SELECT are allowed to be aggregate or distinct + ** queries. + */ + if( pSub->pPrior ){ + if( pSub->pOrderBy ){ + return 0; /* Restriction (20) */ + } + if( isAgg || (p->selFlags & SF_Distinct)!=0 || isLeftJoin>0 ){ + return 0; /* (17d1), (17d2), or (17f) */ + } + for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + assert( pSub->pSrc!=0 ); + assert( (pSub->selFlags & SF_Recursive)==0 ); + assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */ + || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */ + || pSub1->pSrc->nSrc<1 /* (17c) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + || pSub1->pWin /* (17e) */ +#endif + ){ + return 0; + } + testcase( pSub1->pSrc->nSrc>1 ); + } + + /* Restriction (18). */ + if( p->pOrderBy ){ + int ii; + for(ii=0; iipOrderBy->nExpr; ii++){ + if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; + } + } + + /* Restriction (23) */ + if( (p->selFlags & SF_Recursive) ) return 0; + + if( pSrc->nSrc>1 ){ + if( pParse->nSelect>500 ) return 0; + aCsrMap = sqlite3DbMallocZero(db, pParse->nTab*sizeof(int)); + } + } + + /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + pSub->selId, pSub, iFrom)); + + /* Authorize the subquery */ + pParse->zAuthContext = pSubitem->zName; + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); + pParse->zAuthContext = zSavedAuthContext; + + /* Delete the transient structures associated with thesubquery */ + pSub1 = pSubitem->pSelect; + sqlite3DbFree(db, pSubitem->zDatabase); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zDatabase = 0; + pSubitem->zName = 0; + pSubitem->zAlias = 0; + pSubitem->pSelect = 0; + assert( pSubitem->pOn==0 ); + + /* If the sub-query is a compound SELECT statement, then (by restrictions + ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** be of the form: + ** + ** SELECT FROM () + ** + ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** OFFSET clauses and joins them to the left-hand-side of the original + ** using UNION ALL operators. In this case N is the number of simple + ** select statements in the compound sub-query. + ** + ** Example: + ** + ** SELECT a+1 FROM ( + ** SELECT x FROM tab + ** UNION ALL + ** SELECT y FROM tab + ** UNION ALL + ** SELECT abs(z*2) FROM tab2 + ** ) WHERE a!=5 ORDER BY 1 + ** + ** Transformed into: + ** + ** SELECT x+1 FROM tab WHERE x+1!=5 + ** UNION ALL + ** SELECT y+1 FROM tab WHERE y+1!=5 + ** UNION ALL + ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 + ** ORDER BY 1 + ** + ** We call this the "compound-subquery flattening". + */ + for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ + Select *pNew; + ExprList *pOrderBy = p->pOrderBy; + Expr *pLimit = p->pLimit; + Select *pPrior = p->pPrior; + Table *pItemTab = pSubitem->pTab; + pSubitem->pTab = 0; + p->pOrderBy = 0; + p->pPrior = 0; + p->pLimit = 0; + pNew = sqlite3SelectDup(db, p, 0); + p->pLimit = pLimit; + p->pOrderBy = pOrderBy; + p->op = TK_ALL; + pSubitem->pTab = pItemTab; + if( pNew==0 ){ + p->pPrior = pPrior; + }else{ + pNew->selId = ++pParse->nSelect; + if( aCsrMap && db->mallocFailed==0 ){ + renumberCursors(pParse, pNew, iFrom, aCsrMap); + } + pNew->pPrior = pPrior; + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = p; + p->pPrior = pNew; + SELECTTRACE(2,pParse,p,("compound-subquery flattener" + " creates %u as peer\n",pNew->selId)); + } + assert( pSubitem->pSelect==0 ); + } + sqlite3DbFree(db, aCsrMap); + if( db->mallocFailed ){ + pSubitem->pSelect = pSub1; + return 1; + } + + /* Defer deleting the Table object associated with the + ** subquery until code generation is + ** complete, since there may still exist Expr.pTab entries that + ** refer to the subquery even after flattening. Ticket #3346. + ** + ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + */ + if( ALWAYS(pSubitem->pTab!=0) ){ + Table *pTabToDel = pSubitem->pTab; + if( pTabToDel->nTabRef==1 ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3ParserAddCleanup(pToplevel, + (void(*)(sqlite3*,void*))sqlite3DeleteTable, + pTabToDel); + testcase( pToplevel->earlyCleanup ); + }else{ + pTabToDel->nTabRef--; + } + pSubitem->pTab = 0; + } + + /* The following loop runs once for each term in a compound-subquery + ** flattening (as described above). If we are doing a different kind + ** of flattening - a flattening other than a compound-subquery flattening - + ** then this loop only runs once. + ** + ** This loop moves all of the FROM elements of the subquery into the + ** the FROM clause of the outer query. Before doing this, remember + ** the cursor number for the original outer query FROM element in + ** iParent. The iParent cursor will never be used. Subsequent code + ** will scan expressions looking for iParent references and replace + ** those references with expressions that resolve to the subquery FROM + ** elements we are now copying in. + */ + pSub = pSub1; + for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ + int nSubSrc; + u8 jointype = 0; + assert( pSub!=0 ); + pSubSrc = pSub->pSrc; /* FROM clause of subquery */ + nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ + pSrc = pParent->pSrc; /* FROM clause of the outer query */ + + if( pParent==p ){ + jointype = pSubitem->fg.jointype; /* First time through the loop */ + } + + /* The subquery uses a single slot of the FROM clause of the outer + ** query. If the subquery has more than one element in its FROM clause, + ** then expand the outer query to make space for it to hold all elements + ** of the subquery. + ** + ** Example: + ** + ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; + ** + ** The outer query has 3 slots in its FROM clause. One slot of the + ** outer query (the middle slot) is used by the subquery. The next + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. + */ + if( nSubSrc>1 ){ + pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); + if( pSrc==0 ) break; + pParent->pSrc = pSrc; + } + + /* Transfer the FROM clause terms from the subquery into the + ** outer query. + */ + for(i=0; ia[i+iFrom].pUsing); + assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); + pSrc->a[i+iFrom] = pSubSrc->a[i]; + iNewParent = pSubSrc->a[i].iCursor; + memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); + } + pSrc->a[iFrom].fg.jointype = jointype; + + /* Now begin substituting subquery result set expressions for + ** references to the iParent in the outer query. + ** + ** Example: + ** + ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; + ** \ \_____________ subquery __________/ / + ** \_____________________ outer query ______________________________/ + ** + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ + if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + assert( pParent->pOrderBy==0 ); + pParent->pOrderBy = pOrderBy; + pSub->pOrderBy = 0; + } + pWhere = pSub->pWhere; + pSub->pWhere = 0; + if( isLeftJoin>0 ){ + sqlite3SetJoinExpr(pWhere, iNewParent); + } + if( pWhere ){ + if( pParent->pWhere ){ + pParent->pWhere = sqlite3PExpr(pParse, TK_AND, pWhere, pParent->pWhere); + }else{ + pParent->pWhere = pWhere; + } + } + if( db->mallocFailed==0 ){ + SubstContext x; + x.pParse = pParse; + x.iTable = iParent; + x.iNewTable = iNewParent; + x.isLeftJoin = isLeftJoin; + x.pEList = pSub->pEList; + substSelect(&x, pParent, 0); + } + + /* The flattened query is a compound if either the inner or the + ** outer query is a compound. */ + pParent->selFlags |= pSub->selFlags & SF_Compound; + assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ + + /* + ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; + ** + ** One is tempted to try to add a and b to combine the limits. But this + ** does not work if either limit is negative. + */ + if( pSub->pLimit ){ + pParent->pLimit = pSub->pLimit; + pSub->pLimit = 0; + } + + /* Recompute the SrcList_item.colUsed masks for the flattened + ** tables. */ + for(i=0; ia[i+iFrom]); + } + } + + /* Finially, delete what is left of the subquery and return + ** success. + */ + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w,pSub1); + sqlite3SelectDelete(db, pSub1); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + return 1; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** A structure to keep track of all of the column values that are fixed to +** a known value due to WHERE clause constraints of the form COLUMN=VALUE. +*/ +typedef struct WhereConst WhereConst; +struct WhereConst { + Parse *pParse; /* Parsing context */ + int nConst; /* Number for COLUMN=CONSTANT terms */ + int nChng; /* Number of times a constant is propagated */ + Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ +}; + +/* +** Add a new entry to the pConst object. Except, do not add duplicate +** pColumn entires. Also, do not add if doing so would not be appropriate. +** +** The caller guarantees the pColumn is a column and pValue is a constant. +** This routine has to do some additional checks before completing the +** insert. +*/ +static void constInsert( + WhereConst *pConst, /* The WhereConst into which we are inserting */ + Expr *pColumn, /* The COLUMN part of the constraint */ + Expr *pValue, /* The VALUE part of the constraint */ + Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */ +){ + int i; + assert( pColumn->op==TK_COLUMN ); + assert( sqlite3ExprIsConstant(pValue) ); + + if( ExprHasProperty(pColumn, EP_FixedCol) ) return; + if( sqlite3ExprAffinity(pValue)!=0 ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){ + return; + } + + /* 2018-10-25 ticket [cf5ed20f] + ** Make sure the same pColumn is not inserted more than once */ + for(i=0; inConst; i++){ + const Expr *pE2 = pConst->apExpr[i*2]; + assert( pE2->op==TK_COLUMN ); + if( pE2->iTable==pColumn->iTable + && pE2->iColumn==pColumn->iColumn + ){ + return; /* Already present. Return without doing anything. */ + } + } + + pConst->nConst++; + pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr, + pConst->nConst*2*sizeof(Expr*)); + if( pConst->apExpr==0 ){ + pConst->nConst = 0; + }else{ + pConst->apExpr[pConst->nConst*2-2] = pColumn; + pConst->apExpr[pConst->nConst*2-1] = pValue; + } +} + +/* +** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE +** is a constant expression and where the term must be true because it +** is part of the AND-connected terms of the expression. For each term +** found, add it to the pConst structure. +*/ +static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ + Expr *pRight, *pLeft; + if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return; + if( pExpr->op==TK_AND ){ + findConstInWhere(pConst, pExpr->pRight); + findConstInWhere(pConst, pExpr->pLeft); + return; + } + if( pExpr->op!=TK_EQ ) return; + pRight = pExpr->pRight; + pLeft = pExpr->pLeft; + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + constInsert(pConst,pRight,pLeft,pExpr); + } + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + constInsert(pConst,pLeft,pRight,pExpr); + } +} + +/* +** This is a Walker expression callback. pExpr is a candidate expression +** to be replaced by a value. If pExpr is equivalent to one of the +** columns named in pWalker->u.pConst, then overwrite it with its +** corresponding value. +*/ +static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ + int i; + WhereConst *pConst; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + if( ExprHasProperty(pExpr, EP_FixedCol|EP_FromJoin) ){ + testcase( ExprHasProperty(pExpr, EP_FixedCol) ); + testcase( ExprHasProperty(pExpr, EP_FromJoin) ); + return WRC_Continue; + } + pConst = pWalker->u.pConst; + for(i=0; inConst; i++){ + Expr *pColumn = pConst->apExpr[i*2]; + if( pColumn==pExpr ) continue; + if( pColumn->iTable!=pExpr->iTable ) continue; + if( pColumn->iColumn!=pExpr->iColumn ) continue; + /* A match is found. Add the EP_FixedCol property */ + pConst->nChng++; + ExprClearProperty(pExpr, EP_Leaf); + ExprSetProperty(pExpr, EP_FixedCol); + assert( pExpr->pLeft==0 ); + pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0); + break; + } + return WRC_Prune; +} + +/* +** The WHERE-clause constant propagation optimization. +** +** If the WHERE clause contains terms of the form COLUMN=CONSTANT or +** CONSTANT=COLUMN that are top-level AND-connected terms that are not +** part of a ON clause from a LEFT JOIN, then throughout the query +** replace all other occurrences of COLUMN with CONSTANT. +** +** For example, the query: +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b +** +** Is transformed into +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39 +** +** Return true if any transformations where made and false if not. +** +** Implementation note: Constant propagation is tricky due to affinity +** and collating sequence interactions. Consider this example: +** +** CREATE TABLE t1(a INT,b TEXT); +** INSERT INTO t1 VALUES(123,'0123'); +** SELECT * FROM t1 WHERE a=123 AND b=a; +** SELECT * FROM t1 WHERE a=123 AND b=123; +** +** The two SELECT statements above should return different answers. b=a +** is alway true because the comparison uses numeric affinity, but b=123 +** is false because it uses text affinity and '0123' is not the same as '123'. +** To work around this, the expression tree is not actually changed from +** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol +** and the "123" value is hung off of the pLeft pointer. Code generator +** routines know to generate the constant "123" instead of looking up the +** column value. Also, to avoid collation problems, this optimization is +** only attempted if the "a=123" term uses the default BINARY collation. +*/ +static int propagateConstants( + Parse *pParse, /* The parsing context */ + Select *p /* The query in which to propagate constants */ +){ + WhereConst x; + Walker w; + int nChng = 0; + x.pParse = pParse; + do{ + x.nConst = 0; + x.nChng = 0; + x.apExpr = 0; + findConstInWhere(&x, p->pWhere); + if( x.nConst ){ + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = propagateConstantExprRewrite; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.u.pConst = &x; + sqlite3WalkExpr(&w, p->pWhere); + sqlite3DbFree(x.pParse->db, x.apExpr); + nChng += x.nChng; + } + }while( x.nChng ); + return nChng; +} + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +# if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** This function is called to determine whether or not it is safe to +** push WHERE clause expression pExpr down to FROM clause sub-query +** pSubq, which contains at least one window function. Return 1 +** if it is safe and the expression should be pushed down, or 0 +** otherwise. +** +** It is only safe to push the expression down if it consists only +** of constants and copies of expressions that appear in the PARTITION +** BY clause of all window function used by the sub-query. It is safe +** to filter out entire partitions, but not rows within partitions, as +** this may change the results of the window functions. +** +** At the time this function is called it is guaranteed that +** +** * the sub-query uses only one distinct window frame, and +** * that the window frame has a PARTITION BY clase. +*/ +static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ + assert( pSubq->pWin->pPartition ); + assert( (pSubq->selFlags & SF_MultiPart)==0 ); + assert( pSubq->pPrior==0 ); + return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition); +} +# endif /* SQLITE_OMIT_WINDOWFUNC */ +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Make copies of relevant WHERE clause terms of the outer query into +** the WHERE clause of subquery. Example: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; +** +** Transformed into: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) +** WHERE x=5 AND y=10; +** +** The hope is that the terms added to the inner query will make it more +** efficient. +** +** Do not attempt this optimization if: +** +** (1) (** This restriction was removed on 2017-09-29. We used to +** disallow this optimization for aggregate subqueries, but now +** it is allowed by putting the extra terms on the HAVING clause. +** The added HAVING clause is pointless if the subquery lacks +** a GROUP BY clause. But such a HAVING clause is also harmless +** so there does not appear to be any reason to add extra logic +** to suppress it. **) +** +** (2) The inner query is the recursive part of a common table expression. +** +** (3) The inner query has a LIMIT clause (since the changes to the WHERE +** clause would change the meaning of the LIMIT). +** +** (4) The inner query is the right operand of a LEFT JOIN and the +** expression to be pushed down does not come from the ON clause +** on that LEFT JOIN. +** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN where iCursor is not the right-hand table of that +** left join. An example: +** +** SELECT * +** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa +** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2) +** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2); +** +** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9). +** But if the (b2=2) term were to be pushed down into the bb subquery, +** then the (1,1,NULL) row would be suppressed. +** +** (6) Window functions make things tricky as changes to the WHERE clause +** of the inner query could change the window over which window +** functions are calculated. Therefore, do not attempt the optimization +** if: +** +** (6a) The inner query uses multiple incompatible window partitions. +** +** (6b) The inner query is a compound and uses window-functions. +** +** (6c) The WHERE clause does not consist entirely of constants and +** copies of expressions found in the PARTITION BY clause of +** all window-functions used by the sub-query. It is safe to +** filter out entire partitions, as this does not change the +** window over which any window-function is calculated. +** +** (7) The inner query is a Common Table Expression (CTE) that should +** be materialized. (This restriction is implemented in the calling +** routine.) +** +** Return 0 if no changes are made and non-zero if one or more WHERE clause +** terms are duplicated into the subquery. +*/ +static int pushDownWhereTerms( + Parse *pParse, /* Parse context (for malloc() and error reporting) */ + Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ + Expr *pWhere, /* The WHERE clause of the outer query */ + int iCursor, /* Cursor number of the subquery */ + int isLeftJoin /* True if pSubq is the right term of a LEFT JOIN */ +){ + Expr *pNew; + int nChng = 0; + if( pWhere==0 ) return 0; + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pPrior ){ + Select *pSel; + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + if( pSel->pWin ) return 0; /* restriction (6b) */ + } + }else{ + if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; + } +#endif + +#ifdef SQLITE_DEBUG + /* Only the first term of a compound can have a WITH clause. But make + ** sure no other terms are marked SF_Recursive in case something changes + ** in the future. + */ + { + Select *pX; + for(pX=pSubq; pX; pX=pX->pPrior){ + assert( (pX->selFlags & (SF_Recursive))==0 ); + } + } +#endif + + if( pSubq->pLimit!=0 ){ + return 0; /* restriction (3) */ + } + while( pWhere->op==TK_AND ){ + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, + iCursor, isLeftJoin); + pWhere = pWhere->pLeft; + } + if( isLeftJoin + && (ExprHasProperty(pWhere,EP_FromJoin)==0 + || pWhere->iRightJoinTable!=iCursor) + ){ + return 0; /* restriction (4) */ + } + if( ExprHasProperty(pWhere,EP_FromJoin) && pWhere->iRightJoinTable!=iCursor ){ + return 0; /* restriction (5) */ + } + if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + nChng++; + pSubq->selFlags |= SF_PushDown; + while( pSubq ){ + SubstContext x; + pNew = sqlite3ExprDup(pParse->db, pWhere, 0); + unsetJoinExpr(pNew, -1); + x.pParse = pParse; + x.iTable = iCursor; + x.iNewTable = iCursor; + x.isLeftJoin = 0; + x.pEList = pSubq->pEList; + pNew = substExpr(&x, pNew); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ + /* Restriction 6c has prevented push-down in this case */ + sqlite3ExprDelete(pParse->db, pNew); + nChng--; + break; + } +#endif + if( pSubq->selFlags & SF_Aggregate ){ + pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew); + }else{ + pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew); + } + pSubq = pSubq->pPrior; + } + } + return nChng; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** The pFunc is the only aggregate function in the query. Check to see +** if the query is a candidate for the min/max optimization. +** +** If the query is a candidate for the min/max optimization, then set +** *ppMinMax to be an ORDER BY clause to be used for the optimization +** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on +** whether pFunc is a min() or max() function. +** +** If the query is not a candidate for the min/max optimization, return +** WHERE_ORDERBY_NORMAL (which must be zero). +** +** This routine must be called after aggregate functions have been +** located but before their arguments have been subjected to aggregate +** analysis. +*/ +static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + ExprList *pEList = pFunc->x.pList; /* Arguments to agg function */ + const char *zFunc; /* Name of aggregate function pFunc */ + ExprList *pOrderBy; + u8 sortFlags = 0; + + assert( *ppMinMax==0 ); + assert( pFunc->op==TK_AGG_FUNCTION ); + assert( !IsWindowFunc(pFunc) ); + if( pEList==0 + || pEList->nExpr!=1 + || ExprHasProperty(pFunc, EP_WinFunc) + || OptimizationDisabled(db, SQLITE_MinMaxOpt) + ){ + return eRet; + } + zFunc = pFunc->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){ + sortFlags = KEYINFO_ORDER_BIGNULL; + } + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + sortFlags = KEYINFO_ORDER_DESC; + }else{ + return eRet; + } + *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); + assert( pOrderBy!=0 || db->mallocFailed ); + if( pOrderBy ) pOrderBy->a[0].sortFlags = sortFlags; + return eRet; +} + +/* +** The select statement passed as the first argument is an aggregate query. +** The second argument is the associated aggregate-info object. This +** function tests if the SELECT is of the form: +** +** SELECT count(*) FROM +** +** where table is a database table, not a sub-select or view. If the query +** does match this pattern, then a pointer to the Table object representing +** is returned. Otherwise, 0 is returned. +*/ +static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ + Table *pTab; + Expr *pExpr; + + assert( !p->pGroupBy ); + + if( p->pWhere || p->pEList->nExpr!=1 + || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect + ){ + return 0; + } + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + assert( pTab && !pTab->pSelect && pExpr ); + + if( IsVirtual(pTab) ) return 0; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( NEVER(pAggInfo->nFunc==0) ) return 0; + if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; + if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0; + + return pTab; +} + +/* +** If the source-list item passed as an argument was augmented with an +** INDEXED BY clause, then try to locate the specified index. If there +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** pFrom->pIndex and return SQLITE_OK. +*/ +int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ + Table *pTab = pFrom->pTab; + char *zIndexedBy = pFrom->u1.zIndexedBy; + Index *pIdx; + assert( pTab!=0 ); + assert( pFrom->fg.isIndexedBy!=0 ); + + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); + pParse->checkSchema = 1; + return SQLITE_ERROR; + } + pFrom->u2.pIBIndex = pIdx; + return SQLITE_OK; +} + +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; +#ifndef SQLITE_OMIT_WINDOWFUNC + /* If iOrderByCol is already non-zero, then it has already been matched + ** to a result column of the SELECT statement. This occurs when the + ** SELECT is rewritten for window-functions processing and then passed + ** to sqlite3SelectPrep() and similar a second time. The rewriting done + ** by this function is not required in this case. */ + if( a[0].u.x.iOrderByCol ) return WRC_Continue; +#endif + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + p->pNext = 0; + p->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + p->pWinDefn = 0; +#endif + p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; + assert( pNew->pPrior!=0 ); + pNew->pPrior->pNext = pNew; + pNew->pLimit = 0; + return WRC_Continue; +} + +/* +** Check to see if the FROM clause term pFrom has table-valued function +** arguments. If it does, leave an error message in pParse and return +** non-zero, since pFrom is not allowed to be a table-valued function. +*/ +static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){ + if( pFrom->fg.isTabFunc ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); + return 1; + } + return 0; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. +*/ +static struct Cte *searchWith( + With *pWith, /* Current innermost WITH clause */ + SrcItem *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ + const char *zName = pItem->zName; + With *p; + assert( pItem->zDatabase==0 ); + assert( zName!=0 ); + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; + } + } + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +*/ +void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + if( pWith ){ + assert( pParse->pWith!=pWith ); + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + if( bFree ){ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3WithDelete, + pWith); + testcase( pParse->earlyCleanup ); + } + } +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser (on the +** pParse->pWith linked list). And if currently processing a CTE +** CTE expression, through routine checks to see if the reference is +** a recursive reference to the CTE. +** +** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** and other fields are populated accordingly. +** +** Return 0 if no match is found. +** Return 1 if a match is found. +** Return 2 if an error condition is detected. +*/ +static int resolveFromTermToCte( + Parse *pParse, /* The parsing context */ + Walker *pWalker, /* Current tree walker */ + SrcItem *pFrom /* The FROM clause term to check */ +){ + Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* The matching WITH */ + + assert( pFrom->pTab==0 ); + if( pParse->pWith==0 ){ + /* There are no WITH clauses in the stack. No match is possible */ + return 0; + } + if( pFrom->zDatabase!=0 ){ + /* The FROM term contains a schema qualifier (ex: main.t1) and so + ** it cannot possibly be a CTE reference. */ + return 0; + } + pCte = searchWith(pParse->pWith, pFrom, &pWith); + if( pCte ){ + sqlite3 *db = pParse->db; + Table *pTab; + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + Select *pRecTerm; /* Left-most recursive term */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ + int iRecTab = -1; /* Cursor for recursive table */ + CteUse *pCteUse; + + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); + return 2; + } + if( cannotBeFunction(pParse, pFrom) ) return 2; + + assert( pFrom->pTab==0 ); + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return 2; + pCteUse = pCte->pUse; + if( pCteUse==0 ){ + pCte->pUse = pCteUse = sqlite3DbMallocZero(db, sizeof(pCteUse[0])); + if( pCteUse==0 + || sqlite3ParserAddCleanup(pParse,sqlite3DbFree,pCteUse)==0 + ){ + sqlite3DbFree(db, pTab); + return 2; + } + pCteUse->eM10d = pCte->eM10d; + } + pFrom->pTab = pTab; + pTab->nTabRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return 2; + assert( pFrom->pSelect ); + pFrom->fg.isCte = 1; + pFrom->u2.pCteUse = pCteUse; + pCteUse->nUse++; + if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){ + pCteUse->eM10d = M10d_Yes; + } + + /* Check if this is a recursive CTE. */ + pRecTerm = pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + while( bMayRecursive && pRecTerm->op==pSel->op ){ + int i; + SrcList *pSrc = pRecTerm->pSrc; + assert( pRecTerm->pPrior!=0 ); + for(i=0; inSrc; i++){ + SrcItem *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pTab->nTabRef++; + pItem->fg.isRecursive = 1; + if( pRecTerm->selFlags & SF_Recursive ){ + sqlite3ErrorMsg(pParse, + "multiple references to recursive table: %s", pCte->zName + ); + return 2; + } + pRecTerm->selFlags |= SF_Recursive; + if( iRecTab<0 ) iRecTab = pParse->nTab++; + pItem->iCursor = iRecTab; + } + } + if( (pRecTerm->selFlags & SF_Recursive)==0 ) break; + pRecTerm = pRecTerm->pPrior; + } + + pCte->zCteErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; + if( pSel->selFlags & SF_Recursive ){ + int rc; + assert( pRecTerm!=0 ); + assert( (pRecTerm->selFlags & SF_Recursive)==0 ); + assert( pRecTerm->pNext!=0 ); + assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 ); + assert( pRecTerm->pWith==0 ); + pRecTerm->pWith = pSel->pWith; + rc = sqlite3WalkSelect(pWalker, pRecTerm); + pRecTerm->pWith = 0; + if( rc ){ + pParse->pWith = pSavedWith; + return 2; + } + }else{ + if( sqlite3WalkSelect(pWalker, pSel) ){ + pParse->pWith = pSavedWith; + return 2; + } + } + pParse->pWith = pWith; + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + pParse->pWith = pSavedWith; + return 2; + } + pEList = pCte->pCols; + } + + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zCteErr = "multiple recursive references: %s"; + }else{ + pCte->zCteErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zCteErr = 0; + pParse->pWith = pSavedWith; + return 1; /* Success */ + } + return 0; /* No match */ +} +#endif + +#ifndef SQLITE_OMIT_CTE +/* +** If the SELECT passed as the second argument has an associated WITH +** clause, pop it from the stack stored as part of the Parse object. +** +** This function is used as the xSelectCallback2() callback by +** sqlite3SelectExpand() when walking a SELECT tree to resolve table +** names and other FROM clause elements. +*/ +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith || pParse->nErr ); + pParse->pWith = pWith->pOuter; + } + } +} +#else +#define selectPopWith 0 +#endif + +/* +** The SrcList_item structure passed as the second argument represents a +** sub-query in the FROM clause of a SELECT statement. This function +** allocates and populates the SrcList_item.pTab object. If successful, +** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, +** SQLITE_NOMEM. +*/ +int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ + Select *pSel = pFrom->pSelect; + Table *pTab; + + assert( pSel ); + pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + if( pTab==0 ) return SQLITE_NOMEM; + pTab->nTabRef = 1; + if( pFrom->zAlias ){ + pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); + }else{ + pTab->zName = sqlite3MPrintf(pParse->db, "subquery_%u", pSel->selId); + } + while( pSel->pPrior ){ pSel = pSel->pPrior; } + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; + + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** This routine is a Walker callback for "expanding" a SELECT statement. +** "Expanding" means to do the following: +** +** (1) Make sure VDBE cursor numbers have been assigned to every +** element of the FROM clause. +** +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** defines FROM clause. When views appear in the FROM clause, +** fill pTabList->a[].pSelect with a copy of the SELECT statement +** that implements the view. A copy is made of the view's SELECT +** statement so that we can freely modify or delete that statement +** without worrying about messing up the persistent representation +** of the view. +** +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword +** on joins and the ON and USING clause of joins. +** +** (4) Scan the list of columns in the result set (pEList) looking +** for instances of the "*" operator or the TABLE.* operator. +** If found, expand each "*" to be every column in every table +** and TABLE.* to be every column in TABLE. +** +*/ +static int selectExpander(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i, j, k, rc; + SrcList *pTabList; + ExprList *pEList; + SrcItem *pFrom; + sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + u32 elistFlags = 0; + + p->selFlags |= SF_Expanded; + if( db->mallocFailed ){ + return WRC_Abort; + } + assert( p->pSrc!=0 ); + if( (selFlags & SF_Expanded)!=0 ){ + return WRC_Prune; + } + if( pWalker->eCode ){ + /* Renumber selId because it has been copied from a view */ + p->selId = ++pParse->nSelect; + } + pTabList = p->pSrc; + pEList = p->pEList; + sqlite3WithPush(pParse, p->pWith, 0); + + /* Make sure cursor numbers have been assigned to all entries in + ** the FROM clause of the SELECT statement. + */ + sqlite3SrcListAssignCursors(pParse, pTabList); + + /* Look up every table named in the FROM clause of the select. If + ** an entry of the FROM clause is a subquery instead of a table or view, + ** then create a transient table structure to describe the subquery. + */ + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab; + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 ); + if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_SUBQUERY + Select *pSel = pFrom->pSelect; + /* A sub-query in the FROM clause of a SELECT */ + assert( pSel!=0 ); + assert( pFrom->pTab==0 ); + if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; + if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; +#endif +#ifndef SQLITE_OMIT_CTE + }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ + if( rc>1 ) return WRC_Abort; + pTab = pFrom->pTab; + assert( pTab!=0 ); +#endif + }else{ + /* An ordinary table or view name in the FROM clause */ + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + if( pTab==0 ) return WRC_Abort; + if( pTab->nTabRef>=0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } + pTab->nTabRef++; + if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ + return WRC_Abort; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( IsVirtual(pTab) || pTab->pSelect ){ + i16 nCol; + u8 eCodeOrig = pWalker->eCode; + if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; + assert( pFrom->pSelect==0 ); + if( pTab->pSelect + && (db->flags & SQLITE_EnableView)==0 + && pTab->pSchema!=db->aDb[1].pSchema + ){ + sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", + pTab->zName); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) + && pFrom->fg.fromDDL + && ALWAYS(pTab->pVTable!=0) + && pTab->pVTable->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } +#endif + pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + nCol = pTab->nCol; + pTab->nCol = -1; + pWalker->eCode = 1; /* Turn on Select.selId renumbering */ + sqlite3WalkSelect(pWalker, pFrom->pSelect); + pWalker->eCode = eCodeOrig; + pTab->nCol = nCol; + } +#endif + } + + /* Locate the index named by the INDEXED BY clause, if any. */ + if( pFrom->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pFrom) ){ + return WRC_Abort; + } + } + + /* Process NATURAL keywords, and ON and USING clauses of joins. + */ + if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + return WRC_Abort; + } + + /* For every "*" that occurs in the column list, insert the names of + ** all columns in all tables. And for every TABLE.* insert the names + ** of all columns in TABLE. The parser inserted a special expression + ** with the TK_ASTERISK operator for each "*" that it found in the column + ** list. The following code just has to locate the TK_ASTERISK + ** expressions and expand each one to the list of all columns in + ** all tables. + ** + ** The first loop just checks to see if there are any "*" operators + ** that need expanding. + */ + for(k=0; knExpr; k++){ + pE = pEList->a[k].pExpr; + if( pE->op==TK_ASTERISK ) break; + assert( pE->op!=TK_DOT || pE->pRight!=0 ); + assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); + if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; + elistFlags |= pE->flags; + } + if( knExpr ){ + /* + ** If we get here it means the result set contains one or more "*" + ** operators that need to be expanded. Loop through each expression + ** in the result set and expand them one by one. + */ + struct ExprList_item *a = pEList->a; + ExprList *pNew = 0; + int flags = pParse->db->flags; + int longNames = (flags & SQLITE_FullColNames)!=0 + && (flags & SQLITE_ShortColNames)==0; + + for(k=0; knExpr; k++){ + pE = a[k].pExpr; + elistFlags |= pE->flags; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ASTERISK + && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK) + ){ + /* This particular expression does not need to be expanded. + */ + pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); + if( pNew ){ + pNew->a[pNew->nExpr-1].zEName = a[k].zEName; + pNew->a[pNew->nExpr-1].eEName = a[k].eEName; + a[k].zEName = 0; + } + a[k].pExpr = 0; + }else{ + /* This expression is a "*" or a "TABLE.*" and needs to be + ** expanded. */ + int tableSeen = 0; /* Set to 1 when TABLE matches */ + char *zTName = 0; /* text of name of TABLE */ + if( pE->op==TK_DOT ){ + assert( pE->pLeft!=0 ); + assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); + zTName = pE->pLeft->u.zToken; + } + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; + char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; + if( zTabName==0 ){ + zTabName = pTab->zName; + } + if( db->mallocFailed ) break; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; + } + for(j=0; jnCol; j++){ + char *zName = pTab->aCol[j].zName; + char *zColname; /* The computed column name */ + char *zToFree; /* Malloced string that needs to be freed */ + Token sColname; /* Computed column name as a token */ + + assert( zName ); + if( zTName && pSub + && sqlite3MatchEName(&pSub->pEList->a[j], 0, zTName, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + tableSeen = 1; + + if( i>0 && zTName==0 ){ + if( (pFrom->fg.jointype & JT_NATURAL)!=0 + && tableAndColumnIndex(pTabList, i, zName, 0, 0, 1) + ){ + /* In a NATURAL join, omit the join columns from the + ** table to the right of the join */ + continue; + } + if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + /* In a join with a USING clause, omit columns in the + ** using clause from the table on the right. */ + continue; + } + } + pRight = sqlite3Expr(db, TK_ID, zName); + zColname = zName; + zToFree = 0; + if( longNames || pTabList->nSrc>1 ){ + Expr *pLeft; + pLeft = sqlite3Expr(db, TK_ID, zTabName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr); + } + if( longNames ){ + zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + zToFree = zColname; + } + }else{ + pExpr = pRight; + } + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); + sqlite3TokenInit(&sColname, zColname); + sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + sqlite3DbFree(db, pX->zEName); + if( pSub ){ + pX->zEName = sqlite3DbStrDup(db, pSub->pEList->a[j].zEName); + testcase( pX->zEName==0 ); + }else{ + pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zEName==0 ); + } + pX->eEName = ENAME_TAB; + } + sqlite3DbFree(db, zToFree); + } + } + if( !tableSeen ){ + if( zTName ){ + sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + }else{ + sqlite3ErrorMsg(pParse, "no tables specified"); + } + } + } + } + sqlite3ExprListDelete(db, pEList); + p->pEList = pNew; + } + if( p->pEList ){ + if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; + } + if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){ + p->selFlags |= SF_ComplexResult; + } + } + return WRC_Continue; +} + +#if SQLITE_DEBUG +/* +** Always assert. This xSelectCallback2 implementation proves that the +** xSelectCallback2 is never invoked. +*/ +void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + assert( 0 ); +} +#endif +/* +** This routine "expands" a SELECT statement and all of its subqueries. +** For additional information on what it means to "expand" a SELECT +** statement, see the comment on the selectExpand worker callback above. +** +** Expanding a SELECT statement is the first step in processing a +** SELECT statement. The SELECT statement must be expanded before +** name resolution is performed. +** +** If anything goes wrong, an error message is written into pParse. +** The calling function can detect the problem by looking at pParse->nErr +** and/or pParse->db->mallocFailed. +*/ +static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ + Walker w; + w.xExprCallback = sqlite3ExprWalkNoop; + w.pParse = pParse; + if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){ + w.xSelectCallback = convertCompoundSelectToSubquery; + w.xSelectCallback2 = 0; + sqlite3WalkSelect(&w, pSelect); + } + w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; + w.eCode = 0; + sqlite3WalkSelect(&w, pSelect); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() +** interface. +** +** For each FROM-clause subquery, add Column.zType and Column.zColl +** information to the Table structure that represents the result set +** of that subquery. +** +** The Table structure that represents the result set was constructed +** by selectExpander() but the type and collation information was omitted +** at that point because identifiers had not yet been resolved. This +** routine is called after identifier resolution. +*/ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ + Parse *pParse; + int i; + SrcList *pTabList; + SrcItem *pFrom; + + assert( p->selFlags & SF_Resolved ); + if( p->selFlags & SF_HasTypeInfo ) return; + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, + SQLITE_AFF_NONE); + } + } + } +} +#endif + + +/* +** This routine adds datatype and collating sequence information to +** the Table structures of all FROM-clause subqueries in a +** SELECT statement. +** +** Use this routine after name resolution. +*/ +static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ +#ifndef SQLITE_OMIT_SUBQUERY + Walker w; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = selectAddSubqueryTypeInfo; + w.xExprCallback = sqlite3ExprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +#endif +} + + +/* +** This routine sets up a SELECT statement for processing. The +** following is accomplished: +** +** * VDBE Cursor numbers are assigned to all FROM-clause terms. +** * Ephemeral Table objects are created for all FROM-clause subqueries. +** * ON and USING clauses are shifted into WHERE statements +** * Wildcards "*" and "TABLE.*" in result sets are expanded. +** * Identifiers in expression are matched to tables. +** +** This routine acts recursively on all subqueries within the SELECT. +*/ +void sqlite3SelectPrep( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for container */ +){ + assert( p!=0 || pParse->db->mallocFailed ); + if( pParse->db->mallocFailed ) return; + if( p->selFlags & SF_HasTypeInfo ) return; + sqlite3SelectExpand(pParse, p); + if( pParse->nErr || pParse->db->mallocFailed ) return; + sqlite3ResolveSelectNames(pParse, p, pOuterNC); + if( pParse->nErr || pParse->db->mallocFailed ) return; + sqlite3SelectAddTypeInfo(pParse, p); +} + +/* +** Reset the aggregate accumulator. +** +** The aggregate accumulator is a set of memory cells that hold +** intermediate results while calculating an aggregate. This +** routine generates code that stores NULLs in all of those memory +** cells. +*/ +static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pFunc; + int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + if( nReg==0 ) return; + if( pParse->nErr || pParse->db->mallocFailed ) return; +#ifdef SQLITE_DEBUG + /* Verify that all AggInfo registers are within the range specified by + ** AggInfo.mnReg..AggInfo.mxReg */ + assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + for(i=0; inColumn; i++){ + assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg + && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + } + for(i=0; inFunc; i++){ + assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg + && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + } +#endif + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ + if( pFunc->iDistinct>=0 ){ + Expr *pE = pFunc->pFExpr; + assert( !ExprHasProperty(pE, EP_xIsSelect) ); + if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " + "argument"); + pFunc->iDistinct = -1; + }else{ + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO); + } + } + } +} + +/* +** Invoke the OP_AggFinalize opcode for every aggregate function +** in the AggInfo structure. +*/ +static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pF; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + ExprList *pList = pF->pFExpr->x.pList; + assert( !ExprHasProperty(pF->pFExpr, EP_xIsSelect) ); + sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + } +} + + +/* +** Update the accumulator memory cells for an aggregate based on +** the current cursor position. +** +** If regAcc is non-zero and there are no min() or max() aggregates +** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator +** registers if register regAcc contains 0. The caller will take care +** of setting and clearing regAcc. +*/ +static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + int regHit = 0; + int addrHitTest = 0; + struct AggInfo_func *pF; + struct AggInfo_col *pC; + + pAggInfo->directMode = 1; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + int nArg; + int addrNext = 0; + int regAgg; + ExprList *pList = pF->pFExpr->x.pList; + assert( !ExprHasProperty(pF->pFExpr, EP_xIsSelect) ); + assert( !IsWindowFunc(pF->pFExpr) ); + if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ + Expr *pFilter = pF->pFExpr->y.pWin->pFilter; + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + && regAcc + ){ + /* If regAcc==0, there there exists some min() or max() function + ** without a FILTER clause that will ensure the magnet registers + ** are populated. */ + if( regHit==0 ) regHit = ++pParse->nMem; + /* If this is the first row of the group (regAcc contains 0), clear the + ** "magnet" register regHit so that the accumulator registers + ** are populated if the FILTER clause jumps over the the + ** invocation of min() or max() altogether. Or, if this is not + ** the first row (regAcc contains 1), set the magnet register so that + ** the accumulators are not populated unless the min()/max() is invoked + ** and indicates that they should be. */ + sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit); + } + addrNext = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); + } + if( pList ){ + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); + }else{ + nArg = 0; + regAgg = 0; + } + if( pF->iDistinct>=0 ){ + if( addrNext==0 ){ + addrNext = sqlite3VdbeMakeLabel(pParse); + } + testcase( nArg==0 ); /* Error condition */ + testcase( nArg>1 ); /* Also an error */ + codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); + } + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + if( addrNext ){ + sqlite3VdbeResolveLabel(v, addrNext); + } + } + if( regHit==0 && pAggInfo->nAccumulator ){ + regHit = regAcc; + } + if( regHit ){ + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); + } + for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ + sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + } + + pAggInfo->directMode = 0; + if( addrHitTest ){ + sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); + } +} + +/* +** Add a single OP_Explain instruction to the VDBE to explain a simple +** count(*) query ("SELECT count(*) FROM pTab"). +*/ +#ifndef SQLITE_OMIT_EXPLAIN +static void explainSimpleCount( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being queried */ + Index *pIdx /* Index used to optimize scan, or NULL */ +){ + if( pParse->explain==2 ){ + int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); + sqlite3VdbeExplain(pParse, 0, "SCAN TABLE %s%s%s", + pTab->zName, + bCover ? " USING COVERING INDEX " : "", + bCover ? pIdx->zName : "" + ); + } +} +#else +# define explainSimpleCount(a,b,c) +#endif + +/* +** sqlite3WalkExpr() callback used by havingToWhere(). +** +** If the node passed to the callback is a TK_AND node, return +** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes. +** +** Otherwise, return WRC_Prune. In this case, also check if the +** sub-expression matches the criteria for being moved to the WHERE +** clause. If so, add it to the WHERE clause and replace the sub-expression +** within the HAVING expression with a constant "1". +*/ +static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op!=TK_AND ){ + Select *pS = pWalker->u.pSelect; + if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) + && ExprAlwaysFalse(pExpr)==0 + ){ + sqlite3 *db = pWalker->pParse->db; + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + if( pNew ){ + Expr *pWhere = pS->pWhere; + SWAP(Expr, *pNew, *pExpr); + pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew); + pS->pWhere = pNew; + pWalker->eCode = 1; + } + } + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Transfer eligible terms from the HAVING clause of a query, which is +** processed after grouping, to the WHERE clause, which is processed before +** grouping. For example, the query: +** +** SELECT * FROM WHERE a=? GROUP BY b HAVING b=? AND c=? +** +** can be rewritten as: +** +** SELECT * FROM WHERE a=? AND b=? GROUP BY b HAVING c=? +** +** A term of the HAVING expression is eligible for transfer if it consists +** entirely of constants and expressions that are also GROUP BY terms that +** use the "BINARY" collation sequence. +*/ +static void havingToWhere(Parse *pParse, Select *p){ + Walker sWalker; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = havingToWhereExprCb; + sWalker.u.pSelect = p; + sqlite3WalkExpr(&sWalker, p->pHaving); +#if SELECTTRACE_ENABLED + if( sWalker.eCode && (sqlite3SelectTrace & 0x100)!=0 ){ + SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +} + +/* +** Check to see if the pThis entry of pTabList is a self-join of a prior view. +** If it is, then return the SrcList_item for the prior view. If it is not, +** then return 0. +*/ +static SrcItem *isSelfJoinView( + SrcList *pTabList, /* Search for self-joins in this FROM clause */ + SrcItem *pThis /* Search for prior reference to this subquery */ +){ + SrcItem *pItem; + assert( pThis->pSelect!=0 ); + if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + for(pItem = pTabList->a; pItempSelect==0 ) continue; + if( pItem->fg.viaCoroutine ) continue; + if( pItem->zName==0 ) continue; + assert( pItem->pTab!=0 ); + assert( pThis->pTab!=0 ); + if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; + pS1 = pItem->pSelect; + if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + /* The query flattener left two different CTE tables with identical + ** names in the same FROM clause. */ + continue; + } + if( pItem->pSelect->selFlags & SF_PushDown ){ + /* The view was modified by some other optimization such as + ** pushDownWhereTerms() */ + continue; + } + return pItem; + } + return 0; +} + +/* +** Deallocate a single AggInfo object +*/ +static void agginfoFree(sqlite3 *db, AggInfo *p){ + sqlite3DbFree(db, p->aCol); + sqlite3DbFree(db, p->aFunc); + sqlite3DbFreeNN(db, p); +} + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION +/* +** Attempt to transform a query of the form +** +** SELECT count(*) FROM (SELECT x FROM t1 UNION ALL SELECT y FROM t2) +** +** Into this: +** +** SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) +** +** The transformation only works if all of the following are true: +** +** * The subquery is a UNION ALL of two or more terms +** * The subquery does not have a LIMIT clause +** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries +** * The outer query is a simple count(*) with no WHERE clause or other +** extraneous syntax. +** +** Return TRUE if the optimization is undertaken. +*/ +static int countOfViewOptimization(Parse *pParse, Select *p){ + Select *pSub, *pPrior; + Expr *pExpr; + Expr *pCount; + sqlite3 *db; + if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ + if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ + if( p->pWhere ) return 0; + if( p->pGroupBy ) return 0; + pExpr = p->pEList->a[0].pExpr; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ + if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */ + if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ + if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ + pSub = p->pSrc->a[0].pSelect; + if( pSub==0 ) return 0; /* The FROM is a subquery */ + if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */ + do{ + if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ + if( pSub->pWhere ) return 0; /* No WHERE clause */ + if( pSub->pLimit ) return 0; /* No LIMIT clause */ + if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + pSub = pSub->pPrior; /* Repeat over compound */ + }while( pSub ); + + /* If we reach this point then it is OK to perform the transformation */ + + db = pParse->db; + pCount = pExpr; + pExpr = 0; + pSub = p->pSrc->a[0].pSelect; + p->pSrc->a[0].pSelect = 0; + sqlite3SrcListDelete(db, p->pSrc); + p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + while( pSub ){ + Expr *pTerm; + pPrior = pSub->pPrior; + pSub->pPrior = 0; + pSub->pNext = 0; + pSub->selFlags |= SF_Aggregate; + pSub->selFlags &= ~SF_Compound; + pSub->nSelectRow = 0; + sqlite3ExprListDelete(db, pSub->pEList); + pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; + pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); + pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, pTerm, pSub); + if( pExpr==0 ){ + pExpr = pTerm; + }else{ + pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr); + } + pSub = pPrior; + } + p->pEList->a[0].pExpr = pExpr; + p->selFlags &= ~SF_Aggregate; + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; +} +#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ + +/* +** Generate code for the SELECT statement given in the p argument. +** +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. +** +** This routine returns the number of errors. If any errors are +** encountered, then an appropriate error message is left in +** pParse->zErrMsg. +** +** This routine does NOT free the Select structure passed in. The +** calling function needs to do that. +*/ +int sqlite3Select( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + SelectDest *pDest /* What to do with the query results */ +){ + int i, j; /* Loop counters */ + WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ + Vdbe *v; /* The virtual machine under construction */ + int isAgg; /* True for select lists like "count(*)" */ + ExprList *pEList = 0; /* List of columns to extract. */ + SrcList *pTabList; /* List of tables to select from */ + Expr *pWhere; /* The WHERE clause. May be NULL */ + ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ + Expr *pHaving; /* The HAVING clause. May be NULL */ + AggInfo *pAggInfo = 0; /* Aggregate information */ + int rc = 1; /* Value to return from this function */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ + int iEnd; /* Address of the end of the query */ + sqlite3 *db; /* The database connection */ + ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */ + u8 minMaxFlag; /* Flag for min/max queries */ + + db = pParse->db; + v = sqlite3GetVdbe(pParse); + if( p==0 || db->mallocFailed || pParse->nErr ){ + return 1; + } + if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); + if( IgnorableDistinct(pDest) ){ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); + /* All of these destinations are also able to ignore the ORDER BY clause */ + if( p->pOrderBy ){ +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); + } +#endif + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprListDelete, + p->pOrderBy); + testcase( pParse->earlyCleanup ); + p->pOrderBy = 0; + } + p->selFlags &= ~SF_Distinct; + p->selFlags |= SF_NoopOrderBy; + } + sqlite3SelectPrep(pParse, p, 0); + if( pParse->nErr || db->mallocFailed ){ + goto select_end; + } + assert( p->pEList!=0 ); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x104 ){ + SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + /* If the SF_UpdateFrom flag is set, then this function is being called + ** as part of populating the temp table for an UPDATE...FROM statement. + ** In this case, it is an error if the target object (pSrc->a[0]) name + ** or alias is duplicated within FROM clause (pSrc->a[1..n]). */ + if( p->selFlags & SF_UpdateFrom ){ + SrcItem *p0 = &p->pSrc->a[0]; + for(i=1; ipSrc->nSrc; i++){ + SrcItem *p1 = &p->pSrc->a[i]; + if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + sqlite3ErrorMsg(pParse, + "target object/alias may not appear in FROM clause: %s", + p0->zAlias ? p0->zAlias : p0->pTab->zName + ); + goto select_end; + } + } + } + + if( pDest->eDest==SRT_Output ){ + generateColumnNames(pParse, p); + } + +#ifndef SQLITE_OMIT_WINDOWFUNC + rc = sqlite3WindowRewrite(pParse, p); + if( rc ){ + assert( db->mallocFailed || pParse->nErr>0 ); + goto select_end; + } +#if SELECTTRACE_ENABLED + if( p->pWin && (sqlite3SelectTrace & 0x108)!=0 ){ + SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +#endif /* SQLITE_OMIT_WINDOWFUNC */ + pTabList = p->pSrc; + isAgg = (p->selFlags & SF_Aggregate)!=0; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; + + /* Try to do various optimizations (flattening subqueries, and strength + ** reduction of join operators) in the FROM clause up into the main query + */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; !p->pPrior && inSrc; i++){ + SrcItem *pItem = &pTabList->a[i]; + Select *pSub = pItem->pSelect; + Table *pTab = pItem->pTab; + + /* The expander should have already created transient Table objects + ** even for FROM clause elements such as subqueries that do not correspond + ** to a real table */ + assert( pTab!=0 ); + + /* Convert LEFT JOIN into JOIN if there are terms of the right table + ** of the LEFT JOIN used in the WHERE clause. + */ + if( (pItem->fg.jointype & JT_LEFT)!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + && OptimizationEnabled(db, SQLITE_SimplifyJoin) + ){ + SELECTTRACE(0x100,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor); + } + + /* No futher action if this term of the FROM clause is no a subquery */ + if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + + /* Do not try to flatten an aggregate subquery. + ** + ** Flattening an aggregate subquery is only possible if the outer query + ** is not a join. But if the outer query is not a join, then the subquery + ** will be implemented as a co-routine and there is no advantage to + ** flattening in that case. + */ + if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; + assert( pSub->pGroupBy==0 ); + + /* If the outer query contains a "complex" result set (that is, + ** if the result set of the outer query uses functions or subqueries) + ** and if the subquery contains an ORDER BY clause and if + ** it will be implemented as a co-routine, then do not flatten. This + ** restriction allows SQL constructs like this: + ** + ** SELECT expensive_function(x) + ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + ** + ** The expensive_function() is only computed on the 10 rows that + ** are output, rather than every row of the table. + ** + ** The requirement that the outer query have a complex result set + ** means that flattening does occur on simpler SQL constraints without + ** the expensive_function() like: + ** + ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + */ + if( pSub->pOrderBy!=0 + && i==0 + && (p->selFlags & SF_ComplexResult)!=0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) + ){ + continue; + } + + if( flattenSubquery(pParse, p, i, isAgg) ){ + if( pParse->nErr ) goto select_end; + /* This subquery can be absorbed into its parent. */ + i = -1; + } + pTabList = p->pSrc; + if( db->mallocFailed ) goto select_end; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } +#endif + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* Handle compound SELECT statements using the separate multiSelect() + ** procedure. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); +#if SELECTTRACE_ENABLED + SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); + if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + if( p->pNext==0 ) ExplainQueryPlanPop(pParse); + return rc; + } +#endif + + /* Do the WHERE-clause constant propagation optimization if this is + ** a join. No need to speed time on this operation for non-join queries + ** as the equivalent optimization will be handled by query planner in + ** sqlite3WhereBegin(). + */ + if( pTabList->nSrc>1 + && OptimizationEnabled(db, SQLITE_PropagateConst) + && propagateConstants(pParse, p) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + }else{ + SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + } + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION + if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) + && countOfViewOptimization(pParse, p) + ){ + if( db->mallocFailed ) goto select_end; + pEList = p->pEList; + pTabList = p->pSrc; + } +#endif + + /* For each term in the FROM clause, do two things: + ** (1) Authorized unreferenced tables + ** (2) Generate code for all sub-queries + */ + for(i=0; inSrc; i++){ + SrcItem *pItem = &pTabList->a[i]; + SrcItem *pPrior; + SelectDest dest; + Select *pSub; +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + const char *zSavedAuthContext; +#endif + + /* Issue SQLITE_READ authorizations with a fake column name for any + ** tables that are referenced but from which no values are extracted. + ** Examples of where these kinds of null SQLITE_READ authorizations + ** would occur: + ** + ** SELECT count(*) FROM t1; -- SQLITE_READ t1."" + ** SELECT t1.* FROM t1, t2; -- SQLITE_READ t2."" + ** + ** The fake column name is an empty string. It is possible for a table to + ** have a column named by the empty string, in which case there is no way to + ** distinguish between an unreferenced table and an actual reference to the + ** "" column. The original design was for the fake column name to be a NULL, + ** which would be unambiguous. But legacy authorization callbacks might + ** assume the column name is non-NULL and segfault. The use of an empty + ** string for the fake column name seems safer. + */ + if( pItem->colUsed==0 && pItem->zName!=0 ){ + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + } + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + /* Generate code for all sub-queries in the FROM clause + */ + pSub = pItem->pSelect; + if( pSub==0 ) continue; + + /* The code for a subquery should only be generated once, though it is + ** technically harmless for it to be generated multiple times. The + ** following assert() will detect if something changes to cause + ** the same subquery to be coded multiple times, as a signal to the + ** developers to try to optimize the situation. + ** + ** Update 2019-07-24: + ** See ticket https://sqlite.org/src/tktview/c52b09c7f38903b1311cec40. + ** The dbsqlfuzz fuzzer found a case where the same subquery gets + ** coded twice. So this assert() now becomes a testcase(). It should + ** be very rare, though. + */ + testcase( pItem->addrFillSub!=0 ); + + /* Increment Parse.nHeight by the height of the largest expression + ** tree referred to by this, the parent select. The child select + ** may contain expression trees of at most + ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit + ** more conservative than necessary, but much easier than enforcing + ** an exact limit. + */ + pParse->nHeight += sqlite3SelectExprHeight(p); + + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( OptimizationEnabled(db, SQLITE_PushDown) + && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor, + (pItem->fg.jointype & JT_OUTER)!=0) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p, + ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + }else{ + SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + } + + zSavedAuthContext = pParse->zAuthContext; + pParse->zAuthContext = pItem->zName; + + /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if: + ** (1) the subquery is guaranteed to be the outer loop (so that + ** it does not need to be computed more than once), and + ** (2) the subquery is not a CTE that should be materialized + ** + ** TODO: Are there other reasons beside (1) and (2) to use a co-routine + ** implementation? + */ + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop = sqlite3VdbeCurrentAddr(v)+1; + + pItem->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %u", pSub->selId)); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->fg.viaCoroutine = 1; + pItem->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pItem->regReturn); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); + }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ + /* This is a CTE for which materialization code has already been + ** generated. Invoke the subroutine to compute the materialization, + ** the make the pItem->iCursor be a copy of the ephemerial table that + ** holds the result of the materialization. */ + CteUse *pCteUse = pItem->u2.pCteUse; + sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); + if( pItem->iCursor!=pCteUse->iCur ){ + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur); + } + pSub->nSelectRow = pCteUse->nRowEst; + }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){ + /* This view has already been materialized by a prior entry in + ** this same FROM clause. Reuse it. */ + if( pPrior->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + } + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + pSub->nSelectRow = pPrior->pSelect->nSelectRow; + }else{ + /* Materalize the view. If the view is not correlated, generate a + ** subroutine to do the materialization so that subsequent uses of + ** the same view can reuse the materialization. */ + int topAddr; + int onceAddr = 0; + int retAddr; + + testcase( pItem->addrFillSub==0 ); /* Ticket c52b09c7f38903b1311 */ + pItem->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + pItem->addrFillSub = topAddr+1; + if( pItem->fg.isCorrelated==0 ){ + /* If the subquery is not correlated and if we are not inside of + ** a trigger, then we only need to compute the value of the subquery + ** once. */ + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + }else{ + VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = pSub->nSelectRow; + if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); + retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3ClearTempRegCache(pParse); + if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ + CteUse *pCteUse = pItem->u2.pCteUse; + pCteUse->addrM9e = pItem->addrFillSub; + pCteUse->regRtn = pItem->regReturn; + pCteUse->iCur = pItem->iCursor; + pCteUse->nRowEst = pSub->nSelectRow; + } + } + if( db->mallocFailed ) goto select_end; + pParse->nHeight -= sqlite3SelectExprHeight(p); + pParse->zAuthContext = zSavedAuthContext; +#endif + } + + /* Various elements of the SELECT copied into local variables for + ** convenience */ + pEList = p->pEList; + pWhere = p->pWhere; + pGroupBy = p->pGroupBy; + pHaving = p->pHaving; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && p->pWin==0 +#endif + ){ + p->selFlags &= ~SF_Distinct; + pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + p->selFlags |= SF_Aggregate; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + + /* If there is an ORDER BY clause, then create an ephemeral index to + ** do the sorting. But this sorting ephemeral index might end up + ** being unused if the data can be extracted in pre-sorted order. + ** If that is the case, then the OP_OpenEphemeral instruction will be + ** changed to an OP_Noop once we figure out that the sorting index is + ** not needed. The sSort.addrSortIndex variable is used to facilitate + ** that change. + */ + if( sSort.pOrderBy ){ + KeyInfo *pKeyInfo; + pKeyInfo = sqlite3KeyInfoFromExprList( + pParse, sSort.pOrderBy, 0, pEList->nExpr); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); + }else{ + sSort.addrSortIndex = -1; + } + + /* If the output is destined for a temporary table, open that table. + */ + if( pDest->eDest==SRT_EphemTab ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); + } + + /* Set the limiter. + */ + iEnd = sqlite3VdbeMakeLabel(pParse); + if( (p->selFlags & SF_FixedLimit)==0 ){ + p->nSelectRow = 320; /* 4 billion rows */ + } + computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); + sSort.sortFlags |= SORTFLAG_UseSorter; + } + + /* Open an ephemeral index to use for the distinct set. + */ + if( p->selFlags & SF_Distinct ){ + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; + }else{ + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; + } + + if( !isAgg && pGroupBy==0 ){ + /* No aggregate functions and no GROUP BY clause */ + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) + | (p->selFlags & SF_FixedLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = p->pWin; /* Main window object (or NULL) */ + if( pWin ){ + sqlite3WindowCodeInit(pParse, p); + } +#endif + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + + + /* Begin the database scan. */ + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, p->nSelectRow); + if( pWInfo==0 ) goto select_end; + if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ + p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + } + if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ + sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); + } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + + /* If sorting index that was created by a prior OP_OpenEphemeral + ** instruction ended up not being needed, then change the OP_OpenEphemeral + ** into an OP_Noop. + */ + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + assert( p->pEList==pEList ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + int addrGosub = sqlite3VdbeMakeLabel(pParse); + int iCont = sqlite3VdbeMakeLabel(pParse); + int iBreak = sqlite3VdbeMakeLabel(pParse); + int regGosub = ++pParse->nMem; + + sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub); + + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeResolveLabel(v, addrGosub); + VdbeNoopComment((v, "inner-loop subroutine")); + sSort.labelOBLopt = 0; + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp1(v, OP_Return, regGosub); + VdbeComment((v, "end inner-loop subroutine")); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); + + /* End the database scan loop. + */ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + } + }else{ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ + NameContext sNC; /* Name context for processing aggregate information */ + int iAMem; /* First Mem address for storing current GROUP BY */ + int iBMem; /* First Mem address for previous GROUP BY */ + int iUseFlag; /* Mem address holding flag indicating that at least + ** one row of the input to the aggregator has been + ** processed */ + int iAbortFlag; /* Mem address which causes query abort if positive */ + int groupBySort; /* Rows come from source in GROUP BY order */ + int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ + + /* Remove any and all aliases between the result set and the + ** GROUP BY clause. + */ + if( pGroupBy ){ + int k; /* Loop counter */ + struct ExprList_item *pItem; /* For looping over expression in a list */ + + for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY might use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; iinExpr; ii++){ + u8 sortFlags = sSort.pOrderBy->a[ii].sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + } + }else{ + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; + } + + /* Create a label to jump to when we want to abort the query */ + addrEnd = sqlite3VdbeMakeLabel(pParse); + + /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in + ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the + ** SELECT statement. + */ + pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); + if( pAggInfo ){ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + testcase( pParse->earlyCleanup ); + } + if( db->mallocFailed ){ + goto select_end; + } + pAggInfo->selId = p->selId; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + sNC.uNC.pAggInfo = pAggInfo; + VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) + pAggInfo->mnReg = pParse->nMem+1; + pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; + pAggInfo->pGroupBy = pGroupBy; + sqlite3ExprAnalyzeAggList(&sNC, pEList); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); + if( pHaving ){ + if( pGroupBy ){ + assert( pWhere==p->pWhere ); + assert( pHaving==p->pHaving ); + assert( pGroupBy==p->pGroupBy ); + havingToWhere(pParse, p); + pWhere = p->pWhere; + } + sqlite3ExprAnalyzeAggregates(&sNC, pHaving); + } + pAggInfo->nAccumulator = pAggInfo->nColumn; + if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){ + minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy); + }else{ + minMaxFlag = WHERE_ORDERBY_NORMAL; + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.ncFlags |= NC_InAggFunc; + sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); + } +#endif + sNC.ncFlags &= ~NC_InAggFunc; + } + pAggInfo->mxReg = pParse->nMem; + if( db->mallocFailed ) goto select_end; +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + int ii; + SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + sqlite3TreeViewSelect(0, p, 0); + if( minMaxFlag ){ + sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); + sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); + } + for(ii=0; iinColumn; ii++){ + sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", + ii, pAggInfo->aCol[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->aFunc[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } + } +#endif + + + /* Processing for aggregates with GROUP BY is very different and + ** much more complex than aggregates without a GROUP BY. + */ + if( pGroupBy ){ + KeyInfo *pKeyInfo; /* Keying information for the group by clause */ + int addr1; /* A-vs-B comparision jump */ + int addrOutputRow; /* Start of subroutine that outputs a result row */ + int regOutputRow; /* Return address register for output subroutine */ + int addrSetAbort; /* Set the abort flag and return */ + int addrTopOfLoop; /* Top of the input loop */ + int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ + int addrReset; /* Subroutine for resetting the accumulator */ + int regReset; /* Return address register for reset subroutine */ + + /* If there is a GROUP BY clause we might need a sorting index to + ** implement it. Allocate that sorting index now. If it turns out + ** that we do not need it after all, the OP_SorterOpen instruction + ** will be converted into a Noop. + */ + pAggInfo->sortingIdx = pParse->nTab++; + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy, + 0, pAggInfo->nColumn); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, + pAggInfo->sortingIdx, pAggInfo->nSortingColumn, + 0, (char*)pKeyInfo, P4_KEYINFO); + + /* Initialize memory locations used by GROUP BY aggregate processing + */ + iUseFlag = ++pParse->nMem; + iAbortFlag = ++pParse->nMem; + regOutputRow = ++pParse->nMem; + addrOutputRow = sqlite3VdbeMakeLabel(pParse); + regReset = ++pParse->nMem; + addrReset = sqlite3VdbeMakeLabel(pParse); + iAMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + iBMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); + VdbeComment((v, "clear abort flag")); + sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + + /* Begin a loop that will extract all source rows in GROUP BY order. + ** This might involve two separate loops with an OP_Sort in between, or + ** it might be a single loop that uses an index to extract information + ** in the right order to begin with. + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + ); + if( pWInfo==0 ) goto select_end; + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ + /* The optimizer is able to deliver rows in group by order so + ** we do not have to sort. The OP_OpenEphemeral table will be + ** cancelled later because we still need to use the pKeyInfo + */ + groupBySort = 0; + }else{ + /* Rows are coming out in undetermined order. We have to push + ** each row into a sorting index, terminate the first loop, + ** then loop over the sorting index in order to get the output + ** in sorted order + */ + int regBase; + int regRecord; + int nCol; + int nGroupBy; + + explainTempTable(pParse, + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); + + groupBySort = 1; + nGroupBy = pGroupBy->nExpr; + nCol = nGroupBy; + j = nGroupBy; + for(i=0; inColumn; i++){ + if( pAggInfo->aCol[i].iSorterColumn>=j ){ + nCol++; + j++; + } + } + regBase = sqlite3GetTempRange(pParse, nCol); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); + j = nGroupBy; + for(i=0; inColumn; i++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[i]; + if( pCol->iSorterColumn>=j ){ + int r1 = j + regBase; + sqlite3ExprCodeGetColumnOfTable(v, + pCol->pTab, pCol->iTable, pCol->iColumn, r1); + j++; + } + } + regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nCol); + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); + pAggInfo->useSortingIdx = 1; + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + /* Evaluate the current GROUP BY terms and store in b0, b1, b2... + ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) + ** Then compare the current GROUP BY terms against the GROUP BY terms + ** from the previous row currently stored in a0, a1, a2... + */ + addrTopOfLoop = sqlite3VdbeCurrentAddr(v); + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx, + sortOut, sortPTab); + } + for(j=0; jnExpr; j++){ + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + }else{ + pAggInfo->directMode = 1; + sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); + } + } + sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); + + /* Generate code that runs whenever the GROUP BY changes. + ** Changes in the GROUP BY are detected by the previous code + ** block. If there were no changes, this block is skipped. + ** + ** This code copies current group by terms in b0,b1,b2,... + ** over to a0,a1,a2. It then calls the output subroutine + ** and resets the aggregate accumulator registers in preparation + ** for the next GROUP BY batch. + */ + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output one row")); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); + VdbeComment((v, "check abort flag")); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + VdbeComment((v, "reset accumulator")); + + /* Update the aggregate accumulators based on the content of + ** the current row + */ + sqlite3VdbeJumpHere(v, addr1); + updateAccumulator(pParse, iUseFlag, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); + VdbeComment((v, "indicate data in accumulator")); + + /* End of the loop + */ + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); + VdbeCoverage(v); + }else{ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + sqlite3VdbeChangeToNoop(v, addrSortingIdx); + } + + /* Output the final row of result + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output final row")); + + /* Jump over the subroutines + */ + sqlite3VdbeGoto(v, addrEnd); + + /* Generate a subroutine that outputs a single row of the result + ** set. This subroutine first looks at the iUseFlag. If iUseFlag + ** is less than or equal to zero, the subroutine is a no-op. If + ** the processing calls for the query to abort, this subroutine + ** increments the iAbortFlag memory location before returning in + ** order to signal the caller to abort. + */ + addrSetAbort = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); + VdbeComment((v, "set abort flag")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + sqlite3VdbeResolveLabel(v, addrOutputRow); + addrOutputRow = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeCoverage(v); + VdbeComment((v, "Groupby result generator entry point")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + finalizeAggFunctions(pParse, pAggInfo); + sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, -1, &sSort, + &sDistinct, pDest, + addrOutputRow+1, addrSetAbort); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + VdbeComment((v, "end groupby result generator")); + + /* Generate a subroutine that will reset the group-by accumulator + */ + sqlite3VdbeResolveLabel(v, addrReset); + resetAccumulator(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); + sqlite3VdbeAddOp1(v, OP_Return, regReset); + + } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ + else { + Table *pTab; + if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ + /* If isSimpleCount() returns a pointer to a Table structure, then + ** the SQL statement is of the form: + ** + ** SELECT count(*) FROM + ** + ** where the Table structure returned represents table . + ** + ** This statement is so common that it is optimized specially. The + ** OP_Count instruction is executed either on the intkey table that + ** contains the data for table or on one of its indexes. It + ** is better to execute the op on an index, as indexes are almost + ** always spread across less pages than their corresponding tables. + */ + const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */ + Index *pIdx; /* Iterator variable */ + KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ + Index *pBest = 0; /* Best index found so far */ + Pgno iRoot = pTab->tnum; /* Root page of scanned b-tree */ + + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + + /* Search for the index that has the lowest scan cost. + ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** + ** (2013-10-03) Do not count the entries in a partial index. + ** + ** In practice the KeyInfo structure will not be used. It is only + ** passed to keep OP_OpenRead happy. + */ + if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); + if( !p->pSrc->a[0].fg.notIndexed ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ + pBest = pIdx; + } + } + } + if( pBest ){ + iRoot = pBest->tnum; + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest); + } + + /* Open a read-only cursor, execute the OP_Count, close the cursor. */ + sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1); + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); + } + sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + explainSimpleCount(pParse, pTab, pBest); + }else{ + int regAcc = 0; /* "populate accumulators" flag */ + + /* If there are accumulator registers but no min() or max() functions + ** without FILTER clauses, allocate register regAcc. Register regAcc + ** will contain 0 the first time the inner loop runs, and 1 thereafter. + ** The code generated by updateAccumulator() uses this to ensure + ** that the accumulator registers are (a) updated only once if + ** there are no min() or max functions or (b) always updated for the + ** first row visited by the aggregate, so that they are updated at + ** least once even if the FILTER clause means the min() or max() + ** function visits zero rows. */ + if( pAggInfo->nAccumulator ){ + for(i=0; inFunc; i++){ + if( ExprHasProperty(pAggInfo->aFunc[i].pFExpr, EP_WinFunc) ){ + continue; + } + if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){ + break; + } + } + if( i==pAggInfo->nFunc ){ + regAcc = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); + } + } + + /* This case runs if the aggregate has no GROUP BY clause. The + ** processing is much simpler since there is only a single row + ** of output. + */ + assert( p->pGroupBy==0 ); + resetAccumulator(pParse, pAggInfo); + + /* If this query is a candidate for the min/max optimization, then + ** minMaxFlag will have been previously set to either + ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will + ** be an appropriate ORDER BY expression for the optimization. + */ + assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); + assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); + + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, + 0, minMaxFlag, 0); + if( pWInfo==0 ){ + goto select_end; + } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + updateAccumulator(pParse, regAcc, pAggInfo); + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); + if( minMaxFlag ){ + sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); + } + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + finalizeAggFunctions(pParse, pAggInfo); + } + + sSort.pOrderBy = 0; + sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, -1, 0, 0, + pDest, addrEnd, addrEnd); + } + sqlite3VdbeResolveLabel(v, addrEnd); + + } /* endif aggregate query */ + + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ + explainTempTable(pParse, "DISTINCT"); + } + + /* If there is an ORDER BY clause, then we need to sort the results + ** and send them to the callback one by one. + */ + if( sSort.pOrderBy ){ + explainTempTable(pParse, + sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + assert( p->pEList==pEList ); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); + } + + /* Jump here to skip this query + */ + sqlite3VdbeResolveLabel(v, iEnd); + + /* The SELECT has been coded. If there is an error in the Parse structure, + ** set the return code to 1. Otherwise 0. */ + rc = (pParse->nErr>0); + + /* Control jumps to here if an error is encountered above, or upon + ** successful coding of the SELECT. + */ +select_end: + sqlite3ExprListDelete(db, pMinMaxOrderBy); +#ifdef SQLITE_DEBUG + if( pAggInfo && !db->mallocFailed ){ + for(i=0; inColumn; i++){ + Expr *pExpr = pAggInfo->aCol[i].pCExpr; + assert( pExpr!=0 ); + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr!=0 ); + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + } +#endif + +#if SELECTTRACE_ENABLED + SELECTTRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + ExplainQueryPlanPop(pParse); + return rc; +} diff --git a/third_party/sqlite3/shell.c b/third_party/sqlite3/shell.c new file mode 100644 index 000000000..108279c5d --- /dev/null +++ b/third_party/sqlite3/shell.c @@ -0,0 +1,21514 @@ +/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkshellc.tcl. That script combines source +** code from various constituent source files of SQLite into this single +** "shell.c" file used to implement the SQLite command-line shell. +** +** Most of the code found below comes from the "src/shell.c.in" file in +** the canonical SQLite source tree. That main file contains "INCLUDE" +** lines that specify other files in the canonical source tree that are +** inserted to getnerate this complete program source file. +** +** The code from multiple files is combined into this single "shell.c" +** source file to help make the command-line program easier to compile. +** +** To modify this program, get a copy of the canonical SQLite source tree, +** edit the src/shell.c.in" and/or some of the other files that are included +** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. +*/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement the "sqlite" command line +** utility for accessing SQLite databases. +*/ +#if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) +/* This needs to come before any includes for MSVC compiler */ +#define _CRT_SECURE_NO_WARNINGS +#endif +/* clang-format off */ + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +/* +** Warning pragmas copied from msvc.h in the core. +*/ +#if defined(_MSC_VER) +#pragma warning(disable : 4054) +#pragma warning(disable : 4055) +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4130) +#pragma warning(disable : 4152) +#pragma warning(disable : 4189) +#pragma warning(disable : 4206) +#pragma warning(disable : 4210) +#pragma warning(disable : 4232) +#pragma warning(disable : 4244) +#pragma warning(disable : 4305) +#pragma warning(disable : 4306) +#pragma warning(disable : 4702) +#pragma warning(disable : 4706) +#endif /* defined(_MSC_VER) */ + +/* +** No support for loadable extensions in VxWorks. +*/ +#if (defined(__RTP__) || defined(_WRS_KERNEL)) && !SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif + +/* +** Enable large-file support for fopen() and friends on unix. +*/ +#ifndef SQLITE_DISABLE_LFS +# define _LARGE_FILE 1 +# ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +# endif +# define _LARGEFILE_SOURCE 1 +#endif + +#include "libc/assert.h" +#include "libc/fmt/conv.h" +#include "libc/fmt/fmt.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/sqlite3.h" + +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; +typedef unsigned char u8; +#if SQLITE_USER_AUTHENTICATION +#include "third_party/sqlite3/sqlite3userauth.inc" +#endif +#include "libc/str/str.h" + +#if !defined(_WIN32) && !defined(WIN32) +#include "libc/calls/sigbits.h" +#if !defined(__RTP__) && !defined(_WRS_KERNEL) +#include "third_party/musl/passwd.h" +#endif +#endif +#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__) +#include "libc/calls/calls.h" +#include "libc/isystem/unistd.h" +#define GETPID getpid +#if defined(__MINGW32__) +#define DIRENT dirent +#ifndef S_ISLNK +#define S_ISLNK(mode) (0) +#endif +#endif +#else +# define GETPID (int)GetCurrentProcessId +#endif +#include "libc/calls/weirdtypes.h" + +#if HAVE_READLINE +# include +# include +#endif + +#if HAVE_EDITLINE +# include +#endif + +#if HAVE_EDITLINE || HAVE_READLINE + +# define shell_add_history(X) add_history(X) +# define shell_read_history(X) read_history(X) +# define shell_write_history(X) write_history(X) +# define shell_stifle_history(X) stifle_history(X) +# define shell_readline(X) readline(X) + +#elif HAVE_LINENOISE + +#define shell_add_history(X) linenoiseHistoryAdd(X) +#define shell_read_history(X) linenoiseHistoryLoad(X) +#define shell_write_history(X) linenoiseHistorySave(X) +#define shell_stifle_history(X) linenoiseHistorySetMaxLen(X) +#define shell_readline(X) linenoise(X) + +#else + +# define shell_read_history(X) +# define shell_write_history(X) +# define shell_stifle_history(X) + +# define SHELL_USE_LOCAL_GETLINE 1 +#endif + + +#if defined(_WIN32) || defined(WIN32) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef popen +# define popen _popen +# undef pclose +# define pclose _pclose +# endif +#else +/* Make sure isatty() has a prototype. */ + +#if !defined(__RTP__) && !defined(_WRS_KERNEL) +/* popen and pclose are not C89 functions and so are +** sometimes omitted from the "libc/stdio/stdio.h" header */ +#else +#define SQLITE_OMIT_POPEN 1 +#endif +#endif + +#if defined(_WIN32_WCE) +/* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() + * thus we always assume that we have a console. That can be + * overridden with the -batch command line option. + */ +#define isatty(x) 1 +#endif + +/* ctype macros that work with signed characters */ +#define IsSpace(X) isspace((unsigned char)X) +#define IsDigit(X) isdigit((unsigned char)X) +#define ToLower(X) (char)tolower((unsigned char)X) + +#if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include +#endif +#include + +/* string conversion routines only needed on Win32 */ +extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); +extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); +extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); +extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); +#endif + +/* On Windows, we normally run with output mode of TEXT so that \n characters +** are automatically translated into \r\n. However, this behavior needs +** to be disabled in some cases (ex: when generating CSV output and when +** rendering quoted strings that contain \n characters). The following +** routines take care of that. +*/ +#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT +static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); +} +static void setTextMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_TEXT); +} +#else +# define setBinaryMode(X,Y) +# define setTextMode(X,Y) +#endif + + +/* True if the timer is enabled */ +static int enableTimer = 0; + +/* Return the current wall-clock time */ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include "libc/sysv/consts/rusage.h" +#include "libc/time/time.h" + +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. +*/ +static void endTimer(void){ + if( enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + printf("Run Time: real %.3f user %f sys %f\n", + (iEnd - iBegin)*0.001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { +#if !SQLITE_OS_WINRT + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } +#endif + } + return 0; +} + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer && getProcessTimesAddr ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +/* +** Print the timing results. +*/ +static void endTimer(void){ + if( enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + printf("Run Time: real %.3f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER +#define END_TIMER +#define HAS_TIMER 0 +#endif + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* +** If the following flag is set, then command execution stops +** at an error if we are not interactive. +*/ +static int bail_on_error = 0; + +/* +** Threat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; + +/* +** On Windows systems we have to know if standard output is a console +** in order to translate UTF-8 into MBCS. The following variable is +** true if translation is required. +*/ +static int stdout_is_console = 1; + +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *globalDb = 0; + +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; + +#ifdef SQLITE_DEBUG +/* +** Out-of-memory simulator variables +*/ +static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ +static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ +static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ +#endif /* SQLITE_DEBUG */ + +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; + +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ +static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ + +/* +** Render output like fprintf(). Except, if the output is going to the +** console and if this is running on a Windows machine, translate the +** output from UTF-8 into MBCS. +*/ +#if defined(_WIN32) || defined(WIN32) +void utf8_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + if( stdout_is_console && (out==stdout || out==stderr) ){ + char *z1 = sqlite3_vmprintf(zFormat, ap); + char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); + sqlite3_free(z1); + fputs(z2, out); + sqlite3_free(z2); + }else{ + vfprintf(out, zFormat, ap); + } + va_end(ap); +} +#elif !defined(utf8_printf) +# define utf8_printf fprintf +#endif + +/* +** Render output like fprintf(). This should not be used on anything that +** includes string formatting (e.g. "%s"). +*/ +#if !defined(raw_printf) +# define raw_printf fprintf +#endif + +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + raw_printf(stderr,"Error: out of memory\n"); + exit(1); +} + +#ifdef SQLITE_DEBUG +/* This routine is called when a simulated OOM occurs. It is broken +** out as a separate routine to make it easy to set a breakpoint on +** the OOM +*/ +void shellOomFault(void){ + if( oomRepeat>0 ){ + oomRepeat--; + }else{ + oomCounter--; + } +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* This routine is a replacement malloc() that is used to simulate +** Out-Of-Memory (OOM) errors for testing purposes. +*/ +static void *oomMalloc(int nByte){ + if( oomCounter ){ + if( oomCounter==1 ){ + shellOomFault(); + return 0; + }else{ + oomCounter--; + } + } + return defaultMalloc(nByte); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* Register the OOM simulator. This must occur before any memory +** allocations */ +static void registerOomSimulator(void){ + sqlite3_mem_methods mem; + sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem); + defaultMalloc = mem.xMalloc; + mem.xMalloc = oomMalloc; + sqlite3_config(SQLITE_CONFIG_MALLOC, &mem); +} +#endif + +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + utf8_printf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif + +/* +** Output string zUtf to stream pOut as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. +*/ +static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ + int i; + int n; + int aw = w<0 ? -w : w; + for(i=n=0; zUtf[i]; i++){ + if( (zUtf[i]&0xc0)!=0x80 ){ + n++; + if( n==aw ){ + do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + break; + } + } + } + if( n>=aw ){ + utf8_printf(pOut, "%.*s", i, zUtf); + }else if( w<0 ){ + utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + }else{ + utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + } +} + + +/* +** Determines if a string is a number of not. +*/ +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !IsDigit(*z) ){ + return 0; + } + z++; + if( realnum ) *realnum = 0; + while( IsDigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + return *z==0; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0xc0&*(z++))!=0x80 ) n++; + } + return n; +} + +/* +** Return true if zFile does not exist or if it is not an ordinary file. +*/ +#ifdef _WIN32 +# define notNormalFile(X) 0 +#else +static int notNormalFile(const char *zFile){ + struct stat x; + int rc; + memset(&x, 0, sizeof(x)); + rc = stat(zFile, &x); + return rc || !S_ISREG(x.st_mode); +} +#endif + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. +*/ +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; + + while( 1 ){ + if( n+100>nLine ){ + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + if( zLine==0 ) shell_out_of_memory(); + } + if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } + } +#if defined(_WIN32) || defined(WIN32) + /* For interactive input on Windows systems, translate the + ** multi-byte characterset characters into UTF-8. */ + if( stdin_is_interactive && in==stdin ){ + char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); + if( zTrans ){ + int nTrans = strlen30(zTrans)+1; + if( nTrans>nLine ){ + zLine = realloc(zLine, nTrans); + if( zLine==0 ) shell_out_of_memory(); + } + memcpy(zLine, zTrans, nTrans); + sqlite3_free(zTrans); + } + } +#endif /* defined(_WIN32) || defined(WIN32) */ + return zLine; +} + +/* +** Retrieve a single line of input text. +** +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. +** +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. +** +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. +*/ +static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + if( in!=0 ){ + zResult = local_getline(zPrior, in); + }else{ + zPrompt = isContinuation ? continuePrompt : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + printf("%s", zPrompt); + fflush(stdout); + zResult = local_getline(zPrior, stdin); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + if( zResult && *zResult ) shell_add_history(zResult); +#endif + } + return zResult; +} + + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; iz); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ) shell_out_of_memory(); + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + +/* +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +*/ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; + + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.z = 0; + } + return s.z; +} + +/* +** SQL function: shell_module_schema(X) +** +** Return a fake schema for the table-valued function or eponymous virtual +** table X. +*/ +static void shellModuleSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zName = (const char*)sqlite3_value_text(apVal[0]); + char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName); + UNUSED_PARAMETER(nVal); + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} + +/* +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. +*/ +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){ + int n = strlen30(aPrefix[i]); + if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){ + char *z = 0; + char *zFake = 0; + if( zSchema ){ + char cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){ + z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8); + }else{ + z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8); + } + } + if( zName + && aPrefix[i][0]=='V' + && (zFake = shellFakeSchema(db, zSchema, zName))!=0 + ){ + if( z==0 ){ + z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake); + }else{ + z = sqlite3_mprintf("%z\n/* %s */", z, zFake); + } + free(zFake); + } + if( z ){ + sqlite3_result_text(pCtx, z, -1, sqlite3_free); + return; + } + } + } + } + sqlite3_result_value(pCtx, apVal[0]); +} + +/* +** The source code for several run-time loadable extensions is inserted +** below by the ../tool/mkshellc.tcl script. Before processing that included +** code, we need to override some macros to make the included program code +** work here in the middle of this regular program. +*/ +#define SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT2(X) (void)(X) + +#if defined(_WIN32) && defined(_MSC_VER) +/************************* Begin test_windirent.h ******************/ +/* +** 2015 November 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains declarations for most of the opendir() family of +** POSIX functions on Win32 using the MSVCRT. +*/ + +#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) +#define SQLITE_WINDIRENT_H + +/* +** We need several data types from the Windows SDK header. +*/ + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +/* +** We need several support functions from the SQLite core. +*/ + +/* +** We need several things from the ANSI and MSVCRT headers. +*/ + +#include "libc/calls/weirdtypes.h" +#include "libc/errno.h" +#include "libc/limits.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" + +/* +** We may need several defines that should have been in "sys/stat.h". +*/ + +#ifndef S_ISREG +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif + +#ifndef S_ISDIR +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif + +#ifndef S_ISLNK +#define S_ISLNK(mode) (0) +#endif + +/* +** We may need to provide the "mode_t" type. +*/ + +#ifndef MODE_T_DEFINED + #define MODE_T_DEFINED + typedef unsigned short mode_t; +#endif + +/* +** We may need to provide the "ino_t" type. +*/ + +#ifndef INO_T_DEFINED + #define INO_T_DEFINED + typedef unsigned short ino_t; +#endif + +/* +** We need to define "NAME_MAX" if it was not present in "limits.h". +*/ + +#ifndef NAME_MAX +# ifdef FILENAME_MAX +# define NAME_MAX (FILENAME_MAX) +# else +# define NAME_MAX (260) +# endif +#endif + +/* +** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". +*/ + +#ifndef NULL_INTPTR_T +# define NULL_INTPTR_T ((intptr_t)(0)) +#endif + +#ifndef BAD_INTPTR_T +# define BAD_INTPTR_T ((intptr_t)(-1)) +#endif + +/* +** We need to provide the necessary structures and related types. +*/ + +#ifndef DIRENT_DEFINED +#define DIRENT_DEFINED +typedef struct DIRENT DIRENT; +typedef DIRENT *LPDIRENT; +struct DIRENT { + ino_t d_ino; /* Sequence number, do not use. */ + unsigned d_attributes; /* Win32 file attributes. */ + char d_name[NAME_MAX + 1]; /* Name within the directory. */ +}; +#endif + +#ifndef DIR_DEFINED +#define DIR_DEFINED +typedef struct DIR DIR; +typedef DIR *LPDIR; +struct DIR { + intptr_t d_handle; /* Value returned by "_findfirst". */ + DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ + DIRENT d_next; /* DIRENT constructed based on "_findnext". */ +}; +#endif + +/* +** Provide a macro, for use by the implementation, to determine if a +** particular directory entry should be skipped over when searching for +** the next directory entry that should be returned by the readdir() or +** readdir_r() functions. +*/ + +#ifndef is_filtered +# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) +#endif + +/* +** Provide the function prototype for the POSIX compatiable getenv() +** function. This function is not thread-safe. +*/ + +extern const char *windirent_getenv(const char *name); + +/* +** Finally, we can provide the function prototypes for the opendir(), +** readdir(), readdir_r(), and closedir() POSIX functions. +*/ + +extern LPDIR opendir(const char *dirname); +extern LPDIRENT readdir(LPDIR dirp); +extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); +extern INT closedir(LPDIR dirp); + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.h ********************/ +/************************* Begin test_windirent.c ******************/ +/* +** 2015 November 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement most of the opendir() family of +** POSIX functions on Win32 using the MSVCRT. +*/ + +#if defined(_WIN32) && defined(_MSC_VER) +/* #include "third_party/sqlite3/test_windirent.inc" */ + +/* +** Implementation of the POSIX getenv() function using the Win32 API. +** This function is not thread-safe. +*/ +const char *windirent_getenv( + const char *name +){ + static char value[32768]; /* Maximum length, per MSDN */ + DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ + DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ + + memset(value, 0, sizeof(value)); + dwRet = GetEnvironmentVariableA(name, value, dwSize); + if( dwRet==0 || dwRet>dwSize ){ + /* + ** The function call to GetEnvironmentVariableA() failed -OR- + ** the buffer is not large enough. Either way, return NULL. + */ + return 0; + }else{ + /* + ** The function call to GetEnvironmentVariableA() succeeded + ** -AND- the buffer contains the entire value. + */ + return value; + } +} + +/* +** Implementation of the POSIX opendir() function using the MSVCRT. +*/ +LPDIR opendir( + const char *dirname +){ + struct _finddata_t data; + LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); + SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); + + if( dirp==NULL ) return NULL; + memset(dirp, 0, sizeof(DIR)); + + /* TODO: Remove this if Unix-style root paths are not used. */ + if( sqlite3_stricmp(dirname, "/")==0 ){ + dirname = windirent_getenv("SystemDrive"); + } + + memset(&data, 0, sizeof(struct _finddata_t)); + _snprintf(data.name, namesize, "%s\\*", dirname); + dirp->d_handle = _findfirst(data.name, &data); + + if( dirp->d_handle==BAD_INTPTR_T ){ + closedir(dirp); + return NULL; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ){ +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + closedir(dirp); + return NULL; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + } + + dirp->d_first.d_attributes = data.attrib; + strncpy(dirp->d_first.d_name, data.name, NAME_MAX); + dirp->d_first.d_name[NAME_MAX] = '\0'; + + return dirp; +} + +/* +** Implementation of the POSIX readdir() function using the MSVCRT. +*/ +LPDIRENT readdir( + LPDIR dirp +){ + struct _finddata_t data; + + if( dirp==NULL ) return NULL; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + return &dirp->d_first; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + dirp->d_next.d_ino++; + dirp->d_next.d_attributes = data.attrib; + strncpy(dirp->d_next.d_name, data.name, NAME_MAX); + dirp->d_next.d_name[NAME_MAX] = '\0'; + + return &dirp->d_next; +} + +/* +** Implementation of the POSIX readdir_r() function using the MSVCRT. +*/ +INT readdir_r( + LPDIR dirp, + LPDIRENT entry, + LPDIRENT *result +){ + struct _finddata_t data; + + if( dirp==NULL ) return EBADF; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + entry->d_ino = dirp->d_first.d_ino; + entry->d_attributes = dirp->d_first.d_attributes; + strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + *result = NULL; + return ENOENT; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + entry->d_ino = (ino_t)-1; /* not available */ + entry->d_attributes = data.attrib; + strncpy(entry->d_name, data.name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; +} + +/* +** Implementation of the POSIX closedir() function using the MSVCRT. +*/ +INT closedir( + LPDIR dirp +){ + INT result = 0; + + if( dirp==NULL ) return EINVAL; + + if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ + result = _findclose(dirp->d_handle); + } + + sqlite3_free(dirp); + return result; +} + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.c ********************/ +#define dirent DIRENT +#endif +/************************* Begin ../ext/misc/shathree.c ******************/ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions that compute SHA3 hashes. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_query(Y,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. +** +** The sha3_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. +*/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/str/str.h" + +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_uint64 u64; */ +#endif /* SQLITE_AMALGAMATION */ + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + c0 = a00^a20^a40^a10^a30; + c1 = a11^a31^a01^a21^a41; + c2 = a22^a42^a12^a32^a02; + c3 = a33^a03^a23^a43^a13; + c4 = a44^a14^a34^a04^a24; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a31^d1), 44); + b2 = ROL64((a12^d2), 43); + b3 = ROL64((a43^d3), 21); + b4 = ROL64((a24^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+1]; + a31 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a40^d0), 3); + b3 = ROL64((a21^d1), 45); + b4 = ROL64((a02^d2), 61); + b0 = ROL64((a33^d3), 28); + b1 = ROL64((a14^d4), 20); + a40 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a30^d0), 18); + b0 = ROL64((a11^d1), 1); + b1 = ROL64((a42^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a04^d4), 8); + a30 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a20^d0), 36); + b2 = ROL64((a01^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a13^d3), 56); + b0 = ROL64((a44^d4), 27); + a20 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a10^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a22^d2), 62); + b1 = ROL64((a03^d3), 55); + b2 = ROL64((a34^d4), 39); + a10 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + c0 = a00^a40^a30^a20^a10; + c1 = a31^a21^a11^a01^a41; + c2 = a12^a02^a42^a32^a22; + c3 = a43^a33^a23^a13^a03; + c4 = a24^a14^a04^a44^a34; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a21^d1), 44); + b2 = ROL64((a42^d2), 43); + b3 = ROL64((a13^d3), 21); + b4 = ROL64((a34^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+2]; + a21 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a30^d0), 3); + b3 = ROL64((a01^d1), 45); + b4 = ROL64((a22^d2), 61); + b0 = ROL64((a43^d3), 28); + b1 = ROL64((a14^d4), 20); + a30 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a10^d0), 18); + b0 = ROL64((a31^d1), 1); + b1 = ROL64((a02^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a44^d4), 8); + a10 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a40^d0), 36); + b2 = ROL64((a11^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a03^d3), 56); + b0 = ROL64((a24^d4), 27); + a40 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a20^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a12^d2), 62); + b1 = ROL64((a33^d3), 55); + b2 = ROL64((a04^d4), 39); + a20 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + c0 = a00^a30^a10^a40^a20; + c1 = a21^a01^a31^a11^a41; + c2 = a42^a22^a02^a32^a12; + c3 = a13^a43^a23^a03^a33; + c4 = a34^a14^a44^a24^a04; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a01^d1), 44); + b2 = ROL64((a02^d2), 43); + b3 = ROL64((a03^d3), 21); + b4 = ROL64((a04^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+3]; + a01 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a10^d0), 3); + b3 = ROL64((a11^d1), 45); + b4 = ROL64((a12^d2), 61); + b0 = ROL64((a13^d3), 28); + b1 = ROL64((a14^d4), 20); + a10 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a20^d0), 18); + b0 = ROL64((a21^d1), 1); + b1 = ROL64((a22^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a24^d4), 8); + a20 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a30^d0), 36); + b2 = ROL64((a31^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a33^d3), 56); + b0 = ROL64((a34^d4), 27); + a30 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a40^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a42^d2), 62); + b1 = ROL64((a43^d3), 55); + b2 = ROL64((a44^d4), 39); + a40 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z ){ + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + } + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ../ext/misc/shathree.c ********************/ +/************************* Begin ../ext/misc/fileio.c ******************/ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL functions readfile() and +** writefile(), and eponymous virtual type "fsdir". +** +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): +** +** If neither of the optional arguments is present, then this UDF +** function writes blob DATA to file FILE. If successful, the number +** of bytes written is returned. If an error occurs, NULL is returned. +** +** If the first option argument - MODE - is present, then it must +** be passed an integer value that corresponds to a POSIX mode +** value (file type + permissions, as returned in the stat.st_mode +** field by the stat() system call). Three types of files may +** be written/created: +** +** regular files: (mode & 0170000)==0100000 +** symbolic links: (mode & 0170000)==0120000 +** directories: (mode & 0170000)==0040000 +** +** For a directory, the DATA is ignored. For a symbolic link, it is +** interpreted as text and used as the target of the link. For a +** regular file, it is interpreted as a blob and written into the +** named file. Regardless of the type of file, its permissions are +** set to (mode & 0777) before returning. +** +** If the optional MTIME argument is present, then it is interpreted +** as an integer - the number of seconds since the unix epoch. The +** modification-time of the target file is set to this value before +** returning. +** +** If three or more arguments are passed to this function and an +** error is encountered, an exception is raised. +** +** READFILE(FILE): +** +** Read and return the contents of file FILE (type blob) from disk. +** +** FSDIR: +** +** Used as follows: +** +** SELECT * FROM fsdir($path [, $dir]); +** +** Parameter $path is an absolute or relative pathname. If the file that it +** refers to does not exist, it is an error. If the path refers to a regular +** file or symbolic link, it returns a single row. Or, if the path refers +** to a directory, it returns one row for the directory, and one row for each +** file within the hierarchy rooted at $path. +** +** Each row has the following columns: +** +** name: Path to file or directory (text value). +** mode: Value of stat.st_mode for directory entry (an integer). +** mtime: Value of stat.st_mtime for directory entry (an integer). +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. +** +** If a non-NULL value is specified for the optional $dir parameter and +** $path is a relative path, then $path is interpreted relative to $dir. +** And the paths returned in the "name" column of the table are also +** relative to directory $dir. +*/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "libc/calls/sigbits.h" +#include "libc/isystem/unistd.h" +#include "libc/time/time.h" +#include "libc/errno.h" +#include "libc/time/time.h" + +/* +** Structure of the fsdir() table-valued function +*/ + /* 0 1 2 3 4 5 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +#define FSDIR_COLUMN_NAME 0 /* Name of the file */ +#define FSDIR_COLUMN_MODE 1 /* Access mode */ +#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ +#define FSDIR_COLUMN_DATA 3 /* File content */ +#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ + + +/* +** Set the result stored by context ctx to a blob containing the +** contents of file zName. Or, leave the result unchanged (NULL) +** if the file does not exist or is unreadable. +** +** If the file exceeds the SQLite blob size limit, through an +** SQLITE_TOOBIG error. +** +** Throw an SQLITE_IOERR if there are difficulties pulling the file +** off of disk. +*/ +static void readFileContents(sqlite3_context *ctx, const char *zName){ + FILE *in; + sqlite3_int64 nIn; + void *pBuf; + sqlite3 *db; + int mxBlob; + + in = fopen(zName, "rb"); + if( in==0 ){ + /* File does not exist or is unreadable. Leave the result set to NULL. */ + return; + } + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + db = sqlite3_context_db_handle(ctx); + mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + if( nIn>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + fclose(in); + return; + } + pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + if( pBuf==0 ){ + sqlite3_result_error_nomem(ctx); + fclose(in); + return; + } + if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ + sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_result_error_code(ctx, SQLITE_IOERR); + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + readFileContents(context, zName); +} + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +#if defined(_WIN32) +/* +** This function is designed to convert a Win32 FILETIME structure into the +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). +*/ +static sqlite3_uint64 fileTimeToUnixTime( + LPFILETIME pFileTime +){ + SYSTEMTIME epochSystemTime; + ULARGE_INTEGER epochIntervals; + FILETIME epochFileTime; + ULARGE_INTEGER fileIntervals; + + memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); + epochSystemTime.wYear = 1970; + epochSystemTime.wMonth = 1; + epochSystemTime.wDay = 1; + SystemTimeToFileTime(&epochSystemTime, &epochFileTime); + epochIntervals.LowPart = epochFileTime.dwLowDateTime; + epochIntervals.HighPart = epochFileTime.dwHighDateTime; + + fileIntervals.LowPart = pFileTime->dwLowDateTime; + fileIntervals.HighPart = pFileTime->dwHighDateTime; + + return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; +} + +/* +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. +*/ +static void statTimesToUtc( + const char *zPath, + struct stat *pStatBuf +){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(zUnicodeName, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + sqlite3_free(zUnicodeName); + } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = stat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return stat(zPath, pStatBuf); +#endif +} + +/* +** This function is used in place of lstat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls lstat(). +*/ +static int fileLinkStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = lstat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return lstat(zPath, pStatBuf); +#endif +} + +/* +** Argument zFile is the name of a file that will be created and/or written +** by SQL function writefile(). This function ensures that the directory +** zFile will be written to exists, creating it if required. The permissions +** for any path components created by this function are set in accordance +** with the current umask. +** +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, +** SQLITE_OK is returned if the directory is successfully created, or +** SQLITE_ERROR otherwise. +*/ +static int makeDirectory( + const char *zFile +){ + char *zCopy = sqlite3_mprintf("%s", zFile); + int rc = SQLITE_OK; + + if( zCopy==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nCopy = (int)strlen(zCopy); + int i = 1; + + while( rc==SQLITE_OK ){ + struct stat sStat; + int rc2; + + for(; zCopy[i]!='/' && i=0 ){ +#if defined(_WIN32) +#if !SQLITE_OS_WINRT + /* Windows */ + FILETIME lastAccess; + FILETIME lastWrite; + SYSTEMTIME currentTime; + LONGLONG intervals; + HANDLE hFile; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + + GetSystemTime(¤tTime); + SystemTimeToFileTime(¤tTime, &lastAccess); + intervals = Int32x32To64(mtime, 10000000) + 116444736000000000; + lastWrite.dwLowDateTime = (DWORD)intervals; + lastWrite.dwHighDateTime = intervals >> 32; + zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); + if( zUnicodeName==0 ){ + return 1; + } + hFile = CreateFileW( + zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL + ); + sqlite3_free(zUnicodeName); + if( hFile!=INVALID_HANDLE_VALUE ){ + BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite); + CloseHandle(hFile); + return !bResult; + }else{ + return 1; + } +#endif +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ + /* Recent unix */ + struct timespec times[2]; + times[0].tv_nsec = times[1].tv_nsec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){ + return 1; + } +#else + /* Legacy unix */ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } +#endif + } + + return 0; +} + +/* +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. +** Refer to header comments at the top of this file for details. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zFile; + mode_t mode = 0; + int res; + sqlite3_int64 mtime = -1; + + if( argc<2 || argc>4 ){ + sqlite3_result_error(context, + "wrong number of arguments to function writefile()", -1 + ); + return; + } + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + if( argc>=3 ){ + mode = (mode_t)sqlite3_value_int(argv[2]); + } + if( argc==4 ){ + mtime = sqlite3_value_int64(argv[3]); + } + + res = writeFile(context, zFile, argv[1], mode, mtime); + if( res==1 && errno==ENOENT ){ + if( makeDirectory(zFile)==SQLITE_OK ){ + res = writeFile(context, zFile, argv[1], mode, mtime); + } + } + + if( argc>2 && res!=0 ){ + if( S_ISLNK(mode) ){ + ctxErrorMsg(context, "failed to create symlink: %s", zFile); + }else if( S_ISDIR(mode) ){ + ctxErrorMsg(context, "failed to create directory: %s", zFile); + }else{ + ctxErrorMsg(context, "failed to write file: %s", zFile); + } + } +} + +/* +** SQL function: lsmode(MODE) +** +** Given a numberic st_mode from stat(), convert it into a human-readable +** text string in the style of "ls -l". +*/ +static void lsModeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int iMode = sqlite3_value_int(argv[0]); + char z[16]; + (void)argc; + if( S_ISLNK(iMode) ){ + z[0] = 'l'; + }else if( S_ISREG(iMode) ){ + z[0] = '-'; + }else if( S_ISDIR(iMode) ){ + z[0] = 'd'; + }else{ + z[0] = '?'; + } + for(i=0; i<3; i++){ + int m = (iMode >> ((2-i)*3)); + char *a = &z[1 + i*3]; + a[0] = (m & 0x4) ? 'r' : '-'; + a[1] = (m & 0x2) ? 'w' : '-'; + a[2] = (m & 0x1) ? 'x' : '-'; + } + z[10] = '\0'; + sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT); +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Cursor type for recursively iterating through a directory structure. +*/ +typedef struct fsdir_cursor fsdir_cursor; +typedef struct FsdirLevel FsdirLevel; + +struct FsdirLevel { + DIR *pDir; /* From opendir() */ + char *zDir; /* Name of directory (nul-terminated) */ +}; + +struct fsdir_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + + int nLvl; /* Number of entries in aLvl[] array */ + int iLvl; /* Index of current entry */ + FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ + + const char *zBase; + int nBase; + + struct stat sStat; /* Current lstat() results */ + char *zPath; /* Path to current entry */ + sqlite3_int64 iRowid; /* Current rowid */ +}; + +typedef struct fsdir_tab fsdir_tab; +struct fsdir_tab { + sqlite3_vtab base; /* Base class - must be first */ +}; + +/* +** Construct a new fsdir virtual table object. +*/ +static int fsdirConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fsdir_tab *pNew = 0; + int rc; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** This method is the destructor for fsdir vtab objects. +*/ +static int fsdirDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new fsdir_cursor object. +*/ +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + fsdir_cursor *pCur; + (void)p; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iLvl = -1; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset a cursor back to the state it was in when first returned +** by fsdirOpen(). +*/ +static void fsdirResetCursor(fsdir_cursor *pCur){ + int i; + for(i=0; i<=pCur->iLvl; i++){ + FsdirLevel *pLvl = &pCur->aLvl[i]; + if( pLvl->pDir ) closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + } + sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); + pCur->aLvl = 0; + pCur->zPath = 0; + pCur->zBase = 0; + pCur->nBase = 0; + pCur->nLvl = 0; + pCur->iLvl = -1; + pCur->iRowid = 1; +} + +/* +** Destructor for an fsdir_cursor. +*/ +static int fsdirClose(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + + fsdirResetCursor(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Set the error message for the virtual table associated with cursor +** pCur to the results of vprintf(zFmt, ...). +*/ +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + + +/* +** Advance an fsdir_cursor to its next row of output. +*/ +static int fsdirNext(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + mode_t m = pCur->sStat.st_mode; + + pCur->iRowid++; + if( S_ISDIR(m) ){ + /* Descend into this directory */ + int iNew = pCur->iLvl + 1; + FsdirLevel *pLvl; + if( iNew>=pCur->nLvl ){ + int nNew = iNew+1; + sqlite3_int64 nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc64(pCur->aLvl, nByte); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); + pCur->aLvl = aNew; + pCur->nLvl = nNew; + } + pCur->iLvl = iNew; + pLvl = &pCur->aLvl[iNew]; + + pLvl->zDir = pCur->zPath; + pCur->zPath = 0; + pLvl->pDir = opendir(pLvl->zDir); + if( pLvl->pDir==0 ){ + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + return SQLITE_ERROR; + } + } + + while( pCur->iLvl>=0 ){ + FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl]; + struct dirent *pEntry = readdir(pLvl->pDir); + if( pEntry ){ + if( pEntry->d_name[0]=='.' ){ + if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue; + if( pEntry->d_name[1]=='\0' ) continue; + } + sqlite3_free(pCur->zPath); + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + pLvl->pDir = 0; + pLvl->zDir = 0; + pCur->iLvl--; + } + + /* EOF */ + sqlite3_free(pCur->zPath); + pCur->zPath = 0; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int fsdirColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + switch( i ){ + case FSDIR_COLUMN_NAME: { + sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); + break; + } + + case FSDIR_COLUMN_MODE: + sqlite3_result_int64(ctx, pCur->sStat.st_mode); + break; + + case FSDIR_COLUMN_MTIME: + sqlite3_result_int64(ctx, pCur->sStat.st_mtime); + break; + + case FSDIR_COLUMN_DATA: { + mode_t m = pCur->sStat.st_mode; + if( S_ISDIR(m) ){ + sqlite3_result_null(ctx); +#if !defined(_WIN32) && !defined(WIN32) + }else if( S_ISLNK(m) ){ + char aStatic[64]; + char *aBuf = aStatic; + sqlite3_int64 nBuf = 64; + int n; + + while( 1 ){ + n = readlink(pCur->zPath, aBuf, nBuf); + if( nzPath); + } + } + case FSDIR_COLUMN_PATH: + default: { + /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. + ** always return their values as NULL */ + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fsdirEof(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + return (pCur->zPath==0); +} + +/* +** xFilter callback. +** +** idxNum==1 PATH parameter only +** idxNum==2 Both PATH and DIR supplied +*/ +static int fsdirFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + const char *zDir = 0; + fsdir_cursor *pCur = (fsdir_cursor*)cur; + (void)idxStr; + fsdirResetCursor(pCur); + + if( idxNum==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); + return SQLITE_ERROR; + } + + assert( argc==idxNum && (argc==1 || argc==2) ); + zDir = (const char*)sqlite3_value_text(argv[0]); + if( zDir==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); + return SQLITE_ERROR; + } + if( argc==2 ){ + pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + } + if( pCur->zBase ){ + pCur->nBase = (int)strlen(pCur->zBase)+1; + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", zDir); + } + + if( pCur->zPath==0 ){ + return SQLITE_NOMEM; + } + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by values of idxNum: +** +** (1) The path value is supplied by argv[0] +** (2) Path is in argv[0] and dir is in argv[1] +*/ +static int fsdirBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ + int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int seenPath = 0; /* True if an unusable PATH= constraint is seen */ + int seenDir = 0; /* True if an unusable DIR= constraint is seen */ + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; + } + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + } + } + if( seenPath || seenDir ){ + /* If input parameters are unusable, disallow this plan */ + return SQLITE_CONSTRAINT; + } + + if( idxPath<0 ){ + pIdxInfo->idxNum = 0; + /* The pIdxInfo->estimatedCost should have been initialized to a huge + ** number. Leave it unchanged. */ + pIdxInfo->estimatedRows = 0x7fffffff; + }else{ + pIdxInfo->aConstraintUsage[idxPath].omit = 1; + pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + if( idxDir>=0 ){ + pIdxInfo->aConstraintUsage[idxDir].omit = 1; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; + pIdxInfo->idxNum = 2; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 100.0; + } + } + + return SQLITE_OK; +} + +/* +** Register the "fsdir" virtual table. +*/ +static int fsdirRegister(sqlite3 *db){ + static sqlite3_module fsdirModule = { + 0, /* iVersion */ + 0, /* xCreate */ + fsdirConnect, /* xConnect */ + fsdirBestIndex, /* xBestIndex */ + fsdirDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + fsdirOpen, /* xOpen - open a cursor */ + fsdirClose, /* xClose - close a cursor */ + fsdirFilter, /* xFilter - configure scan constraints */ + fsdirNext, /* xNext - advance a cursor */ + fsdirEof, /* xEof - check for end of scan */ + fsdirColumn, /* xColumn - read data */ + fsdirRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + }; + + int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define fsdirRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 + +#endif +int sqlite3_fileio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readfile", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writefile", -1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + writefileFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0, + lsModeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = fsdirRegister(db); + } + return rc; +} + +/************************* End ../ext/misc/fileio.c ********************/ +/************************* Begin ../ext/misc/completion.c ******************/ +/* +** 2017-07-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an eponymous virtual table that returns suggested +** completions for a partial SQL input. +** +** Suggested usage: +** +** SELECT DISTINCT candidate COLLATE nocase +** FROM completion($prefix,$wholeline) +** ORDER BY 1; +** +** The two query parameters are optional. $prefix is the text of the +** current word being typed and that is to be completed. $wholeline is +** the complete input line, used for context. +** +** The raw completion() table might return the same candidate multiple +** times, for example if the same column name is used to two or more +** tables. And the candidates are returned in an arbitrary order. Hence, +** the DISTINCT and ORDER BY are recommended. +** +** This virtual table operates at the speed of human typing, and so there +** is no attempt to make it fast. Even a slow implementation will be much +** faster than any human can type. +** +*/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/str/str.h" + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* completion_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a completion virtual table +*/ +typedef struct completion_vtab completion_vtab; +struct completion_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this completion vtab */ +}; + +/* completion_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct completion_cursor completion_cursor; +struct completion_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int nPrefix, nLine; /* Number of bytes in zPrefix and zLine */ + char *zPrefix; /* The prefix for the word we want to complete */ + char *zLine; /* The whole that we want to complete */ + const char *zCurrentRow; /* Current output row */ + int szRow; /* Length of the zCurrentRow string */ + sqlite3_stmt *pStmt; /* Current statement */ + sqlite3_int64 iRowid; /* The rowid */ + int ePhase; /* Current phase */ + int j; /* inter-phase counter */ +}; + +/* Values for ePhase: +*/ +#define COMPLETION_FIRST_PHASE 1 +#define COMPLETION_KEYWORDS 1 +#define COMPLETION_PRAGMAS 2 +#define COMPLETION_FUNCTIONS 3 +#define COMPLETION_COLLATIONS 4 +#define COMPLETION_INDEXES 5 +#define COMPLETION_TRIGGERS 6 +#define COMPLETION_DATABASES 7 +#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */ +#define COMPLETION_COLUMNS 9 +#define COMPLETION_MODULES 10 +#define COMPLETION_EOF 11 + +/* +** The completionConnect() method is invoked to create a new +** completion_vtab that describes the completion virtual table. +** +** Think of this routine as the constructor for completion_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the completion_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against completion will look like. +*/ +static int completionConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + completion_vtab *pNew; + int rc; + + (void)(pAux); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + (void)(argv); /* Unused parameter */ + (void)(pzErr); /* Unused parameter */ + +/* Column numbers */ +#define COMPLETION_COLUMN_CANDIDATE 0 /* Suggested completion of the input */ +#define COMPLETION_COLUMN_PREFIX 1 /* Prefix of the word to be completed */ +#define COMPLETION_COLUMN_WHOLELINE 2 /* Entire line seen so far */ +#define COMPLETION_COLUMN_PHASE 3 /* ePhase - used for debugging only */ + + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(" + " candidate TEXT," + " prefix TEXT HIDDEN," + " wholeline TEXT HIDDEN," + " phase INT HIDDEN" /* Used for debugging only */ + ")"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for completion_cursor objects. +*/ +static int completionDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new completion_cursor object. +*/ +static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + completion_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((completion_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset the completion_cursor. +*/ +static void completionCursorReset(completion_cursor *pCur){ + sqlite3_free(pCur->zPrefix); pCur->zPrefix = 0; pCur->nPrefix = 0; + sqlite3_free(pCur->zLine); pCur->zLine = 0; pCur->nLine = 0; + sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; + pCur->j = 0; +} + +/* +** Destructor for a completion_cursor. +*/ +static int completionClose(sqlite3_vtab_cursor *cur){ + completionCursorReset((completion_cursor*)cur); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Advance a completion_cursor to its next row of output. +** +** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object +** record the current state of the scan. This routine sets ->zCurrentRow +** to the current row of output and then returns. If no more rows remain, +** then ->ePhase is set to COMPLETION_EOF which will signal the virtual +** table that has reached the end of its scan. +** +** The current implementation just lists potential identifiers and +** keywords and filters them by zPrefix. Future enhancements should +** take zLine into account to try to restrict the set of identifiers and +** keywords based on what would be legal at the current point of input. +*/ +static int completionNext(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + int eNextPhase = 0; /* Next phase to try if current phase reaches end */ + int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + pCur->iRowid++; + while( pCur->ePhase!=COMPLETION_EOF ){ + switch( pCur->ePhase ){ + case COMPLETION_KEYWORDS: { + if( pCur->j >= sqlite3_keyword_count() ){ + pCur->zCurrentRow = 0; + pCur->ePhase = COMPLETION_DATABASES; + }else{ + sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow); + } + iCol = -1; + break; + } + case COMPLETION_DATABASES: { + if( pCur->pStmt==0 ){ + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, + &pCur->pStmt, 0); + } + iCol = 1; + eNextPhase = COMPLETION_TABLES; + break; + } + case COMPLETION_TABLES: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT name FROM \"%w\".sqlite_schema", + zSql, zSep, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_COLUMNS; + break; + } + case COMPLETION_COLUMNS: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" + " JOIN pragma_table_info(sm.name,%Q) AS pti" + " WHERE sm.type='table'", + zSql, zSep, zDb, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_EOF; + break; + } + } + if( iCol<0 ){ + /* This case is when the phase presets zCurrentRow */ + if( pCur->zCurrentRow==0 ) continue; + }else{ + if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){ + /* Extract the next row of content */ + pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol); + pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); + }else{ + /* When all rows are finished, advance to the next phase */ + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + pCur->ePhase = eNextPhase; + continue; + } + } + if( pCur->nPrefix==0 ) break; + if( pCur->nPrefix<=pCur->szRow + && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 + ){ + break; + } + } + + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the completion_cursor +** is currently pointing. +*/ +static int completionColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + completion_cursor *pCur = (completion_cursor*)cur; + switch( i ){ + case COMPLETION_COLUMN_CANDIDATE: { + sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PREFIX: { + sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_WHOLELINE: { + sqlite3_result_text(ctx, pCur->zLine, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PHASE: { + sqlite3_result_int(ctx, pCur->ePhase); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + completion_cursor *pCur = (completion_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int completionEof(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + return pCur->ePhase >= COMPLETION_EOF; +} + +/* +** This method is called to "rewind" the completion_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to completionColumn() or completionRowid() or +** completionEof(). +*/ +static int completionFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + completion_cursor *pCur = (completion_cursor *)pVtabCursor; + int iArg = 0; + (void)(idxStr); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + completionCursorReset(pCur); + if( idxNum & 1 ){ + pCur->nPrefix = sqlite3_value_bytes(argv[iArg]); + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + iArg = 1; + } + if( idxNum & 2 ){ + pCur->nLine = sqlite3_value_bytes(argv[iArg]); + if( pCur->nLine>0 ){ + pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zLine==0 ) return SQLITE_NOMEM; + } + } + if( pCur->zLine!=0 && pCur->zPrefix==0 ){ + int i = pCur->nLine; + while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ + i--; + } + pCur->nPrefix = pCur->nLine - i; + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + } + pCur->iRowid = 0; + pCur->ePhase = COMPLETION_FIRST_PHASE; + return completionNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the completion virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** There are two hidden parameters that act as arguments to the table-valued +** function: "prefix" and "wholeline". Bit 0 of idxNum is set if "prefix" +** is available and bit 1 is set if "wholeline" is available. +*/ +static int completionBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int prefixIdx = -1; /* Index of the start= constraint, or -1 if none */ + int wholelineIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that completeFilter() expects */ + const struct sqlite3_index_constraint *pConstraint; + + (void)(tab); /* Unused parameter */ + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case COMPLETION_COLUMN_PREFIX: + prefixIdx = i; + idxNum |= 1; + break; + case COMPLETION_COLUMN_WHOLELINE: + wholelineIdx = i; + idxNum |= 2; + break; + } + } + if( prefixIdx>=0 ){ + pIdxInfo->aConstraintUsage[prefixIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[prefixIdx].omit = 1; + } + if( wholelineIdx>=0 ){ + pIdxInfo->aConstraintUsage[wholelineIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[wholelineIdx].omit = 1; + } + pIdxInfo->idxNum = idxNum; + pIdxInfo->estimatedCost = (double)5000 - 1000*nArg; + pIdxInfo->estimatedRows = 500 - 100*nArg; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** completion virtual table. +*/ +static sqlite3_module completionModule = { + 0, /* iVersion */ + 0, /* xCreate */ + completionConnect, /* xConnect */ + completionBestIndex, /* xBestIndex */ + completionDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + completionOpen, /* xOpen - open a cursor */ + completionClose, /* xClose - close a cursor */ + completionFilter, /* xFilter - configure scan constraints */ + completionNext, /* xNext - advance a cursor */ + completionEof, /* xEof - check for end of scan */ + completionColumn, /* xColumn - read data */ + completionRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3CompletionVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "completion", &completionModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 + +#endif +int sqlite3_completion_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)(pzErrMsg); /* Unused parameter */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3CompletionVtabInit(db); +#endif + return rc; +} + +/************************* End ../ext/misc/completion.c ********************/ +/************************* Begin ../ext/misc/appendvfs.c ******************/ +/* +** 2017-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that allows an SQLite database to be +** appended onto the end of some other file, such as an executable. +** +** A special record must appear at the end of the file that identifies the +** file as an appended database and provides the offset to the first page +** of the exposed content. (Or, it is the length of the content prefix.) +** For best performance page 1 should be located at a disk page boundary, +** though that is not required. +** +** When opening a database using this VFS, the connection might treat +** the file as an ordinary SQLite database, or it might treat it as a +** database appended onto some other file. The decision is made by +** applying the following rules in order: +** +** (1) An empty file is an ordinary database. +** +** (2) If the file ends with the appendvfs trailer string +** "Start-Of-SQLite3-NNNNNNNN" that file is an appended database. +** +** (3) If the file begins with the standard SQLite prefix string +** "SQLite format 3", that file is an ordinary database. +** +** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is +** set, then a new database is appended to the already existing file. +** +** (5) Otherwise, SQLITE_CANTOPEN is returned. +** +** To avoid unnecessary complications with the PENDING_BYTE, the size of +** the file containing the database is limited to 1GiB. (1073741824 bytes) +** This VFS will not read or write past the 1GiB mark. This restriction +** might be lifted in future versions. For now, if you need a larger +** database, then keep it in a separate file. +** +** If the file being opened is a plain database (not an appended one), then +** this shim is a pass-through into the default underlying VFS. (rule 3) +**/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/str/str.h" + +/* The append mark at the end of the database is: +** +** Start-Of-SQLite3-NNNNNNNN +** 123456789 123456789 12345 +** +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is +** the offset to page 1, and also the length of the prefix content. +*/ +#define APND_MARK_PREFIX "Start-Of-SQLite3-" +#define APND_MARK_PREFIX_SZ 17 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) + +/* +** Maximum size of the combined prefix + database + append-mark. This +** must be less than 0x40000000 to avoid locking issues on Windows. +*/ +#define APND_MAX_SIZE (0x40000000) + +/* +** Try to align the database to an even multiple of APND_ROUNDUP bytes. +*/ +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 +#endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs ApndVfs; +typedef struct ApndFile ApndFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) + +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ +struct ApndFile { + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ +}; + +/* +** Methods for ApndFile +*/ +static int apndClose(sqlite3_file*); +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int apndTruncate(sqlite3_file*, sqlite3_int64 size); +static int apndSync(sqlite3_file*, int flags); +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int apndLock(sqlite3_file*, int); +static int apndUnlock(sqlite3_file*, int); +static int apndCheckReservedLock(sqlite3_file*, int *pResOut); +static int apndFileControl(sqlite3_file*, int op, void *pArg); +static int apndSectorSize(sqlite3_file*); +static int apndDeviceCharacteristics(sqlite3_file*); +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int apndShmLock(sqlite3_file*, int offset, int n, int flags); +static void apndShmBarrier(sqlite3_file*); +static int apndShmUnmap(sqlite3_file*, int deleteFlag); +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for ApndVfs +*/ +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void apndDlClose(sqlite3_vfs*, void*); +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int apndSleep(sqlite3_vfs*, int microseconds); +static int apndCurrentTime(sqlite3_vfs*, double*); +static int apndGetLastError(sqlite3_vfs*, int, char *); +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs apnd_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "apndvfs", /* zName */ + 0, /* pAppData (set when registered) */ + apndOpen, /* xOpen */ + apndDelete, /* xDelete */ + apndAccess, /* xAccess */ + apndFullPathname, /* xFullPathname */ + apndDlOpen, /* xDlOpen */ + apndDlError, /* xDlError */ + apndDlSym, /* xDlSym */ + apndDlClose, /* xDlClose */ + apndRandomness, /* xRandomness */ + apndSleep, /* xSleep */ + apndCurrentTime, /* xCurrentTime */ + apndGetLastError, /* xGetLastError */ + apndCurrentTimeInt64, /* xCurrentTimeInt64 */ + apndSetSystemCall, /* xSetSystemCall */ + apndGetSystemCall, /* xGetSystemCall */ + apndNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods apnd_io_methods = { + 3, /* iVersion */ + apndClose, /* xClose */ + apndRead, /* xRead */ + apndWrite, /* xWrite */ + apndTruncate, /* xTruncate */ + apndSync, /* xSync */ + apndFileSize, /* xFileSize */ + apndLock, /* xLock */ + apndUnlock, /* xUnlock */ + apndCheckReservedLock, /* xCheckReservedLock */ + apndFileControl, /* xFileControl */ + apndSectorSize, /* xSectorSize */ + apndDeviceCharacteristics, /* xDeviceCharacteristics */ + apndShmMap, /* xShmMap */ + apndShmLock, /* xShmLock */ + apndShmBarrier, /* xShmBarrier */ + apndShmUnmap, /* xShmUnmap */ + apndFetch, /* xFetch */ + apndUnfetch /* xUnfetch */ +}; + +/* +** Close an apnd-file. +*/ +static int apndClose(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from an apnd-file. +*/ +static int apndRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. +*/ +static int apndWriteMark( + ApndFile *paf, + sqlite3_file *pFile, + sqlite_int64 iWriteEnd +){ + sqlite_int64 iPgOne = paf->iPgOne; + unsigned char a[APND_MARK_SIZE]; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); + memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); + while( --i >= 0 ){ + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; + } + iWriteEnd += paf->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; + } + return rc; +} + +/* +** Write data to an apnd-file. +*/ +static int apndWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; + pFile = ORIGFILE(pFile); + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); + if( SQLITE_OK!=rc ) return rc; + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Truncate an apnd-file. +*/ +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); +} + +/* +** Sync an apnd-file. +*/ +static int apndSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. +*/ +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; +} + +/* +** Lock an apnd-file. +*/ +static int apndLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock an apnd-file. +*/ +static int apndUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an apnd-file. +*/ +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on an apnd-file. +*/ +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ + ApndFile *paf = (ApndFile *)pFile; + int rc; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_SIZE_HINT ) *(sqlite3_int64*)pArg += paf->iPgOne; + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne,*(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an apnd-file. +*/ +static int apndSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by an apnd-file. +*/ +static int apndDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int apndShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void apndShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int apndFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ + return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); +} + +/* +** Try to read the append-mark off the end of a file. Return the +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; +** +** An append-mark is only valid if the NNNNNNNN start-of-database offset +** indicates that the appended database contains at least one page. The +** start-of-database value must be a multiple of 512. +*/ +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc, i; + sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); + unsigned char a[APND_MARK_SIZE]; + + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc ) return -1; + if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]< (sz - APND_MARK_SIZE - 512) ) return -1; + if( iMark & 0x1ff ) return -1; + return iMark; +} + +static const char apvfsSqliteHdr[] = "SQLite format 3"; +/* +** Check to see if the file is an appendvfs SQLite database file. +** Return true iff it is such. Parameter sz is the file's size. +*/ +static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + sqlite3_int64 iMark = apndReadMark(sz, pFile); + if( iMark>=0 ){ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ + return 1; /* It's an appendvfs database */ + } + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 + ){ + return 0; + }else{ + return 1; + } +} + +/* +** Open an apnd file handle. +*/ +static int apndOpen( + sqlite3_vfs *pApndVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); + int rc; + sqlite3_int64 sz = 0; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + /* The appendvfs is not to be used for transient or temporary databases. + ** Just use the base VFS open to initialize the given file object and + ** open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); + } + memset(pApndFile, 0, sizeof(ApndFile)); + pFile->pMethods = &apnd_io_methods; + pApndFile->iMark = -1; /* Append mark not yet written */ + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + } + if( rc ){ + pBaseFile->pMethods->xClose(pBaseFile); + pFile->pMethods = 0; + return rc; + } + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + ** the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); + return SQLITE_OK; + } + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ + return SQLITE_OK; + } + if( (flags & SQLITE_OPEN_CREATE)==0 ){ + pBaseFile->pMethods->xClose(pBaseFile); + rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); + } + return rc; +} + +/* +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int apndAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int apndFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + + +#ifdef _WIN32 + +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + (void)db; + pOrig = sqlite3_vfs_find(0); + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 0); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} + +/************************* End ../ext/misc/appendvfs.c ********************/ +/************************* Begin ../ext/misc/memtrace.c ******************/ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include "libc/assert.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/memtrace.c ********************/ +/************************* Begin ../ext/misc/uint.c ******************/ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the maginitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/str/str.h" + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + (void)notUsed; + while( ia); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object. Initialize it to the number given +** by the input string. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, + sqlite3_value *pIn, + int nAlt, + const unsigned char *zAlt +){ + Decimal *p; + int n, i; + const unsigned char *zIn; + int iExp = 0; + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_no_mem; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + if( zAlt ){ + n = nAlt, + zIn = zAlt; + }else{ + if( sqlite3_value_type(pIn)==SQLITE_NULL ){ + p->a = 0; + p->isNull = 1; + return p; + } + n = sqlite3_value_bytes(pIn); + zIn = sqlite3_value_text(pIn); + } + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_no_mem; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_no_mem; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_no_mem; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_no_mem: + if( pCtx ) sqlite3_result_error_nomem(pCtx); + sqlite3_free(p); + return 0; +} + +/* +** Make the given Decimal the result. +*/ +static void decimal_result(sqlite3_context *pCtx, Decimal *p){ + char *z; + int i, j; + int n; + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** SQL Function: decimal(X) +** +** Convert input X into decimal and then back into text +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0, 0); + UNUSED_PARAMETER(argc); + decimal_result(context, p); + decimal_free(p); +} + +/* +** Compare to Decimal objects. Return negative, 0, or positive if the +** first object is less than, equal to, or greater than the second. +** +** Preconditions for this routine: +** +** pA!=0 +** pA->isNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 0, 0); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 0, 0); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimal_new(0, 0, nKey1, zA); + Decimal *pB = decimal_new(0, 0, nKey2, zB); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + UNUSED_PARAMETER(argc); + if( pB==0 ) return; + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 0, 0); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 0, 0); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 0, 0); + Decimal *pB = decimal_new(context, argv[1], 0, 0); + signed char *acc = 0; + int i, j, k; + int minFrac; + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + sqlite3_result_error_nomem(context); + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + decimal_result(context, pA); + +mul_end: + sqlite3_free(acc); + decimal_free(pA); + decimal_free(pB); +} + +#ifdef _WIN32 + +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, decimalFunc }, + { "decimal_cmp", 2, decimalCmpFunc }, + { "decimal_add", 2, decimalAddFunc }, + { "decimal_sub", 2, decimalSubFunc }, + { "decimal_mul", 2, decimalMulFunc }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +/* #include "third_party/sqlite3/sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include "libc/assert.h" +#include "libc/str/str.h" + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + m >>= 1-e; + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + + +#ifdef _WIN32 + +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i"); + for(i=0; iout,"\n"); + } + raw_printf(p->out,"\n"); + } + if( azArg==0 ) break; + raw_printf(p->out,""); + for(i=0; iout,"\n"); + } + raw_printf(p->out,"\n"); + break; + } + case MODE_Tcl: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,azCol[i] ? azCol[i] : ""); + if(iout, "%s", p->colSeparator); + } + utf8_printf(p->out, "%s", p->rowSeparator); + } + if( azArg==0 ) break; + for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); + if(iout, "%s", p->colSeparator); + } + utf8_printf(p->out, "%s", p->rowSeparator); + break; + } + case MODE_Csv: { + setBinaryMode(p->out, 1); + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout, "%s", p->rowSeparator); + } + if( nArg>0 ){ + for(i=0; iout, "%s", p->rowSeparator); + } + setTextMode(p->out, 1); + break; + } + case MODE_Insert: { + if( azArg==0 ) break; + utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + raw_printf(p->out,"("); + for(i=0; i0 ) raw_printf(p->out, ","); + if( quoteChar(azCol[i]) ){ + char *z = sqlite3_mprintf("\"%w\"", azCol[i]); + utf8_printf(p->out, "%s", z); + sqlite3_free(z); + }else{ + raw_printf(p->out, "%s", azCol[i]); + } + } + raw_printf(p->out,")"); + } + p->cnt++; + for(i=0; iout, i>0 ? "," : " VALUES("); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + utf8_printf(p->out,"NULL"); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p->out, azArg[i]); + }else{ + output_quoted_escaped_string(p->out, azArg[i]); + } + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + raw_printf(p->out, "1e999"); + }else if( ur==0xfff0000000000000LL ){ + raw_printf(p->out, "-1e999"); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p->out, azArg[i]); + }else{ + output_quoted_escaped_string(p->out, azArg[i]); + } + } + raw_printf(p->out,");\n"); + break; + } + case MODE_Json: { + if( azArg==0 ) break; + if( p->cnt==0 ){ + fputs("[{", p->out); + }else{ + fputs(",\n{", p->out); + } + p->cnt++; + for(i=0; iout, azCol[i], -1); + putc(':', p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + fputs("null",p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + raw_printf(p->out, "1e999"); + }else if( ur==0xfff0000000000000LL ){ + raw_printf(p->out, "-1e999"); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_json_string(p->out, pBlob, nBlob); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_json_string(p->out, azArg[i], -1); + }else{ + utf8_printf(p->out,"%s", azArg[i]); + } + if( iout); + } + } + putc('}', p->out); + break; + } + case MODE_Quote: { + if( azArg==0 ) break; + if( p->cnt==0 && p->showHeader ){ + for(i=0; i0 ) fputs(p->colSeparator, p->out); + output_quoted_string(p->out, azCol[i]); + } + fputs(p->rowSeparator, p->out); + } + p->cnt++; + for(i=0; i0 ) fputs(p->colSeparator, p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + utf8_printf(p->out,"NULL"); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_quoted_string(p->out, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + utf8_printf(p->out,"%s", azArg[i]); + }else{ + output_quoted_string(p->out, azArg[i]); + } + } + fputs(p->rowSeparator, p->out); + break; + } + case MODE_Ascii: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); + utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : ""); + } + utf8_printf(p->out, "%s", p->rowSeparator); + } + if( azArg==0 ) break; + for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); + utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue); + } + utf8_printf(p->out, "%s", p->rowSeparator); + break; + } + case MODE_EQP: { + eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); + break; + } + } + return 0; +} + +/* +** This is the callback routine that the SQLite library +** invokes for each row of a query result. +*/ +static int callback(void *pArg, int nArg, char **azArg, char **azCol){ + /* since we don't have type info, call the shell_callback with a NULL value */ + return shell_callback(pArg, nArg, azArg, azCol, NULL); +} + +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + UNUSED_PARAMETER(az); + if( azArg==0 ) return 0; + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_schema\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO [_shell$self]\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); +} + + +/* +** Set the destination table field of the ShellState structure to +** the name of the table given. Escape any quote characters in the +** table name. +*/ +static void set_table_name(ShellState *p, const char *zName){ + int i, n; + char cQuote; + char *z; + + if( p->zDestTable ){ + free(p->zDestTable); + p->zDestTable = 0; + } + if( zName==0 ) return; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += n+2; + z = p->zDestTable = malloc( n+1 ); + if( z==0 ) shell_out_of_memory(); + n = 0; + if( cQuote ) z[n++] = cQuote; + for(i=0; zName[i]; i++){ + z[n++] = zName[i]; + if( zName[i]==cQuote ) z[n++] = cQuote; + } + if( cQuote ) z[n++] = cQuote; + z[n] = 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static int run_table_dump_query( + ShellState *p, /* Query context */ + const char *zSelect /* SELECT statement to extract content */ +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); + if( rc!=SQLITE_OK || !pSelect ){ + utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + return rc; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + utf8_printf(p->out, "%s", z); + for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + raw_printf(p->out, "\n;\n"); + }else{ + raw_printf(p->out, ";\n"); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } + return rc; +} + +/* +** Allocate space and save off current error string. +*/ +static char *save_err_msg( + sqlite3 *db /* Database to query */ +){ + int nErrMsg = 1+strlen30(sqlite3_errmsg(db)); + char *zErrMsg = sqlite3_malloc64(nErrMsg); + if( zErrMsg ){ + memcpy(zErrMsg, sqlite3_errmsg(db), nErrMsg); + } + return zErrMsg; +} + +#ifdef __linux__ +/* +** Attempt to display I/O stats on Linux using /proc/PID/io +*/ +static void displayLinuxIoStats(FILE *out){ + FILE *in; + char z[200]; + sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); + in = fopen(z, "rb"); + if( in==0 ) return; + while( fgets(z, sizeof(z), in)!=0 ){ + static const struct { + const char *zPattern; + const char *zDesc; + } aTrans[] = { + { "rchar: ", "Bytes received by read():" }, + { "wchar: ", "Bytes sent to write():" }, + { "syscr: ", "Read() system calls:" }, + { "syscw: ", "Write() system calls:" }, + { "read_bytes: ", "Bytes read from storage:" }, + { "write_bytes: ", "Bytes written to storage:" }, + { "cancelled_write_bytes: ", "Cancelled write bytes:" }, + }; + int i; + for(i=0; i1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + raw_printf(p->out, "%-36s %s\n", zLabel, zLine); +} + +/* +** Display memory stats. +*/ +static int display_stats( + sqlite3 *db, /* Database to query */ + ShellState *pArg, /* Pointer to ShellState */ + int bReset /* True to reset the stats */ +){ + int iCur; + int iHiwtr; + FILE *out; + if( pArg==0 || pArg->out==0 ) return 0; + out = pArg->out; + + if( pArg->pStmt && pArg->statsOn==2 ){ + int nCol, i, x; + sqlite3_stmt *pStmt = pArg->pStmt; + char z[100]; + nCol = sqlite3_column_count(pStmt); + raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + for(i=0; istatsOn==3 ){ + if( pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + raw_printf(pArg->out, "VM-steps: %d\n", iCur); + } + return 0; + } + + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); + if( pArg->shellFlgs & SHFLG_Pagecache ){ + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); + } + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); +#ifdef YYTRACKMAXSTACKDEPTH + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); +#endif + + if( db ){ + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, + "Lookaside Slots Used: %d (max %d)\n", + iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Successful lookaside attempts: %d\n", + iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Lookaside failures due to size: %d\n", + iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Lookaside failures due to OOM: %d\n", + iHiwtr); + } + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Pager Heap Usage: %d bytes\n", + iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache hits: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", + iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", + iCur); + } + + if( pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); + raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); + raw_printf(pArg->out, "Sort Operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); + raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + } + +#ifdef __linux__ + displayLinuxIoStats(pArg->out); +#endif + + /* Do not remove this machine readable comment: extra-stats-output-here */ + + return 0; +} + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + int i, k, n, mx; + raw_printf(pArg->out, "-------- scanstats --------\n"); + mx = 0; + for(k=0; k<=mx; k++){ + double rEstLoop = 1.0; + for(i=n=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nLoop, nVisit; + double rEst; + int iSid; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>mx ) mx = iSid; + if( iSid!=k ) continue; + if( n==0 ){ + rEstLoop = (double)nLoop; + if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k); + } + n++; + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); + rEstLoop *= rEst; + raw_printf(pArg->out, + " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + ); + } + } + raw_printf(pArg->out, "---------------------------\n"); +#endif +} + +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int str_in_array(const char *zStr, const char **azArray){ + int i; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** If compiled statement pSql appears to be an EXPLAIN statement, allocate +** and populate the ShellState.aiIndent[] array with the number of +** spaces each opcode should be indented before it is output. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ + const char *zSql; /* The text of the SQL statement */ + const char *z; /* Used to check if this is an EXPLAIN */ + int *abYield = 0; /* True if op is an OP_Yield */ + int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ + int iOp; /* Index of operation in p->aiIndent[] */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* Try to figure out if this is really an EXPLAIN statement. If this + ** cannot be verified, return early. */ + if( sqlite3_column_count(pSql)!=8 ){ + p->cMode = p->mode; + return; + } + zSql = sqlite3_sql(pSql); + if( zSql==0 ) return; + for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); + if( sqlite3_strnicmp(z, "explain", 7) ){ + p->cMode = p->mode; + return; + } + + for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ + int i; + int iAddr = sqlite3_column_int(pSql, 0); + const char *zOp = (const char*)sqlite3_column_text(pSql, 1); + + /* Set p2 to the P2 field of the current opcode. Then, assuming that + ** p2 is an instruction address, set variable p2op to the index of that + ** instruction in the aiIndent[] array. p2 and p2op may be different if + ** the current instruction is part of a sub-program generated by an + ** SQL trigger or foreign key. */ + int p2 = sqlite3_column_int(pSql, 3); + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the p->aiIndent array as required */ + if( iOp>=nAlloc ){ + if( iOp==0 ){ + /* Do further verfication that this is explain output. Abort if + ** it is not */ + static const char *explainCols[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; + int jj; + for(jj=0; jjcMode = p->mode; + sqlite3_reset(pSql); + return; + } + } + } + nAlloc += 100; + p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + if( p->aiIndent==0 ) shell_out_of_memory(); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( abYield==0 ) shell_out_of_memory(); + } + abYield[iOp] = str_in_array(zOp, azYield); + p->aiIndent[iOp] = 0; + p->nIndent = iOp+1; + + if( str_in_array(zOp, azNext) ){ + for(i=p2op; iaiIndent[i] += 2; + } + if( str_in_array(zOp, azGoto) && p2opnIndent + && (abYield[p2op] || sqlite3_column_int(pSql, 2)) + ){ + for(i=p2op; iaiIndent[i] += 2; + } + } + + p->iIndent = 0; + sqlite3_free(abYield); + sqlite3_reset(pSql); +} + +/* +** Free the array allocated by explain_data_prepare(). +*/ +static void explain_data_delete(ShellState *p){ + sqlite3_free(p->aiIndent); + p->aiIndent = 0; + p->nIndent = 0; + p->iIndent = 0; +} + +/* +** Disable and restore .wheretrace and .selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +} +static void restore_debug_trace_modes(void){ + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +} + +/* Create the TEMP table used to store parameter bindings */ +static void bind_table_init(ShellState *p){ + int wrSchema = 0; + int defensiveMode = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &defensiveMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, + "CREATE TABLE IF NOT EXISTS temp.sqlite_parameters(\n" + " key TEXT PRIMARY KEY,\n" + " value ANY\n" + ") WITHOUT ROWID;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, defensiveMode, 0); +} + +/* +** Bind parameters on a prepared statement. +** +** Parameter bindings are taken from a TEMP table of the form: +** +** CREATE TEMP TABLE sqlite_parameters(key TEXT PRIMARY KEY, value) +** WITHOUT ROWID; +** +** No bindings occur if this table does not exist. The name of the table +** begins with "sqlite_" so that it will not collide with ordinary application +** tables. The table must be in the TEMP schema. +*/ +static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ + int nVar; + int i; + int rc; + sqlite3_stmt *pQ = 0; + + nVar = sqlite3_bind_parameter_count(pStmt); + if( nVar==0 ) return; /* Nothing to do */ + if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", + "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ + return; /* Parameter table does not exist */ + } + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); + if( rc || pQ==0 ) return; + for(i=1; i<=nVar; i++){ + char zNum[30]; + const char *zVar = sqlite3_bind_parameter_name(pStmt, i); + if( zVar==0 ){ + sqlite3_snprintf(sizeof(zNum),zNum,"?%d",i); + zVar = zNum; + } + sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); + if( sqlite3_step(pQ)==SQLITE_ROW ){ + sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); + }else{ + sqlite3_bind_null(pStmt, i); + } + sqlite3_reset(pQ); + } + sqlite3_finalize(pQ); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void print_box_line(FILE *out, int N){ + const char zDash[] = + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; + const int nDash = sizeof(zDash) - 1; + N *= 3; + while( N>nDash ){ + utf8_printf(out, zDash); + N -= nDash; + } + utf8_printf(out, "%.*s", N, zDash); +} + +/* +** Draw a horizontal separator for a MODE_Box table. +*/ +static void print_box_row_separator( + ShellState *p, + int nArg, + const char *zSep1, + const char *zSep2, + const char *zSep3 +){ + int i; + if( nArg>0 ){ + utf8_printf(p->out, "%s", zSep1); + print_box_line(p->out, p->actualWidth[0]+2); + for(i=1; iout, "%s", zSep2); + print_box_line(p->out, p->actualWidth[i]+2); + } + utf8_printf(p->out, "%s", zSep3); + } + fputs("\n", p->out); +} + + + +/* +** Run a prepared statement and output the result in one of the +** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, +** or MODE_Box. +** +** This is different from ordinary exec_prepared_stmt() in that +** it has to run the entire query and gather the results into memory +** first, in order to determine column widths, before providing +** any output. +*/ +static void exec_prepared_stmt_columnar( + ShellState *p, /* Pointer to ShellState */ + sqlite3_stmt *pStmt /* Statment to run */ +){ + sqlite3_int64 nRow = 0; + int nColumn = 0; + char **azData = 0; + sqlite3_int64 nAlloc = 0; + const char *z; + int rc; + sqlite3_int64 i, nData; + int j, nTotal, w, n; + const char *colSep = 0; + const char *rowSep = 0; + + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ) return; + nColumn = sqlite3_column_count(pStmt); + nAlloc = nColumn*4; + if( nAlloc<=0 ) nAlloc = 1; + azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); + if( azData==0 ) shell_out_of_memory(); + for(i=0; i= nAlloc ){ + nAlloc *= 2; + azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); + if( azData==0 ) shell_out_of_memory(); + } + nRow++; + for(i=0; ip->nWidth ){ + p->colWidth = realloc(p->colWidth, nColumn*2*sizeof(int)); + if( p->colWidth==0 ) shell_out_of_memory(); + for(i=p->nWidth; icolWidth[i] = 0; + p->nWidth = nColumn; + p->actualWidth = &p->colWidth[nColumn]; + } + memset(p->actualWidth, 0, nColumn*sizeof(int)); + for(i=0; icolWidth[i]; + if( w<0 ) w = -w; + p->actualWidth[i] = w; + } + nTotal = nColumn*(nRow+1); + for(i=0; inullValue; + n = strlenChar(z); + j = i%nColumn; + if( n>p->actualWidth[j] ) p->actualWidth[j] = n; + } + if( seenInterrupt ) goto columnar_end; + if( nColumn==0 ) goto columnar_end; + switch( p->cMode ){ + case MODE_Column: { + colSep = " "; + rowSep = "\n"; + if( p->showHeader ){ + for(i=0; iactualWidth[i]; + if( p->colWidth[i]<0 ) w = -w; + utf8_width_print(p->out, w, azData[i]); + fputs(i==nColumn-1?"\n":" ", p->out); + } + for(i=0; iout, p->actualWidth[i]); + fputs(i==nColumn-1?"\n":" ", p->out); + } + } + break; + } + case MODE_Table: { + colSep = " | "; + rowSep = " |\n"; + print_row_separator(p, nColumn, "+"); + fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "+"); + break; + } + case MODE_Markdown: { + colSep = " | "; + rowSep = " |\n"; + fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "|"); + break; + } + case MODE_Box: { + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); + utf8_printf(p->out, BOX_13 " "); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + } + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + break; + } + } + for(i=nColumn, j=0; icMode!=MODE_Column ){ + utf8_printf(p->out, "%s", p->cMode==MODE_Box?BOX_13" ":"| "); + } + z = azData[i]; + if( z==0 ) z = p->nullValue; + w = p->actualWidth[j]; + if( p->colWidth[j]<0 ) w = -w; + utf8_width_print(p->out, w, z); + if( j==nColumn-1 ){ + utf8_printf(p->out, "%s", rowSep); + j = -1; + if( seenInterrupt ) goto columnar_end; + }else{ + utf8_printf(p->out, "%s", colSep); + } + } + if( p->cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); + } +columnar_end: + if( seenInterrupt ){ + utf8_printf(p->out, "Interrupt\n"); + } + nData = (nRow+1)*nColumn; + for(i=0; icMode==MODE_Column + || pArg->cMode==MODE_Table + || pArg->cMode==MODE_Box + || pArg->cMode==MODE_Markdown + ){ + exec_prepared_stmt_columnar(pArg, pStmt); + return; + } + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + rc = SQLITE_NOMEM; + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; icMode==MODE_Insert ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + if( pArg->cMode==MODE_Json ){ + fputs("]\n", pArg->out); + } + } + } +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** This function is called to process SQL if the previous shell command +** was ".expert". It passes the SQL in the second argument directly to +** the sqlite3expert object. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error +** code. In this case, (*pzErr) may be set to point to a buffer containing +** an English language error message. It is the responsibility of the +** caller to eventually free this buffer using sqlite3_free(). +*/ +static int expertHandleSQL( + ShellState *pState, + const char *zSql, + char **pzErr +){ + assert( pState->expert.pExpert ); + assert( pzErr==0 || *pzErr==0 ); + return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr); +} + +/* +** This function is called either to silently clean up the object +** created by the ".expert" command (if bCancel==1), or to generate a +** report from it and then clean it up (if bCancel==0). +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error +** code. In this case, (*pzErr) may be set to point to a buffer containing +** an English language error message. It is the responsibility of the +** caller to eventually free this buffer using sqlite3_free(). +*/ +static int expertFinish( + ShellState *pState, + int bCancel, + char **pzErr +){ + int rc = SQLITE_OK; + sqlite3expert *p = pState->expert.pExpert; + assert( p ); + assert( bCancel || pzErr==0 || *pzErr==0 ); + if( bCancel==0 ){ + FILE *out = pState->out; + int bVerbose = pState->expert.bVerbose; + + rc = sqlite3_expert_analyze(p, pzErr); + if( rc==SQLITE_OK ){ + int nQuery = sqlite3_expert_count(p); + int i; + + if( bVerbose ){ + const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); + raw_printf(out, "-- Candidates -----------------------------\n"); + raw_printf(out, "%s\n", zCand); + } + for(i=0; iexpert.pExpert = 0; + return rc; +} + +/* +** Implementation of ".expert" dot command. +*/ +static int expertDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int rc = SQLITE_OK; + char *zErr = 0; + int i; + int iSample = 0; + + assert( pState->expert.pExpert==0 ); + memset(&pState->expert, 0, sizeof(ExpertInfo)); + + for(i=1; rc==SQLITE_OK && i=2 && 0==strncmp(z, "-verbose", n) ){ + pState->expert.bVerbose = 1; + } + else if( n>=2 && 0==strncmp(z, "-sample", n) ){ + if( i==(nArg-1) ){ + raw_printf(stderr, "option requires an argument: %s\n", z); + rc = SQLITE_ERROR; + }else{ + iSample = (int)integerValue(azArg[++i]); + if( iSample<0 || iSample>100 ){ + raw_printf(stderr, "value out of range: %s\n", azArg[i]); + rc = SQLITE_ERROR; + } + } + } + else{ + raw_printf(stderr, "unknown option: %s\n", z); + rc = SQLITE_ERROR; + } + } + + if( rc==SQLITE_OK ){ + pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); + if( pState->expert.pExpert==0 ){ + raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr); + rc = SQLITE_ERROR; + }else{ + sqlite3_expert_config( + pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample + ); + } + } + + return rc; +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Execute a statement or set of statements. Print +** any result rows/columns depending on the current mode +** set via the supplied callback. +** +** This is very similar to SQLite's built-in sqlite3_exec() +** function except it takes a slightly different callback +** and callback data argument. +*/ +static int shell_exec( + ShellState *pArg, /* Pointer to ShellState */ + const char *zSql, /* SQL to be evaluated */ + char **pzErrMsg /* Error msg written here */ +){ + sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ + int rc = SQLITE_OK; /* Return Code */ + int rc2; + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3 *db = pArg->db; + + if( pzErrMsg ){ + *pzErrMsg = NULL; + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pArg->expert.pExpert ){ + rc = expertHandleSQL(pArg, zSql, pzErrMsg); + return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg); + } +#endif + + while( zSql[0] && (SQLITE_OK == rc) ){ + static const char *zStmtSql; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); + if( SQLITE_OK != rc ){ + if( pzErrMsg ){ + *pzErrMsg = save_err_msg(db); + } + }else{ + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + while( IsSpace(zSql[0]) ) zSql++; + continue; + } + zStmtSql = sqlite3_sql(pStmt); + if( zStmtSql==0 ) zStmtSql = ""; + while( IsSpace(zStmtSql[0]) ) zStmtSql++; + + /* save off the prepared statment handle and reset row count */ + if( pArg ){ + pArg->pStmt = pStmt; + pArg->cnt = 0; + } + + /* echo the sql statement if echo on */ + if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ + utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); + } + + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ + if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ + sqlite3_stmt *pExplain; + char *zEQP; + int triggerEQP = 0; + disable_debug_trace_modes(); + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); + if( pArg->autoEQP>=AUTOEQP_trigger ){ + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); + } + zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pExplain)==SQLITE_ROW ){ + const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); + int iEqpId = sqlite3_column_int(pExplain, 0); + int iParentId = sqlite3_column_int(pExplain, 1); + if( zEQPLine==0 ) zEQPLine = ""; + if( zEQPLine[0]=='-' ) eqp_render(pArg); + eqp_append(pArg, iEqpId, iParentId, zEQPLine); + } + eqp_render(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + if( pArg->autoEQP>=AUTOEQP_full ){ + /* Also do an EXPLAIN for ".eqp full" mode */ + zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + pArg->cMode = MODE_Explain; + explain_data_prepare(pArg, pExplain); + exec_prepared_stmt(pArg, pExplain); + explain_data_delete(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + } + if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); + /* Reprepare pStmt before reactiving trace modes */ + sqlite3_finalize(pStmt); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( pArg ) pArg->pStmt = pStmt; + } + restore_debug_trace_modes(); + } + + if( pArg ){ + pArg->cMode = pArg->mode; + if( pArg->autoExplain ){ + if( sqlite3_stmt_isexplain(pStmt)==1 ){ + pArg->cMode = MODE_Explain; + } + if( sqlite3_stmt_isexplain(pStmt)==2 ){ + pArg->cMode = MODE_EQP; + } + } + + /* If the shell is currently in ".explain" mode, gather the extra + ** data required to add indents to the output.*/ + if( pArg->cMode==MODE_Explain ){ + explain_data_prepare(pArg, pStmt); + } + } + + bind_prepared_stmt(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + eqp_render(pArg); + + /* print usage stats if stats on */ + if( pArg && pArg->statsOn ){ + display_stats(db, pArg, 0); + } + + /* print loop-counters if required */ + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); + } + + /* Finalize the statement just executed. If this fails, save a + ** copy of the error message. Otherwise, set zSql to point to the + ** next statement to execute. */ + rc2 = sqlite3_finalize(pStmt); + if( rc!=SQLITE_NOMEM ) rc = rc2; + if( rc==SQLITE_OK ){ + zSql = zLeftover; + while( IsSpace(zSql[0]) ) zSql++; + }else if( pzErrMsg ){ + *pzErrMsg = save_err_msg(db); + } + + /* clear saved stmt handle */ + if( pArg ){ + pArg->pStmt = NULL; + } + } + } /* end while */ + + return rc; +} + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ) shell_out_of_memory(); + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + if( azCol==0 ) return 0; + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + +/* +** Toggle the reverse_unordered_selects setting. +*/ +static void toggleSelectOrder(sqlite3 *db){ + sqlite3_stmt *pStmt = 0; + int iSetting = 0; + char zStmt[100]; + sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iSetting = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + sqlite3_snprintf(sizeof(zStmt), zStmt, + "PRAGMA reverse_unordered_selects(%d)", !iSetting); + sqlite3_exec(db, zStmt, 0, 0, 0); +} + +/* +** This is a different callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + ShellState *p = (ShellState *)pArg; + int dataOnly; + int noSys; + + UNUSED_PARAMETER(azNotUsed); + if( nArg!=3 || azArg==0 ) return 0; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0; + noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; + + if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ + if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ + if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( dataOnly ){ + /* no-op */ + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + char *zIns; + if( !p->writableSchema ){ + raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); + p->writableSchema = 1; + } + zIns = sqlite3_mprintf( + "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + utf8_printf(p->out, "%s\n", zIns); + sqlite3_free(zIns); + return 0; + }else{ + printSchemaLine(p->out, zSql, ";\n"); + } + + if( strcmp(zType, "table")==0 ){ + ShellText sSelect; + ShellText sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; + + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; + } + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); + } + appendText(&sTable, ")", 0); + } + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p, sSelect.z, 0); + if( (rc&0xff)==SQLITE_CORRUPT ){ + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + toggleSelectOrder(p->db); + shell_exec(p, sSelect.z, 0); + toggleSelectOrder(p->db); + } + p->zDestTable = savedDestTable; + p->mode = savedMode; + freeText(&sTable); + freeText(&sSelect); + if( rc ) p->nErr++; + } + return 0; +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static int run_schema_dump_query( + ShellState *p, + const char *zQuery +){ + int rc; + char *zErr = 0; + rc = sqlite3_exec(p->db, zQuery, dump_callback, p, &zErr); + if( rc==SQLITE_CORRUPT ){ + char *zQ2; + int len = strlen30(zQuery); + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + if( zErr ){ + utf8_printf(p->out, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + zErr = 0; + } + zQ2 = malloc( len+100 ); + if( zQ2==0 ) return rc; + sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); + rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); + if( rc ){ + utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr); + }else{ + rc = SQLITE_CORRUPT; + } + sqlite3_free(zErr); + free(zQ2); + } + return rc; +} + +/* +** Text of help messages. +** +** The help text for each individual command begins with a line that starts +** with ".". Subsequent lines are supplimental information. +** +** There must be two or more spaces between the end of the command and the +** start of the description of what that command does. +*/ +static const char *(azHelp[]) = { +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + ".archive ... Manage SQL archives", + " Each command must have exactly one of the following options:", + " -c, --create Create a new archive", + " -u, --update Add or update files with changed mtime", + " -i, --insert Like -u but always add even if unchanged", + " -t, --list List contents of archive", + " -x, --extract Extract files from archive", + " Optional arguments:", + " -v, --verbose Print each filename as it is processed", + " -f FILE, --file FILE Use archive FILE (default is current db)", + " -a FILE, --append FILE Open FILE using the apndvfs VFS", + " -C DIR, --directory DIR Read/extract files from directory DIR", + " -n, --dryrun Show the SQL that would have occurred", + " Examples:", + " .ar -cf ARCHIVE foo bar # Create ARCHIVE from files foo and bar", + " .ar -tf ARCHIVE # List members of ARCHIVE", + " .ar -xvf ARCHIVE # Verbosely extract files from ARCHIVE", + " See also:", + " http://sqlite.org/cli.html#sqlar_archive_support", +#endif +#ifndef SQLITE_OMIT_AUTHORIZATION + ".auth ON|OFF Show authorizer callbacks", +#endif + ".backup ?DB? FILE Backup DB (default \"main\") to FILE", + " --append Use the appendvfs", + " --async Write to FILE without journal and fsync()", + ".bail on|off Stop after hitting an error. Default OFF", + ".binary on|off Turn binary output on or off. Default OFF", + ".cd DIRECTORY Change the working directory to DIRECTORY", + ".changes on|off Show number of rows changed by SQL", + ".check GLOB Fail if output since .testcase does not match", + ".clone NEWDB Clone data into NEWDB from the existing database", + ".databases List names and files of attached databases", + ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", + ".dbinfo ?DB? Show status information about the database", + ".dump ?OBJECTS? Render database content as SQL", + " Options:", + " --data-only Output only INSERT statements", + " --newlines Allow unescaped newline characters in output", + " --nosys Omit system tables (ex: \"sqlite_stat1\")", + " --preserve-rowids Include ROWID values in the output", + " OBJECTS is a LIKE pattern for tables, indexes, triggers or views to dump", + " Additional LIKE patterns can be given in subsequent arguments", + ".echo on|off Turn command echo on or off", + ".eqp on|off|full|... Enable or disable automatic EXPLAIN QUERY PLAN", + " Other Modes:", +#ifdef SQLITE_DEBUG + " test Show raw EXPLAIN QUERY PLAN output", + " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", +#endif + " trigger Like \"full\" but also show trigger bytecode", + ".excel Display the output of next command in spreadsheet", + " --bom Put a UTF8 byte-order mark on intermediate file", + ".exit ?CODE? Exit this program with return-code CODE", + ".expert EXPERIMENTAL. Suggest indexes for queries", + ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", + ".filectrl CMD ... Run various sqlite3_file_control() operations", + " --schema SCHEMA Use SCHEMA instead of \"main\"", + " --help Show CMD details", + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", + ".headers on|off Turn display of headers on or off", + ".help ?-all? ?PATTERN? Show help text for PATTERN", + ".import FILE TABLE Import data from FILE into TABLE", + " Options:", + " --ascii Use \\037 and \\036 as column and row separators", + " --csv Use , and \\n as column and row separators", + " --skip N Skip the first N rows of input", + " -v \"Verbose\" - increase auxiliary output", + " Notes:", + " * If TABLE does not exist, it is created. The first row of input", + " determines the column names.", + " * If neither --csv or --ascii are used, the input mode is derived", + " from the \".mode\" output mode", + " * If FILE begins with \"|\" then it is a command that generates the", + " input text.", +#ifndef SQLITE_OMIT_TEST_CONTROL + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", +#endif + ".indexes ?TABLE? Show names of indexes", + " If TABLE is specified, only show indexes for", + " tables matching TABLE using the LIKE operator.", +#ifdef SQLITE_ENABLE_IOTRACE + ".iotrace FILE Enable I/O diagnostic logging to FILE", +#endif + ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", + ".lint OPTIONS Report potential schema issues.", + " Options:", + " fkey-indexes Find missing foreign key indexes", +#ifndef SQLITE_OMIT_LOAD_EXTENSION + ".load FILE ?ENTRY? Load an extension library", +#endif + ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", + ".mode MODE ?TABLE? Set output mode", + " MODE is one of:", + " ascii Columns/rows delimited by 0x1F and 0x1E", + " box Tables using unicode box-drawing characters", + " csv Comma-separated values", + " column Output in columns. (See .width)", + " html HTML
"); + output_html_string(p->out, azCol[i]); + raw_printf(p->out,"
"); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + raw_printf(p->out,"
code", + " insert SQL insert statements for TABLE", + " json Results in a JSON array", + " line One value per line", + " list Values delimited by \"|\"", + " markdown Markdown table format", + " quote Escape answers as for SQL", + " table ASCII-art table", + " tabs Tab-separated values", + " tcl TCL list elements", + ".nullvalue STRING Use STRING in place of NULL values", + ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", + " If FILE begins with '|' then open as a pipe", + " --bom Put a UTF8 byte-order mark at the beginning", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet (same as \".excel\")", +#ifdef SQLITE_DEBUG + ".oom ?--repeat M? ?N? Simulate an OOM error on the N-th allocation", +#endif + ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", + " Options:", + " --append Use appendvfs to append database to the end of FILE", +#ifdef SQLITE_ENABLE_DESERIALIZE + " --deserialize Load into memory useing sqlite3_deserialize()", + " --hexdb Load the output of \"dbtotxt\" as an in-memory db", + " --maxsize N Maximum size for --hexdb or --deserialized database", +#endif + " --new Initialize FILE to an empty database", + " --nofollow Do not follow symbolic links", + " --readonly Open FILE readonly", + " --zip FILE is a ZIP archive", + ".output ?FILE? Send output to FILE or stdout if FILE is omitted", + " If FILE begins with '|' then open it as a pipe.", + " Options:", + " --bom Prefix output with a UTF8 byte-order mark", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet", + ".parameter CMD ... Manage SQL parameter bindings", + " clear Erase all bindings", + " init Initialize the TEMP table that holds bindings", + " list List the current parameter bindings", + " set PARAMETER VALUE Given SQL parameter PARAMETER a value of VALUE", + " PARAMETER should start with one of: $ : @ ?", + " unset PARAMETER Remove PARAMETER from the binding table", + ".print STRING... Print literal STRING", +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + ".progress N Invoke progress handler after every N opcodes", + " --limit N Interrupt after N progress callbacks", + " --once Do no more than one progress interrupt", + " --quiet|-q No output except at interrupts", + " --reset Reset the count for each input and interrupt", +#endif + ".prompt MAIN CONTINUE Replace the standard prompts", + ".quit Exit this program", + ".read FILE Read input from FILE", +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + ".recover Recover as much data as possible from corrupt db.", + " --freelist-corrupt Assume the freelist is corrupt", + " --recovery-db NAME Store recovery metadata in database file NAME", + " --lost-and-found TABLE Alternative name for the lost-and-found table", + " --no-rowids Do not attempt to recover rowid values", + " that are not also INTEGER PRIMARY KEYs", +#endif + ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", + ".save FILE Write in-memory database into FILE", + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", + ".schema ?PATTERN? Show the CREATE statements matching PATTERN", + " Options:", + " --indent Try to pretty-print the schema", + " --nosys Omit objects whose names start with \"sqlite_\"", + ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + " Options:", + " --init Create a new SELFTEST table", + " -v Verbose output", + ".separator COL ?ROW? Change the column and row separators", +#if defined(SQLITE_ENABLE_SESSION) + ".session ?NAME? CMD ... Create or control sessions", + " Subcommands:", + " attach TABLE Attach TABLE", + " changeset FILE Write a changeset into FILE", + " close Close one session", + " enable ?BOOLEAN? Set or query the enable bit", + " filter GLOB... Reject tables matching GLOBs", + " indirect ?BOOLEAN? Mark or query the indirect status", + " isempty Query whether the session is empty", + " list List currently open session names", + " open DB NAME Open a new session on DB", + " patchset FILE Write a patchset into FILE", + " If ?NAME? is omitted, the first defined session is used.", +#endif + ".sha3sum ... Compute a SHA3 hash of database content", + " Options:", + " --schema Also hash the sqlite_schema table", + " --sha3-224 Use the sha3-224 algorithm", + " --sha3-256 Use the sha3-256 algorithm (default)", + " --sha3-384 Use the sha3-384 algorithm", + " --sha3-512 Use the sha3-512 algorithm", + " Any other argument is a LIKE pattern for tables to hash", +#ifndef SQLITE_NOHAVE_SYSTEM + ".shell CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".show Show the current values for various settings", + ".stats ?ARG? Show stats or turn stats on or off", + " off Turn off automatic stat display", + " on Turn on automatic stat display", + " stmt Show statement stats", + " vmstep Show the virtual machine step count only", +#ifndef SQLITE_NOHAVE_SYSTEM + ".system CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", + ".testcase NAME Begin redirecting output to 'testcase-out.txt'", + ".testctrl CMD ... Run various sqlite3_test_control() operations", + " Run \".testctrl\" with no arguments for details", + ".timeout MS Try opening locked tables for MS milliseconds", + ".timer on|off Turn SQL timer on or off", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ +#ifdef SQLITE_DEBUG + ".unmodule NAME ... Unregister virtual table modules", + " --allexcept Unregister everything except those named", +#endif + ".vfsinfo ?AUX? Information about the top-level VFS", + ".vfslist List all available VFSes", + ".vfsname ?AUX? Print the name of the VFS stack", + ".width NUM1 NUM2 ... Set minimum column widths for columnar output", + " Negative values right-justify", +}; + +/* +** Output help text. +** +** zPattern describes the set of commands for which help text is provided. +** If zPattern is NULL, then show all commands, but only give a one-line +** description of each. +** +** Return the number of matches. +*/ +static int showHelp(FILE *out, const char *zPattern){ + int i = 0; + int j = 0; + int n = 0; + char *zPat; + if( zPattern==0 + || zPattern[0]=='0' + || strcmp(zPattern,"-a")==0 + || strcmp(zPattern,"-all")==0 + || strcmp(zPattern,"--all")==0 + ){ + /* Show all commands, but only one line per command */ + if( zPattern==0 ) zPattern = ""; + for(i=0; ip); + sqlite3_free(pSession->zName); + for(i=0; inFilter; i++){ + sqlite3_free(pSession->azFilter[i]); + } + sqlite3_free(pSession->azFilter); + memset(pSession, 0, sizeof(OpenSession)); +} +#endif + +/* +** Close all OpenSession objects and release all associated resources. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static void session_close_all(ShellState *p){ + int i; + for(i=0; inSession; i++){ + session_close(&p->aSession[i]); + } + p->nSession = 0; +} +#else +# define session_close_all(X) +#endif + +/* +** Implementation of the xFilter function for an open session. Omit +** any tables named by ".session filter" but let all other table through. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static int session_filter(void *pCtx, const char *zTab){ + OpenSession *pSession = (OpenSession*)pCtx; + int i; + for(i=0; inFilter; i++){ + if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0; + } + return 1; +} +#endif + +/* +** Try to deduce the type of file for zName based on its content. Return +** one of the SHELL_OPEN_* constants. +** +** If the file does not exist or is empty but its name looks like a ZIP +** archive and the dfltZip flag is true, then assume it is a ZIP archive. +** Otherwise, assume an ordinary database regardless of the filename if +** the type cannot be determined from content. +*/ +int deduceDatabaseType(const char *zName, int dfltZip){ + FILE *f = fopen(zName, "rb"); + size_t n; + int rc = SHELL_OPEN_UNSPEC; + char zBuf[100]; + if( f==0 ){ + if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + return SHELL_OPEN_ZIPFILE; + }else{ + return SHELL_OPEN_NORMAL; + } + } + n = fread(zBuf, 16, 1, f); + if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){ + fclose(f); + return SHELL_OPEN_NORMAL; + } + fseek(f, -25, SEEK_END); + n = fread(zBuf, 25, 1, f); + if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){ + rc = SHELL_OPEN_APPENDVFS; + }else{ + fseek(f, -22, SEEK_END); + n = fread(zBuf, 22, 1, f); + if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05 + && zBuf[3]==0x06 ){ + rc = SHELL_OPEN_ZIPFILE; + }else if( n==0 && dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + rc = SHELL_OPEN_ZIPFILE; + } + } + fclose(f); + return rc; +} + +#ifdef SQLITE_ENABLE_DESERIALIZE +/* +** Reconstruct an in-memory database using the output from the "dbtotxt" +** program. Read content from the file in p->zDbFilename. If p->zDbFilename +** is 0, then read from standard input. +*/ +static unsigned char *readHexDb(ShellState *p, int *pnData){ + unsigned char *a = 0; + int nLine; + int n = 0; + int pgsz = 0; + int iOffset = 0; + int j, k; + int rc; + FILE *in; + unsigned int x[16]; + char zLine[1000]; + if( p->zDbFilename ){ + in = fopen(p->zDbFilename, "r"); + if( in==0 ){ + utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename); + return 0; + } + nLine = 0; + }else{ + in = p->in; + nLine = p->lineno; + if( in==0 ) in = stdin; + } + *pnData = 0; + nLine++; + if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); + if( rc!=2 ) goto readHexDb_error; + if( n<0 ) goto readHexDb_error; + if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error; + n = (n+pgsz-1)&~(pgsz-1); /* Round n up to the next multiple of pgsz */ + a = sqlite3_malloc( n ? n : 1 ); + if( a==0 ){ + utf8_printf(stderr, "Out of memory!\n"); + goto readHexDb_error; + } + memset(a, 0, n); + if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ + utf8_printf(stderr, "invalid pagesize\n"); + goto readHexDb_error; + } + for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + rc = sscanf(zLine, "| page %d offset %d", &j, &k); + if( rc==2 ){ + iOffset = k; + continue; + } + if( strncmp(zLine, "| end ", 6)==0 ){ + break; + } + rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", + &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], + &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); + if( rc==17 ){ + k = iOffset+j; + if( k+16<=n ){ + int ii; + for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; + } + } + } + *pnData = n; + if( in!=p->in ){ + fclose(in); + }else{ + p->lineno = nLine; + } + return a; + +readHexDb_error: + if( in!=p->in ){ + fclose(in); + }else{ + while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ + nLine++; + if(strncmp(zLine, "| end ", 6)==0 ) break; + } + p->lineno = nLine; + } + sqlite3_free(a); + utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); + return 0; +} +#endif /* SQLITE_ENABLE_DESERIALIZE */ + +/* +** Scalar function "shell_int32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void shellInt32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + UNUSED_PARAMETER(argc); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + + ((sqlite3_int64)a[1]<<16) + + ((sqlite3_int64)a[2]<< 8) + + ((sqlite3_int64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, +** using "..." with internal double-quote characters doubled. +*/ +static void shellIdQuote( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName = (const char*)sqlite3_value_text(argv[0]); + UNUSED_PARAMETER(argc); + if( zName ){ + char *z = sqlite3_mprintf("\"%w\"", zName); + sqlite3_result_text(context, z, -1, sqlite3_free); + } +} + +/* +** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. +*/ +static void shellUSleepFunc( + sqlite3_context *context, + int argcUnused, + sqlite3_value **argv +){ + int sleep = sqlite3_value_int(argv[0]); + (void)argcUnused; + sqlite3_sleep(sleep/1000); + sqlite3_result_int(context, sleep); +} + +/* +** Scalar function "shell_escape_crnl" used by the .recover command. +** The argument passed to this function is the output of built-in +** function quote(). If the first character of the input is "'", +** indicating that the value passed to quote() was a text value, +** then this function searches the input for "\n" and "\r" characters +** and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void shellEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + UNUSED_PARAMETER(argc); + if( zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + +/* Flags for open_db(). +** +** The default behavior of open_db() is to exit(1) if the database fails to +** open. The OPEN_DB_KEEPALIVE flag changes that so that it prints an error +** but still returns without calling exit. +** +** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a +** ZIP archive if the file does not exist or is empty and its name matches +** the *.zip pattern. +*/ +#define OPEN_DB_KEEPALIVE 0x001 /* Return after error if true */ +#define OPEN_DB_ZIPFILE 0x002 /* Open as ZIP if name matches *.zip */ + +/* +** Make sure the database is open. If it is not, then open it. If +** the database fails to open, print an error message and exit. +*/ +static void open_db(ShellState *p, int openFlags){ + if( p->db==0 ){ + if( p->openMode==SHELL_OPEN_UNSPEC ){ + if( p->zDbFilename==0 || p->zDbFilename[0]==0 ){ + p->openMode = SHELL_OPEN_NORMAL; + }else{ + p->openMode = (u8)deduceDatabaseType(p->zDbFilename, + (openFlags & OPEN_DB_ZIPFILE)!=0); + } + } + switch( p->openMode ){ + case SHELL_OPEN_APPENDVFS: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); + break; + } + case SHELL_OPEN_HEXDB: + case SHELL_OPEN_DESERIALIZE: { + sqlite3_open(0, &p->db); + break; + } + case SHELL_OPEN_ZIPFILE: { + sqlite3_open(":memory:", &p->db); + break; + } + case SHELL_OPEN_READONLY: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READONLY|p->openFlags, 0); + break; + } + case SHELL_OPEN_UNSPEC: + case SHELL_OPEN_NORMAL: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0); + break; + } + } + globalDb = p->db; + if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ + utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", + p->zDbFilename, sqlite3_errmsg(p->db)); + if( openFlags & OPEN_DB_KEEPALIVE ){ + sqlite3_open(":memory:", &p->db); + return; + } + exit(1); + } +#ifndef SQLITE_OMIT_LOAD_EXTENSION + sqlite3_enable_load_extension(p->db, 1); +#endif + sqlite3_fileio_init(p->db, 0, 0); + sqlite3_shathree_init(p->db, 0, 0); + sqlite3_completion_init(p->db, 0, 0); + sqlite3_uint_init(p->db, 0, 0); + sqlite3_decimal_init(p->db, 0, 0); + sqlite3_ieee_init(p->db, 0, 0); + sqlite3_series_init(p->db, 0, 0); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + sqlite3_dbdata_init(p->db, 0, 0); +#endif +#ifdef SQLITE_HAVE_ZLIB + sqlite3_zipfile_init(p->db, 0, 0); + sqlite3_sqlar_init(p->db, 0, 0); +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, + shellAddSchemaName, 0, 0); + sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, + shellModuleSchema, 0, 0); + sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, + shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, + shellEscapeCrnl, 0, 0); + sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, + shellInt32, 0, 0); + sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, + shellIdQuote, 0, 0); + sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, + shellUSleepFunc, 0, 0); +#ifndef SQLITE_NOHAVE_SYSTEM + sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, + editFunc, 0, 0); + sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, + editFunc, 0, 0); +#endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ + char *zSql = sqlite3_mprintf( + "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename); + sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +#ifdef SQLITE_ENABLE_DESERIALIZE + else + if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){ + int rc; + int nData = 0; + unsigned char *aData; + if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + aData = (unsigned char*)readFile(p->zDbFilename, &nData); + }else{ + aData = readHexDb(p, &nData); + if( aData==0 ){ + return; + } + } + rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + } + if( p->szMax>0 ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); + } + } +#endif + } +} + +/* +** Attempt to close the databaes connection. Report errors. +*/ +void close_db(sqlite3 *db){ + int rc = sqlite3_close(db); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", + rc, sqlite3_errmsg(db)); + } +} + +#if HAVE_READLINE || HAVE_EDITLINE +/* +** Readline completion callbacks +*/ +static char *readline_completion_generator(const char *text, int state){ + static sqlite3_stmt *pStmt = 0; + char *zRet; + if( state==0 ){ + char *zSql; + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" + " FROM completion(%Q) ORDER BY 1", text); + sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + zRet = strdup((const char*)sqlite3_column_text(pStmt, 0)); + }else{ + sqlite3_finalize(pStmt); + pStmt = 0; + zRet = 0; + } + return zRet; +} +static char **readline_completion(const char *zText, int iStart, int iEnd){ + rl_attempted_completion_over = 1; + return rl_completion_matches(zText, readline_completion_generator); +} + +#elif HAVE_LINENOISE +/* +** Linenoise completion callback +*/ +static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ + int nLine = strlen30(zLine); + int i, iStart; + sqlite3_stmt *pStmt = 0; + char *zSql; + char zBuf[1000]; + + if( nLine>sizeof(zBuf)-30 ) return; + if( zLine[0]=='.' || zLine[0]=='#') return; + for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} + if( i==nLine-1 ) return; + iStart = i+1; + memcpy(zBuf, zLine, iStart); + zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" + " FROM completion(%Q,%Q) ORDER BY 1", + &zLine[iStart], zLine); + sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + sqlite3_exec(globalDb, "PRAGMA page_count", 0, 0, 0); /* Load the schema */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCompletion = (const char*)sqlite3_column_text(pStmt, 0); + int nCompletion = sqlite3_column_bytes(pStmt, 0); + if( iStart+nCompletion < sizeof(zBuf)-1 ){ + memcpy(zBuf+iStart, zCompletion, nCompletion+1); + linenoiseAddCompletion(lc, zBuf); + } + } + sqlite3_finalize(pStmt); +} +#endif + +/* +** Do C-language style dequoting. +** +** \a -> alarm +** \b -> backspace +** \t -> tab +** \n -> newline +** \v -> vertical tab +** \f -> form feed +** \r -> carriage return +** \s -> space +** \" -> " +** \' -> ' +** \\ -> backslash +** \NNN -> ascii character NNN in octal +*/ +static void resolve_backslashes(char *z){ + int i, j; + char c; + while( *z && *z!='\\' ) z++; + for(i=j=0; (c = z[i])!=0; i++, j++){ + if( c=='\\' && z[i+1]!=0 ){ + c = z[++i]; + if( c=='a' ){ + c = '\a'; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='"' ){ + c = '"'; + }else if( c=='\'' ){ + c = '\''; + }else if( c=='\\' ){ + c = '\\'; + }else if( c>='0' && c<='7' ){ + c -= '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + } + } + } + } + z[j] = c; + } + if( j=0; i++){} + }else{ + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + } + if( i>0 && zArg[i]==0 ) return (int)(integerValue(zArg) & 0xffffffff); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; + } + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", + zArg); + return 0; +} + +/* +** Set or clear a shell flag according to a boolean value. +*/ +static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ + if( booleanValue(zArg) ){ + ShellSetFlag(p, mFlag); + }else{ + ShellClearFlag(p, mFlag); + } +} + +/* +** Close an output file, assuming it is not stderr or stdout +*/ +static void output_file_close(FILE *f){ + if( f && f!=stdout && f!=stderr ) fclose(f); +} + +/* +** Try to open an output file. The names "stdout" and "stderr" are +** recognized and do the right thing. NULL is returned if the output +** filename is "off". +*/ +static FILE *output_file_open(const char *zFile, int bTextMode){ + FILE *f; + if( strcmp(zFile,"stdout")==0 ){ + f = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + f = stderr; + }else if( strcmp(zFile, "off")==0 ){ + f = 0; + }else{ + f = fopen(zFile, bTextMode ? "w" : "wb"); + if( f==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } + return f; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** A routine for handling output from sqlite3_trace(). +*/ +static int sql_trace_callback( + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ +){ + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } + } + return 0; +} +#endif + +/* +** A no-op routine that runs with the ".breakpoint" doc-command. This is +** a useful spot to set a debugger breakpoint. +*/ +static void test_breakpoint(void){ + static int nCall = 0; + nCall++; +} + +/* +** An object used to read a CSV and other files for import. +*/ +typedef struct ImportCtx ImportCtx; +struct ImportCtx { + const char *zFile; /* Name of the input file */ + FILE *in; /* Read the CSV text from this input stream */ + int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close in */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + int nRow; /* Number of rows imported */ + int nErr; /* Number of errors encountered */ + int bNotFirst; /* True if one or more bytes already read */ + int cTerm; /* Character that terminated the most recent field */ + int cColSep; /* The column separator character. (Usually ",") */ + int cRowSep; /* The row separator character. (Usually "\n") */ +}; + +/* Clean up resourced used by an ImportCtx */ +static void import_cleanup(ImportCtx *p){ + if( p->in!=0 && p->xCloser!=0 ){ + p->xCloser(p->in); + p->in = 0; + } + sqlite3_free(p->z); + p->z = 0; +} + +/* Append a single byte to z[] */ +static void import_append_char(ImportCtx *p, int c){ + if( p->n+1>=p->nAlloc ){ + p->nAlloc += p->nAlloc + 100; + p->z = sqlite3_realloc64(p->z, p->nAlloc); + if( p->z==0 ) shell_out_of_memory(); + } + p->z[p->n++] = (char)c; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Use p->cSep as the column separator. The default is ",". +** + Use p->rSep as the row separator. The default is "\n". +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** + Report syntax errors on stderr +*/ +static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ + int c; + int cSep = p->cColSep; + int rSep = p->cRowSep; + p->n = 0; + c = fgetc(p->in); + if( c==EOF || seenInterrupt ){ + p->cTerm = EOF; + return 0; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + int cQuote = c; + pc = ppc = 0; + while( 1 ){ + c = fgetc(p->in); + if( c==rSep ) p->nLine++; + if( c==cQuote ){ + if( pc==cQuote ){ + pc = 0; + continue; + } + } + if( (c==cSep && pc==cQuote) + || (c==rSep && pc==cQuote) + || (c==rSep && pc=='\r' && ppc==cQuote) + || (c==EOF && pc==cQuote) + ){ + do{ p->n--; }while( p->z[p->n]!=cQuote ); + p->cTerm = c; + break; + } + if( pc==cQuote && c!='\r' ){ + utf8_printf(stderr, "%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); + } + if( c==EOF ){ + utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n", + p->zFile, startLine, cQuote); + p->cTerm = c; + break; + } + import_append_char(p, c); + ppc = pc; + pc = c; + } + }else{ + /* If this is the first field being parsed and it begins with the + ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + if( (c&0xff)==0xef && p->bNotFirst==0 ){ + import_append_char(p, c); + c = fgetc(p->in); + if( (c&0xff)==0xbb ){ + import_append_char(p, c); + c = fgetc(p->in); + if( (c&0xff)==0xbf ){ + p->bNotFirst = 1; + p->n = 0; + return csv_read_one_field(p); + } + } + } + while( c!=EOF && c!=cSep && c!=rSep ){ + import_append_char(p, c); + c = fgetc(p->in); + } + if( c==rSep ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = c; + } + if( p->z ) p->z[p->n] = 0; + p->bNotFirst = 1; + return p->z; +} + +/* Read a single field of ASCII delimited text. +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Use p->cSep as the column separator. The default is "\x1F". +** + Use p->rSep as the row separator. The default is "\x1E". +** + Keep track of the row number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** + Report syntax errors on stderr +*/ +static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ + int c; + int cSep = p->cColSep; + int rSep = p->cRowSep; + p->n = 0; + c = fgetc(p->in); + if( c==EOF || seenInterrupt ){ + p->cTerm = EOF; + return 0; + } + while( c!=EOF && c!=cSep && c!=rSep ){ + import_append_char(p, c); + c = fgetc(p->in); + } + if( c==rSep ){ + p->nLine++; + } + p->cTerm = c; + if( p->z ) p->z[p->n] = 0; + return p->z; +} + +/* +** Try to transfer data for table zTable. If an error is seen while +** moving forward, try to go backwards. The backwards movement won't +** work for WITHOUT ROWID tables. +*/ +static void tryToCloneData( + ShellState *p, + sqlite3 *newDb, + const char *zTable +){ + sqlite3_stmt *pQuery = 0; + sqlite3_stmt *pInsert = 0; + char *zQuery = 0; + char *zInsert = 0; + int rc; + int i, j, n; + int nTable = strlen30(zTable); + int k = 0; + int cnt = 0; + const int spinRate = 10000; + + zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error %d: %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_data_xfer; + } + n = sqlite3_column_count(pQuery); + zInsert = sqlite3_malloc64(200 + nTable + n*3); + if( zInsert==0 ) shell_out_of_memory(); + sqlite3_snprintf(200+nTable,zInsert, + "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable); + i = strlen30(zInsert); + for(j=1; jdb, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable); + break; + } + } /* End for(k=0...) */ + +end_data_xfer: + sqlite3_finalize(pQuery); + sqlite3_finalize(pInsert); + sqlite3_free(zQuery); + sqlite3_free(zInsert); +} + + +/* +** Try to transfer all rows of the schema that match zWhere. For +** each row, invoke xForEach() on the object defined by that row. +** If an error is encountered while moving forward through the +** sqlite_schema table, try again moving backwards. +*/ +static void tryToCloneSchema( + ShellState *p, + sqlite3 *newDb, + const char *zWhere, + void (*xForEach)(ShellState*,sqlite3*,const char*) +){ + sqlite3_stmt *pQuery = 0; + char *zQuery = 0; + int rc; + const unsigned char *zName; + const unsigned char *zSql; + char *zErrMsg = 0; + + zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" + " WHERE %s", zWhere); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_schema_xfer; + } + while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ + zName = sqlite3_column_text(pQuery, 0); + zSql = sqlite3_column_text(pQuery, 1); + printf("%s... ", zName); fflush(stdout); + sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_free(zErrMsg); + zErrMsg = 0; + } + if( xForEach ){ + xForEach(p, newDb, (const char*)zName); + } + printf("done\n"); + } + if( rc!=SQLITE_DONE ){ + sqlite3_finalize(pQuery); + sqlite3_free(zQuery); + zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" + " WHERE %s ORDER BY rowid DESC", zWhere); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_schema_xfer; + } + while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ + zName = sqlite3_column_text(pQuery, 0); + zSql = sqlite3_column_text(pQuery, 1); + printf("%s... ", zName); fflush(stdout); + sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_free(zErrMsg); + zErrMsg = 0; + } + if( xForEach ){ + xForEach(p, newDb, (const char*)zName); + } + printf("done\n"); + } + } +end_schema_xfer: + sqlite3_finalize(pQuery); + sqlite3_free(zQuery); +} + +/* +** Open a new database file named "zNewDb". Try to recover as much information +** as possible out of the main database (which might be corrupt) and write it +** into zNewDb. +*/ +static void tryToClone(ShellState *p, const char *zNewDb){ + int rc; + sqlite3 *newDb = 0; + if( access(zNewDb,0)==0 ){ + utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb); + return; + } + rc = sqlite3_open(zNewDb, &newDb); + if( rc ){ + utf8_printf(stderr, "Cannot create output database: %s\n", + sqlite3_errmsg(newDb)); + }else{ + sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); + sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0); + tryToCloneSchema(p, newDb, "type='table'", tryToCloneData); + tryToCloneSchema(p, newDb, "type!='table'", 0); + sqlite3_exec(newDb, "COMMIT;", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); + } + close_db(newDb); +} + +/* +** Change the output file back to stdout. +** +** If the p->doXdgOpen flag is set, that means the output was being +** redirected to a temporary file named by p->zTempFile. In that case, +** launch start/open/xdg-open on that temporary file. +*/ +static void output_reset(ShellState *p){ + if( p->outfile[0]=='|' ){ +#ifndef SQLITE_OMIT_POPEN + pclose(p->out); +#endif + }else{ + output_file_close(p->out); +#ifndef SQLITE_NOHAVE_SYSTEM + if( p->doXdgOpen ){ + const char *zXdgOpenCmd = +#if defined(_WIN32) + "start"; +#elif defined(__APPLE__) + "open"; +#else + "xdg-open"; +#endif + char *zCmd; + zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); + if( system(zCmd) ){ + utf8_printf(stderr, "Failed: [%s]\n", zCmd); + }else{ + /* Give the start/open/xdg-open command some time to get + ** going before we continue, and potential delete the + ** p->zTempFile data file out from under it */ + sqlite3_sleep(2000); + } + sqlite3_free(zCmd); + outputModePop(p); + p->doXdgOpen = 0; + } +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ + } + p->outfile[0] = 0; + p->out = stdout; +} + +/* +** Run an SQL command and return the single integer result. +*/ +static int db_int(ShellState *p, const char *zSql){ + sqlite3_stmt *pStmt; + int res = 0; + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + res = sqlite3_column_int(pStmt,0); + } + sqlite3_finalize(pStmt); + return res; +} + +/* +** Convert a 2-byte or 4-byte big-endian integer into a native integer +*/ +static unsigned int get2byteInt(unsigned char *a){ + return (a[0]<<8) + a[1]; +} +static unsigned int get4byteInt(unsigned char *a){ + return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; +} + +/* +** Implementation of the ".dbinfo" command. +** +** Return 1 on error, 2 to exit, and 0 otherwise. +*/ +static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ + static const struct { const char *zName; int ofst; } aField[] = { + { "file change counter:", 24 }, + { "database page count:", 28 }, + { "freelist page count:", 36 }, + { "schema cookie:", 40 }, + { "schema format:", 44 }, + { "default cache size:", 48 }, + { "autovacuum top root:", 52 }, + { "incremental vacuum:", 64 }, + { "text encoding:", 56 }, + { "user version:", 60 }, + { "application id:", 68 }, + { "software version:", 96 }, + }; + static const struct { const char *zName; const char *zSql; } aQuery[] = { + { "number of tables:", + "SELECT count(*) FROM %s WHERE type='table'" }, + { "number of indexes:", + "SELECT count(*) FROM %s WHERE type='index'" }, + { "number of triggers:", + "SELECT count(*) FROM %s WHERE type='trigger'" }, + { "number of views:", + "SELECT count(*) FROM %s WHERE type='view'" }, + { "schema size:", + "SELECT total(length(sql)) FROM %s" }, + }; + int i, rc; + unsigned iDataVersion; + char *zSchemaTab; + char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; + unsigned char aHdr[100]; + open_db(p, 0); + if( p->db==0 ) return 1; + rc = sqlite3_prepare_v2(p->db, + "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + return 1; + } + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ + raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); + return 1; + } + i = get2byteInt(aHdr+16); + if( i==1 ) i = 65536; + utf8_printf(p->out, "%-20s %d\n", "database page size:", i); + utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + for(i=0; iout, "%-20s %u", aField[i].zName, val); + switch( ofst ){ + case 56: { + if( val==1 ) raw_printf(p->out, " (utf8)"); + if( val==2 ) raw_printf(p->out, " (utf16le)"); + if( val==3 ) raw_printf(p->out, " (utf16be)"); + } + } + raw_printf(p->out, "\n"); + } + if( zDb==0 ){ + zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); + }else if( strcmp(zDb,"temp")==0 ){ + zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema"); + }else{ + zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb); + } + for(i=0; iout, "%-20s %d\n", aQuery[i].zName, val); + } + sqlite3_free(zSchemaTab); + sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); + utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); + return 0; +} + +/* +** Print the current sqlite3_errmsg() value to stderr and return 1. +*/ +static int shellDatabaseError(sqlite3 *db){ + const char *zErr = sqlite3_errmsg(db); + utf8_printf(stderr, "Error: %s\n", zErr); + return 1; +} + +/* +** Compare the pattern in zGlob[] against the text in z[]. Return TRUE +** if they match and FALSE (0) if they do not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +** +** ' ' Any span of whitespace matches any other span of +** whitespace. +** +** Extra whitespace at the end of z[] is ignored. +*/ +static int testcase_glob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( IsSpace(c) ){ + if( !IsSpace(*z) ) return 0; + while( IsSpace(*zGlob) ) zGlob++; + while( IsSpace(*z) ) z++; + }else if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && testcase_glob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( testcase_glob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++; + if( !IsDigit(z[0]) ) return 0; + z++; + while( IsDigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + while( IsSpace(*z) ){ z++; } + return *z==0; +} + + +/* +** Compare the string as a command-line option with either one or two +** initial "-" characters. +*/ +static int optionMatch(const char *zStr, const char *zOpt){ + if( zStr[0]!='-' ) return 0; + zStr++; + if( zStr[0]=='-' ) zStr++; + return strcmp(zStr, zOpt)==0; +} + +/* +** Delete a file. +*/ +int shellDeleteFile(const char *zFilename){ + int rc; +#ifdef _WIN32 + wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename); + rc = _wunlink(z); + sqlite3_free(z); +#else + rc = unlink(zFilename); +#endif + return rc; +} + +/* +** Try to delete the temporary file (if there is one) and free the +** memory used to hold the name of the temp file. +*/ +static void clearTempFile(ShellState *p){ + if( p->zTempFile==0 ) return; + if( p->doXdgOpen ) return; + if( shellDeleteFile(p->zTempFile) ) return; + sqlite3_free(p->zTempFile); + p->zTempFile = 0; +} + +/* +** Create a new temp file name with the given suffix. +*/ +static void newTempFile(ShellState *p, const char *zSuffix){ + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + if( p->db ){ + sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); + } + if( p->zTempFile==0 ){ + /* If p->db is an in-memory database then the TEMPFILENAME file-control + ** will not work and we will need to fallback to guessing */ + char *zTemp; + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r), &r); + zTemp = getenv("TEMP"); + if( zTemp==0 ) zTemp = getenv("TMP"); + if( zTemp==0 ){ +#ifdef _WIN32 + zTemp = "\\tmp"; +#else + zTemp = "/tmp"; +#endif + } + p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); + }else{ + p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + } + if( p->zTempFile==0 ){ + raw_printf(stderr, "out of memory\n"); + exit(1); + } +} + + +/* +** The implementation of SQL scalar function fkey_collate_clause(), used +** by the ".lint fkey-indexes" command. This scalar function is always +** called with four arguments - the parent table name, the parent column name, +** the child table name and the child column name. +** +** fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col') +** +** If either of the named tables or columns do not exist, this function +** returns an empty string. An empty string is also returned if both tables +** and columns exist but have the same default collation sequence. Or, +** if both exist but the default collation sequences are different, this +** function returns the string " COLLATE ", where +** is the default collation sequence of the parent column. +*/ +static void shellFkeyCollateClause( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zParent; + const char *zParentCol; + const char *zParentSeq; + const char *zChild; + const char *zChildCol; + const char *zChildSeq = 0; /* Initialize to avoid false-positive warning */ + int rc; + + assert( nVal==4 ); + zParent = (const char*)sqlite3_value_text(apVal[0]); + zParentCol = (const char*)sqlite3_value_text(apVal[1]); + zChild = (const char*)sqlite3_value_text(apVal[2]); + zChildCol = (const char*)sqlite3_value_text(apVal[3]); + + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = sqlite3_table_column_metadata( + db, "main", zParent, zParentCol, 0, &zParentSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_table_column_metadata( + db, "main", zChild, zChildCol, 0, &zChildSeq, 0, 0, 0 + ); + } + + if( rc==SQLITE_OK && sqlite3_stricmp(zParentSeq, zChildSeq) ){ + char *z = sqlite3_mprintf(" COLLATE %s", zParentSeq); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); + sqlite3_free(z); + } +} + + +/* +** The implementation of dot-command ".lint fkey-indexes". +*/ +static int lintFkeyIndexes( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + sqlite3 *db = pState->db; /* Database handle to query "main" db of */ + FILE *out = pState->out; /* Stream to write non-error output to */ + int bVerbose = 0; /* If -verbose is present */ + int bGroupByParent = 0; /* If -groupbyparent is present */ + int i; /* To iterate through azArg[] */ + const char *zIndent = ""; /* How much to indent CREATE INDEX by */ + int rc; /* Return code */ + sqlite3_stmt *pSql = 0; /* Compiled version of SQL statement below */ + + /* + ** This SELECT statement returns one row for each foreign key constraint + ** in the schema of the main database. The column values are: + ** + ** 0. The text of an SQL statement similar to: + ** + ** "EXPLAIN QUERY PLAN SELECT 1 FROM child_table WHERE child_key=?" + ** + ** This SELECT is similar to the one that the foreign keys implementation + ** needs to run internally on child tables. If there is an index that can + ** be used to optimize this query, then it can also be used by the FK + ** implementation to optimize DELETE or UPDATE statements on the parent + ** table. + ** + ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by + ** the EXPLAIN QUERY PLAN command matches this pattern, then the schema + ** contains an index that can be used to optimize the query. + ** + ** 2. Human readable text that describes the child table and columns. e.g. + ** + ** "child_table(child_key1, child_key2)" + ** + ** 3. Human readable text that describes the parent table and columns. e.g. + ** + ** "parent_table(parent_key1, parent_key2)" + ** + ** 4. A full CREATE INDEX statement for an index that could be used to + ** optimize DELETE or UPDATE statements on the parent table. e.g. + ** + ** "CREATE INDEX child_table_child_key ON child_table(child_key)" + ** + ** 5. The name of the parent table. + ** + ** These six values are used by the C logic below to generate the report. + */ + const char *zSql = + "SELECT " + " 'EXPLAIN QUERY PLAN SELECT 1 FROM ' || quote(s.name) || ' WHERE '" + " || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' " + " || fkey_collate_clause(" + " f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')" + ", " + " 'SEARCH TABLE ' || s.name || ' USING COVERING INDEX*('" + " || group_concat('*=?', ' AND ') || ')'" + ", " + " s.name || '(' || group_concat(f.[from], ', ') || ')'" + ", " + " f.[table] || '(' || group_concat(COALESCE(f.[to], p.[name])) || ')'" + ", " + " 'CREATE INDEX ' || quote(s.name ||'_'|| group_concat(f.[from], '_'))" + " || ' ON ' || quote(s.name) || '('" + " || group_concat(quote(f.[from]) ||" + " fkey_collate_clause(" + " f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]), ', ')" + " || ');'" + ", " + " f.[table] " + "FROM sqlite_schema AS s, pragma_foreign_key_list(s.name) AS f " + "LEFT JOIN pragma_table_info AS p ON (pk-1=seq AND p.arg=f.[table]) " + "GROUP BY s.name, f.id " + "ORDER BY (CASE WHEN ? THEN f.[table] ELSE s.name END)" + ; + const char *zGlobIPK = "SEARCH TABLE * USING INTEGER PRIMARY KEY (rowid=?)"; + + for(i=2; i1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){ + bVerbose = 1; + } + else if( n>1 && sqlite3_strnicmp("-groupbyparent", azArg[i], n)==0 ){ + bGroupByParent = 1; + zIndent = " "; + } + else{ + raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", + azArg[0], azArg[1] + ); + return SQLITE_ERROR; + } + } + + /* Register the fkey_collate_clause() SQL function */ + rc = sqlite3_create_function(db, "fkey_collate_clause", 4, SQLITE_UTF8, + 0, shellFkeyCollateClause, 0, 0 + ); + + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pSql, 1, bGroupByParent); + } + + if( rc==SQLITE_OK ){ + int rc2; + char *zPrev = 0; + while( SQLITE_ROW==sqlite3_step(pSql) ){ + int res = -1; + sqlite3_stmt *pExplain = 0; + const char *zEQP = (const char*)sqlite3_column_text(pSql, 0); + const char *zGlob = (const char*)sqlite3_column_text(pSql, 1); + const char *zFrom = (const char*)sqlite3_column_text(pSql, 2); + const char *zTarget = (const char*)sqlite3_column_text(pSql, 3); + const char *zCI = (const char*)sqlite3_column_text(pSql, 4); + const char *zParent = (const char*)sqlite3_column_text(pSql, 5); + + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc!=SQLITE_OK ) break; + if( SQLITE_ROW==sqlite3_step(pExplain) ){ + const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3); + res = ( + 0==sqlite3_strglob(zGlob, zPlan) + || 0==sqlite3_strglob(zGlobIPK, zPlan) + ); + } + rc = sqlite3_finalize(pExplain); + if( rc!=SQLITE_OK ) break; + + if( res<0 ){ + raw_printf(stderr, "Error: internal error"); + break; + }else{ + if( bGroupByParent + && (bVerbose || res==0) + && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) + ){ + raw_printf(out, "-- Parent table %s\n", zParent); + sqlite3_free(zPrev); + zPrev = sqlite3_mprintf("%s", zParent); + } + + if( res==0 ){ + raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + }else if( bVerbose ){ + raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n", + zIndent, zFrom, zTarget + ); + } + } + } + sqlite3_free(zPrev); + + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + rc2 = sqlite3_finalize(pSql); + if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ + rc = rc2; + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + }else{ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + return rc; +} + +/* +** Implementation of ".lint" dot command. +*/ +static int lintDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int n; + n = (nArg>=2 ? strlen30(azArg[1]) : 0); + if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage; + return lintFkeyIndexes(pState, azArg, nArg); + + usage: + raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]); + raw_printf(stderr, "Where sub-commands are:\n"); + raw_printf(stderr, " fkey-indexes\n"); + return SQLITE_ERROR; +} + +#if !defined SQLITE_OMIT_VIRTUALTABLE +static void shellPrepare( + sqlite3 *db, + int *pRc, + const char *zSql, + sqlite3_stmt **ppStmt +){ + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "sql error: %s (%d)\n", + sqlite3_errmsg(db), sqlite3_errcode(db) + ); + *pRc = rc; + } + } +} + +/* +** Create a prepared statement using printf-style arguments for the SQL. +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellPreparePrintf( + sqlite3 *db, + int *pRc, + sqlite3_stmt **ppStmt, + const char *zFmt, + ... +){ + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellPrepare(db, pRc, z, ppStmt); + sqlite3_free(z); + } + } +} + +/* Finalize the prepared statement created using shellPreparePrintf(). +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellFinalize( + int *pRc, + sqlite3_stmt *pStmt +){ + if( pStmt ){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ){ + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + } + *pRc = rc; + } + } +} + +/* Reset the prepared statement created using shellPreparePrintf(). +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellReset( + int *pRc, + sqlite3_stmt *pStmt +){ + int rc = sqlite3_reset(pStmt); + if( *pRc==SQLITE_OK ){ + if( rc!=SQLITE_OK ){ + sqlite3 *db = sqlite3_db_handle(pStmt); + raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + } + *pRc = rc; + } +} +#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +/****************************************************************************** +** The ".archive" or ".ar" command. +*/ +/* +** Structure representing a single ".ar" command. +*/ +typedef struct ArCommand ArCommand; +struct ArCommand { + u8 eCmd; /* An AR_CMD_* value */ + u8 bVerbose; /* True if --verbose */ + u8 bZip; /* True if the archive is a ZIP */ + u8 bDryRun; /* True if --dry-run */ + u8 bAppend; /* True if --append */ + u8 fromCmdLine; /* Run from -A instead of .archive */ + int nArg; /* Number of command arguments */ + char *zSrcTable; /* "sqlar", "zipfile($file)" or "zip" */ + const char *zFile; /* --file argument, or NULL */ + const char *zDir; /* --directory argument, or NULL */ + char **azArg; /* Array of command arguments */ + ShellState *p; /* Shell state */ + sqlite3 *db; /* Database containing the archive */ +}; + +/* +** Print a usage message for the .ar command to stderr and return SQLITE_ERROR. +*/ +static int arUsage(FILE *f){ + showHelp(f,"archive"); + return SQLITE_ERROR; +} + +/* +** Print an error message for the .ar command to stderr and return +** SQLITE_ERROR. +*/ +static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + utf8_printf(stderr, "Error: %s\n", z); + if( pAr->fromCmdLine ){ + utf8_printf(stderr, "Use \"-A\" for more help\n"); + }else{ + utf8_printf(stderr, "Use \".archive --help\" for more help\n"); + } + sqlite3_free(z); + return SQLITE_ERROR; +} + +/* +** Values for ArCommand.eCmd. +*/ +#define AR_CMD_CREATE 1 +#define AR_CMD_UPDATE 2 +#define AR_CMD_INSERT 3 +#define AR_CMD_EXTRACT 4 +#define AR_CMD_LIST 5 +#define AR_CMD_HELP 6 + +/* +** Other (non-command) switches. +*/ +#define AR_SWITCH_VERBOSE 7 +#define AR_SWITCH_FILE 8 +#define AR_SWITCH_DIRECTORY 9 +#define AR_SWITCH_APPEND 10 +#define AR_SWITCH_DRYRUN 11 + +static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ + switch( eSwitch ){ + case AR_CMD_CREATE: + case AR_CMD_EXTRACT: + case AR_CMD_LIST: + case AR_CMD_UPDATE: + case AR_CMD_INSERT: + case AR_CMD_HELP: + if( pAr->eCmd ){ + return arErrorMsg(pAr, "multiple command options"); + } + pAr->eCmd = eSwitch; + break; + + case AR_SWITCH_DRYRUN: + pAr->bDryRun = 1; + break; + case AR_SWITCH_VERBOSE: + pAr->bVerbose = 1; + break; + case AR_SWITCH_APPEND: + pAr->bAppend = 1; + /* Fall thru into --file */ + case AR_SWITCH_FILE: + pAr->zFile = zArg; + break; + case AR_SWITCH_DIRECTORY: + pAr->zDir = zArg; + break; + } + + return SQLITE_OK; +} + +/* +** Parse the command line for an ".ar" command. The results are written into +** structure (*pAr). SQLITE_OK is returned if the command line is parsed +** successfully, otherwise an error message is written to stderr and +** SQLITE_ERROR returned. +*/ +static int arParseCommand( + char **azArg, /* Array of arguments passed to dot command */ + int nArg, /* Number of entries in azArg[] */ + ArCommand *pAr /* Populate this object */ +){ + struct ArSwitch { + const char *zLong; + char cShort; + u8 eSwitch; + u8 bArg; + } aSwitch[] = { + { "create", 'c', AR_CMD_CREATE, 0 }, + { "extract", 'x', AR_CMD_EXTRACT, 0 }, + { "insert", 'i', AR_CMD_INSERT, 0 }, + { "list", 't', AR_CMD_LIST, 0 }, + { "update", 'u', AR_CMD_UPDATE, 0 }, + { "help", 'h', AR_CMD_HELP, 0 }, + { "verbose", 'v', AR_SWITCH_VERBOSE, 0 }, + { "file", 'f', AR_SWITCH_FILE, 1 }, + { "append", 'a', AR_SWITCH_APPEND, 1 }, + { "directory", 'C', AR_SWITCH_DIRECTORY, 1 }, + { "dryrun", 'n', AR_SWITCH_DRYRUN, 0 }, + }; + int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch); + struct ArSwitch *pEnd = &aSwitch[nSwitch]; + + if( nArg<=1 ){ + utf8_printf(stderr, "Wrong number of arguments. Usage:\n"); + return arUsage(stderr); + }else{ + char *z = azArg[1]; + if( z[0]!='-' ){ + /* Traditional style [tar] invocation */ + int i; + int iArg = 2; + for(i=0; z[i]; i++){ + const char *zArg = 0; + struct ArSwitch *pOpt; + for(pOpt=&aSwitch[0]; pOptcShort ) break; + } + if( pOpt==pEnd ){ + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); + } + if( pOpt->bArg ){ + if( iArg>=nArg ){ + return arErrorMsg(pAr, "option requires an argument: %c",z[i]); + } + zArg = azArg[iArg++]; + } + if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR; + } + pAr->nArg = nArg-iArg; + if( pAr->nArg>0 ){ + pAr->azArg = &azArg[iArg]; + } + }else{ + /* Non-traditional invocation */ + int iArg; + for(iArg=1; iArgazArg = &azArg[iArg]; + pAr->nArg = nArg-iArg; + break; + } + n = strlen30(z); + + if( z[1]!='-' ){ + int i; + /* One or more short options */ + for(i=1; icShort ) break; + } + if( pOpt==pEnd ){ + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); + } + if( pOpt->bArg ){ + if( i<(n-1) ){ + zArg = &z[i+1]; + i = n; + }else{ + if( iArg>=(nArg-1) ){ + return arErrorMsg(pAr, "option requires an argument: %c", + z[i]); + } + zArg = azArg[++iArg]; + } + } + if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR; + } + }else if( z[2]=='\0' ){ + /* A -- option, indicating that all remaining command line words + ** are command arguments. */ + pAr->azArg = &azArg[iArg+1]; + pAr->nArg = nArg-iArg-1; + break; + }else{ + /* A long option */ + const char *zArg = 0; /* Argument for option, if any */ + struct ArSwitch *pMatch = 0; /* Matching option */ + struct ArSwitch *pOpt; /* Iterator */ + for(pOpt=&aSwitch[0]; pOptzLong; + if( (n-2)<=strlen30(zLong) && 0==memcmp(&z[2], zLong, n-2) ){ + if( pMatch ){ + return arErrorMsg(pAr, "ambiguous option: %s",z); + }else{ + pMatch = pOpt; + } + } + } + + if( pMatch==0 ){ + return arErrorMsg(pAr, "unrecognized option: %s", z); + } + if( pMatch->bArg ){ + if( iArg>=(nArg-1) ){ + return arErrorMsg(pAr, "option requires an argument: %s", z); + } + zArg = azArg[++iArg]; + } + if( arProcessSwitch(pAr, pMatch->eSwitch, zArg) ) return SQLITE_ERROR; + } + } + } + } + + return SQLITE_OK; +} + +/* +** This function assumes that all arguments within the ArCommand.azArg[] +** array refer to archive members, as for the --extract or --list commands. +** It checks that each of them are present. If any specified file is not +** present in the archive, an error is printed to stderr and an error +** code returned. Otherwise, if all specified arguments are present in +** the archive, SQLITE_OK is returned. +** +** This function strips any trailing '/' characters from each argument. +** This is consistent with the way the [tar] command seems to work on +** Linux. +*/ +static int arCheckEntries(ArCommand *pAr){ + int rc = SQLITE_OK; + if( pAr->nArg ){ + int i, j; + sqlite3_stmt *pTest = 0; + + shellPreparePrintf(pAr->db, &rc, &pTest, + "SELECT name FROM %s WHERE name=$name", + pAr->zSrcTable + ); + j = sqlite3_bind_parameter_index(pTest, "$name"); + for(i=0; inArg && rc==SQLITE_OK; i++){ + char *z = pAr->azArg[i]; + int n = strlen30(z); + int bOk = 0; + while( n>0 && z[n-1]=='/' ) n--; + z[n] = '\0'; + sqlite3_bind_text(pTest, j, z, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pTest) ){ + bOk = 1; + } + shellReset(&rc, pTest); + if( rc==SQLITE_OK && bOk==0 ){ + utf8_printf(stderr, "not found in archive: %s\n", z); + rc = SQLITE_ERROR; + } + } + shellFinalize(&rc, pTest); + } + return rc; +} + +/* +** Format a WHERE clause that can be used against the "sqlar" table to +** identify all archive members that match the command arguments held +** in (*pAr). Leave this WHERE clause in (*pzWhere) before returning. +** The caller is responsible for eventually calling sqlite3_free() on +** any non-NULL (*pzWhere) value. +*/ +static void arWhereClause( + int *pRc, + ArCommand *pAr, + char **pzWhere /* OUT: New WHERE clause */ +){ + char *zWhere = 0; + if( *pRc==SQLITE_OK ){ + if( pAr->nArg==0 ){ + zWhere = sqlite3_mprintf("1"); + }else{ + int i; + const char *zSep = ""; + for(i=0; inArg; i++){ + const char *z = pAr->azArg[i]; + zWhere = sqlite3_mprintf( + "%z%s name = '%q' OR substr(name,1,%d) = '%q/'", + zWhere, zSep, z, strlen30(z)+1, z + ); + if( zWhere==0 ){ + *pRc = SQLITE_NOMEM; + break; + } + zSep = " OR "; + } + } + } + *pzWhere = zWhere; +} + +/* +** Implementation of .ar "lisT" command. +*/ +static int arListCommand(ArCommand *pAr){ + const char *zSql = "SELECT %s FROM %s WHERE %s"; + const char *azCols[] = { + "name", + "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name" + }; + + char *zWhere = 0; + sqlite3_stmt *pSql = 0; + int rc; + + rc = arCheckEntries(pAr); + arWhereClause(&rc, pAr, &zWhere); + + shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], + pAr->zSrcTable, zWhere); + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( pAr->bVerbose ){ + utf8_printf(pAr->p->out, "%s % 10d %s %s\n", + sqlite3_column_text(pSql, 0), + sqlite3_column_int(pSql, 1), + sqlite3_column_text(pSql, 2), + sqlite3_column_text(pSql, 3) + ); + }else{ + utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + } + } + } + shellFinalize(&rc, pSql); + sqlite3_free(zWhere); + return rc; +} + + +/* +** Implementation of .ar "eXtract" command. +*/ +static int arExtractCommand(ArCommand *pAr){ + const char *zSql1 = + "SELECT " + " ($dir || name)," + " writefile(($dir || name), %s, mode, mtime) " + "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" + " AND name NOT GLOB '*..[/\\]*'"; + + const char *azExtraArg[] = { + "sqlar_uncompress(data, sz)", + "data" + }; + + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + char *zDir = 0; + char *zWhere = 0; + int i, j; + + /* If arguments are specified, check that they actually exist within + ** the archive before proceeding. And formulate a WHERE clause to + ** match them. */ + rc = arCheckEntries(pAr); + arWhereClause(&rc, pAr, &zWhere); + + if( rc==SQLITE_OK ){ + if( pAr->zDir ){ + zDir = sqlite3_mprintf("%s/", pAr->zDir); + }else{ + zDir = sqlite3_mprintf(""); + } + if( zDir==0 ) rc = SQLITE_NOMEM; + } + + shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, + azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere + ); + + if( rc==SQLITE_OK ){ + j = sqlite3_bind_parameter_index(pSql, "$dir"); + sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); + + /* Run the SELECT statement twice. The first time, writefile() is called + ** for all archive members that should be extracted. The second time, + ** only for the directories. This is because the timestamps for + ** extracted directories must be reset after they are populated (as + ** populating them changes the timestamp). */ + for(i=0; i<2; i++){ + j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); + sqlite3_bind_int(pSql, j, i); + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + } + } + } + shellReset(&rc, pSql); + } + shellFinalize(&rc, pSql); + } + + sqlite3_free(zDir); + sqlite3_free(zWhere); + return rc; +} + +/* +** Run the SQL statement in zSql. Or if doing a --dryrun, merely print it out. +*/ +static int arExecSql(ArCommand *pAr, const char *zSql){ + int rc; + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", zSql); + rc = SQLITE_OK; + }else{ + char *zErr = 0; + rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); + if( zErr ){ + utf8_printf(stdout, "ERROR: %s\n", zErr); + sqlite3_free(zErr); + } + } + return rc; +} + + +/* +** Implementation of .ar "create", "insert", and "update" commands. +** +** create -> Create a new SQL archive +** insert -> Insert or reinsert all files listed +** update -> Insert files that have changed or that were not +** previously in the archive +** +** Create the "sqlar" table in the database if it does not already exist. +** Then add each file in the azFile[] array to the archive. Directories +** are added recursively. If argument bVerbose is non-zero, a message is +** printed on stdout for each file archived. +** +** The create command is the same as update, except that it drops +** any existing "sqlar" table before beginning. The "insert" command +** always overwrites every file named on the command-line, where as +** "update" only overwrites if the size or mtime or mode has changed. +*/ +static int arCreateOrUpdateCommand( + ArCommand *pAr, /* Command arguments and options */ + int bUpdate, /* true for a --create. */ + int bOnlyIfChanged /* Only update if file has changed */ +){ + const char *zCreate = + "CREATE TABLE IF NOT EXISTS sqlar(\n" + " name TEXT PRIMARY KEY, -- name of the file\n" + " mode INT, -- access permissions\n" + " mtime INT, -- last modification time\n" + " sz INT, -- original file size\n" + " data BLOB -- compressed content\n" + ")"; + const char *zDrop = "DROP TABLE IF EXISTS sqlar"; + const char *zInsertFmt[2] = { + "REPLACE INTO %s(name,mode,mtime,sz,data)\n" + " SELECT\n" + " %s,\n" + " mode,\n" + " mtime,\n" + " CASE substr(lsmode(mode),1,1)\n" + " WHEN '-' THEN length(data)\n" + " WHEN 'd' THEN 0\n" + " ELSE -1 END,\n" + " sqlar_compress(data)\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + , + "REPLACE INTO %s(name,mode,mtime,data)\n" + " SELECT\n" + " %s,\n" + " mode,\n" + " mtime,\n" + " data\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + }; + int i; /* For iterating through azFile[] */ + int rc; /* Return code */ + const char *zTab = 0; /* SQL table into which to insert */ + char *zSql; + char zTemp[50]; + char *zExists = 0; + + arExecSql(pAr, "PRAGMA page_size=512"); + rc = arExecSql(pAr, "SAVEPOINT ar;"); + if( rc!=SQLITE_OK ) return rc; + zTemp[0] = 0; + if( pAr->bZip ){ + /* Initialize the zipfile virtual table, if necessary */ + if( pAr->zFile ){ + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r),&r); + sqlite3_snprintf(sizeof(zTemp),zTemp,"zip%016llx",r); + zTab = zTemp; + zSql = sqlite3_mprintf( + "CREATE VIRTUAL TABLE temp.%s USING zipfile(%Q)", + zTab, pAr->zFile + ); + rc = arExecSql(pAr, zSql); + sqlite3_free(zSql); + }else{ + zTab = "zip"; + } + }else{ + /* Initialize the table for an SQLAR */ + zTab = "sqlar"; + if( bUpdate==0 ){ + rc = arExecSql(pAr, zDrop); + if( rc!=SQLITE_OK ) goto end_ar_transaction; + } + rc = arExecSql(pAr, zCreate); + } + if( bOnlyIfChanged ){ + zExists = sqlite3_mprintf( + " AND NOT EXISTS(" + "SELECT 1 FROM %s AS mem" + " WHERE mem.name=disk.name" + " AND mem.mtime=disk.mtime" + " AND mem.mode=disk.mode)", zTab); + }else{ + zExists = sqlite3_mprintf(""); + } + if( zExists==0 ) rc = SQLITE_NOMEM; + for(i=0; inArg && rc==SQLITE_OK; i++){ + char *zSql2 = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab, + pAr->bVerbose ? "shell_putsnl(name)" : "name", + pAr->azArg[i], pAr->zDir, zExists); + rc = arExecSql(pAr, zSql2); + sqlite3_free(zSql2); + } +end_ar_transaction: + if( rc!=SQLITE_OK ){ + sqlite3_exec(pAr->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0); + }else{ + rc = arExecSql(pAr, "RELEASE ar;"); + if( pAr->bZip && pAr->zFile ){ + zSql = sqlite3_mprintf("DROP TABLE %s", zTemp); + arExecSql(pAr, zSql); + sqlite3_free(zSql); + } + } + sqlite3_free(zExists); + return rc; +} + +/* +** Implementation of ".ar" dot command. +*/ +static int arDotCommand( + ShellState *pState, /* Current shell tool state */ + int fromCmdLine, /* True if -A command-line option, not .ar cmd */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + ArCommand cmd; + int rc; + memset(&cmd, 0, sizeof(cmd)); + cmd.fromCmdLine = fromCmdLine; + rc = arParseCommand(azArg, nArg, &cmd); + if( rc==SQLITE_OK ){ + int eDbType = SHELL_OPEN_UNSPEC; + cmd.p = pState; + cmd.db = pState->db; + if( cmd.zFile ){ + eDbType = deduceDatabaseType(cmd.zFile, 1); + }else{ + eDbType = pState->openMode; + } + if( eDbType==SHELL_OPEN_ZIPFILE ){ + if( cmd.eCmd==AR_CMD_EXTRACT || cmd.eCmd==AR_CMD_LIST ){ + if( cmd.zFile==0 ){ + cmd.zSrcTable = sqlite3_mprintf("zip"); + }else{ + cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile); + } + } + cmd.bZip = 1; + }else if( cmd.zFile ){ + int flags; + if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS; + if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT + || cmd.eCmd==AR_CMD_UPDATE ){ + flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + }else{ + flags = SQLITE_OPEN_READONLY; + } + cmd.db = 0; + if( cmd.bDryRun ){ + utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile, + eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); + } + rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, + eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "cannot open file: %s (%s)\n", + cmd.zFile, sqlite3_errmsg(cmd.db) + ); + goto end_ar_command; + } + sqlite3_fileio_init(cmd.db, 0, 0); + sqlite3_sqlar_init(cmd.db, 0, 0); + sqlite3_create_function(cmd.db, "shell_putsnl", 1, SQLITE_UTF8, cmd.p, + shellPutsFunc, 0, 0); + + } + if( cmd.zSrcTable==0 && cmd.bZip==0 && cmd.eCmd!=AR_CMD_HELP ){ + if( cmd.eCmd!=AR_CMD_CREATE + && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) + ){ + utf8_printf(stderr, "database does not contain an 'sqlar' table\n"); + rc = SQLITE_ERROR; + goto end_ar_command; + } + cmd.zSrcTable = sqlite3_mprintf("sqlar"); + } + + switch( cmd.eCmd ){ + case AR_CMD_CREATE: + rc = arCreateOrUpdateCommand(&cmd, 0, 0); + break; + + case AR_CMD_EXTRACT: + rc = arExtractCommand(&cmd); + break; + + case AR_CMD_LIST: + rc = arListCommand(&cmd); + break; + + case AR_CMD_HELP: + arUsage(pState->out); + break; + + case AR_CMD_INSERT: + rc = arCreateOrUpdateCommand(&cmd, 1, 0); + break; + + default: + assert( cmd.eCmd==AR_CMD_UPDATE ); + rc = arCreateOrUpdateCommand(&cmd, 1, 1); + break; + } + } +end_ar_command: + if( cmd.db!=pState->db ){ + close_db(cmd.db); + } + sqlite3_free(cmd.zSrcTable); + + return rc; +} +/* End of the ".archive" or ".ar" command logic +*******************************************************************************/ +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +/* +** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, the SQL statement or statements in zSql are executed using +** database connection db and the error code written to *pRc before +** this function returns. +*/ +static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + char *zErr = 0; + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", zErr); + } + *pRc = rc; + } +} + +/* +** Like shellExec(), except that zFmt is a printf() style format string. +*/ +static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellExec(db, pRc, z); + } + sqlite3_free(z); + } +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, an attempt is made to allocate, zero and return a pointer +** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set +** to SQLITE_NOMEM and NULL returned. +*/ +static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, zFmt is treated as a printf() style string. The result of +** formatting it along with any trailing arguments is written into a +** buffer obtained from sqlite3_malloc(), and pointer to which is returned. +** It is the responsibility of the caller to eventually free this buffer +** using a call to sqlite3_free(). +** +** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL +** pointer returned. +*/ +static char *shellMPrintf(int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return z; +} + +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + char *zQuoted; /* Quoted version of table name */ + int nCol; /* Number of columns in table */ + char **azlCol; /* Array of column lists */ + int iPk; /* Index of IPK column */ +}; + +/* +** Free a RecoverTable object allocated by recoverFindTable() or +** recoverOrphanTable(). +*/ +static void recoverFreeTable(RecoverTable *pTab){ + if( pTab ){ + sqlite3_free(pTab->zQuoted); + if( pTab->azlCol ){ + int i; + for(i=0; i<=pTab->nCol; i++){ + sqlite3_free(pTab->azlCol[i]); + } + sqlite3_free(pTab->azlCol); + } + sqlite3_free(pTab); + } +} + +/* +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. +** Otherwise, it allocates and returns a RecoverTable object based on the +** final four arguments passed to this function. It is the responsibility +** of the caller to eventually free the returned object using +** recoverFreeTable(). +*/ +static RecoverTable *recoverNewTable( + int *pRc, /* IN/OUT: Error code */ + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + RecoverTable *pTab = 0; + + pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, + shellIdQuote, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk to the index + ** of the column, where columns are 0-numbered from left to right. + ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, + ** leave zPk as "_rowid_" and pTab->iPk at -2. */ + pTab->iPk = -2; + if( bIntkey ){ + shellPreparePrintf(dbtmp, &rc, &pPkFinder, + "SELECT cid, name FROM pragma_table_info(%Q) " + " WHERE pk=1 AND type='integer' COLLATE nocase" + " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" + , zName, zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ + pTab->iPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + } + + pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); + pTab->nCol = nSqlCol; + + if( bIntkey ){ + pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); + }else{ + pTab->azlCol[0] = shellMPrintf(&rc, ""); + } + i = 1; + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT %Q || group_concat(shell_idquote(name), ', ') " + " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " + "FROM pragma_table_info(%Q)", + bIntkey ? ", " : "", pTab->iPk, + bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", + zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zText = (const char*)sqlite3_column_text(pStmt, 0); + pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); + i++; + } + shellFinalize(&rc, pStmt); + + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; + if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ + recoverFreeTable(pTab); + pTab = 0; + } + return pTab; +} + +/* +** This function is called to search the schema recovered from the +** sqlite_schema table of the (possibly) corrupt database as part +** of a ".recover" command. Specifically, for a table with root page +** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the +** table must be a WITHOUT ROWID table, or if non-zero, not one of +** those. +** +** If a table is found, a (RecoverTable*) object is returned. Or, if +** no such table is found, but bIntkey is false and iRoot is the +** root page of an index in the recovered schema, then (*pbNoop) is +** set to true and NULL returned. Or, if there is no such table or +** index, NULL is returned and (*pbNoop) set to 0, indicating that +** the caller should write data to the orphans table. +*/ +static RecoverTable *recoverFindTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + int iRoot, /* Root page of table */ + int bIntkey, /* True for an intkey table */ + int nCol, /* Number of columns in table */ + int *pbNoop /* OUT: True if iRoot is root of index */ +){ + sqlite3_stmt *pStmt = 0; + RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); + break; + } + } + + shellFinalize(pRc, pStmt); + *pbNoop = bNoop; + return pRet; +} + +/* +** Return a RecoverTable object representing the orphans table. +*/ +static RecoverTable *recoverOrphanTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + const char *zLostAndFound, /* Base name for orphans table */ + int nCol /* Number of user data columns */ +){ + RecoverTable *pTab = 0; + if( nCol>=0 && *pRc==SQLITE_OK ){ + int i; + + /* This block determines the name of the orphan table. The prefered + ** name is zLostAndFound. But if that clashes with another name + ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 + ** and so on until a non-clashing name is found. */ + int iTab = 0; + char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); + sqlite3_stmt *pTest = 0; + shellPrepare(pState->db, pRc, + "SELECT 1 FROM recovery.schema WHERE name=?", &pTest + ); + if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ + shellReset(pRc, pTest); + sqlite3_free(zTab); + zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); + sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + } + shellFinalize(pRc, pTest); + + pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pTab ){ + pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); + pTab->nCol = nCol; + pTab->iPk = -2; + if( nCol>0 ){ + pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); + if( pTab->azlCol ){ + pTab->azlCol[nCol] = shellMPrintf(pRc, ""); + for(i=nCol-1; i>=0; i--){ + pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); + } + } + } + + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + }else{ + raw_printf(pState->out, + "CREATE TABLE %s(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted + ); + for(i=0; iout, ", c%d", i); + } + raw_printf(pState->out, ");\n"); + } + } + sqlite3_free(zTab); + } + return pTab; +} + +/* +** This function is called to recover data from the database. A script +** to construct a new database containing all recovered data is output +** on stream pState->out. +*/ +static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ + int rc = SQLITE_OK; + sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ + sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ + const char *zLostAndFound = "lost_and_found"; + int i; + int nOrphan = -1; + RecoverTable *pOrphan = 0; + + int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + int bRowids = 1; /* 0 if --no-rowids */ + for(i=1; iout, azArg[0]); + return 1; + } + } + + shellExecPrintf(pState->db, &rc, + /* Attach an in-memory database named 'recovery'. Create an indexed + ** cache of the sqlite_dbptr virtual table. */ + "PRAGMA writable_schema = on;" + "ATTACH %Q AS recovery;" + "DROP TABLE IF EXISTS recovery.dbptr;" + "DROP TABLE IF EXISTS recovery.freelist;" + "DROP TABLE IF EXISTS recovery.map;" + "DROP TABLE IF EXISTS recovery.schema;" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + ); + + if( bFreelist ){ + shellExec(pState->db, &rc, + "WITH trunk(pgno) AS (" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " + " WHERE x>0" + " UNION" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " + " FROM trunk WHERE x>0" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " + " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" + " UNION ALL" + " SELECT data, n-1, shell_int32(data, 2+n) " + " FROM freelist WHERE n>=0" + ")" + "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" + ); + } + + /* If this is an auto-vacuum database, add all pointer-map pages to + ** the freelist table. Do this regardless of whether or not + ** --freelist-corrupt was specified. */ + shellExec(pState->db, &rc, + "WITH ptrmap(pgno) AS (" + " SELECT 2 WHERE shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" + " )" + " UNION ALL " + " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " + " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" + ")" + "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" + ); + + shellExec(pState->db, &rc, + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr" + " WHERE pgno NOT IN freelist AND child NOT IN freelist;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + + /* Create the "map" table that will (eventually) contain instructions + ** for dealing with each page in the db that contains one or more + ** records. */ + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" + + /* Populate table [map]. If there are circular loops of pages in the + ** database, the following adds all pages in such a loop to the map + ** as individual root pages. This could be handled better. */ + "WITH pages(i, maxlen) AS (" + " SELECT page_count, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" + " ) FROM pragma_page_count WHERE page_count>0" + " UNION ALL" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" + ")" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" + " WITH p(orig, pgno, parent) AS (" + " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" + " UNION " + " SELECT i, p.parent, " + " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" + " )" + " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" + ") " + "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" + + /* Extract data from page 1 and any linked pages into table + ** recovery.schema. With the same schema as an sqlite_schema table. */ + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + "INSERT INTO recovery.schema SELECT " + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=1" + ")" + "GROUP BY pgno, cell;" + "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" + ); + + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + /* ".recover" might output content in an order which causes immediate + ** foreign key constraints to be violated. So disable foreign-key + ** constraint enforcement to prevent problems when running the output + ** script. */ + raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); + raw_printf(pState->out, "BEGIN;\n"); + raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' AND sql LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", + &zCreateTable[12] + ); + } + shellFinalize(&rc, pStmt); + } + + /* Figure out if an orphan table will be required. And if so, how many + ** user columns it should contain */ + shellPrepare(pState->db, &rc, + "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" + , &pLoop + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + nOrphan = sqlite3_column_int(pLoop, 0); + } + shellFinalize(&rc, pLoop); + pLoop = 0; + + shellPrepare(pState->db, &rc, + "SELECT pgno FROM recovery.map WHERE root=?", &pPages + ); + + shellPrepare(pState->db, &rc, + "SELECT max(field), group_concat(shell_escape_crnl(quote" + "(case when (? AND field<0) then NULL else value end)" + "), ', ')" + ", min(field) " + "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" + "GROUP BY cell", &pCells + ); + + /* Loop through each root page. */ + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" + " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" + ")", &pLoop + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + int iRoot = sqlite3_column_int(pLoop, 0); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); + int bNoop = 0; + RecoverTable *pTab; + + assert( bIntkey==0 || bIntkey==1 ); + pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + if( bNoop || rc ) continue; + if( pTab==0 ){ + if( pOrphan==0 ){ + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); + } + pTab = pOrphan; + if( pTab==0 ) break; + } + + if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } + sqlite3_bind_int(pPages, 1, iRoot); + if( bRowids==0 && pTab->iPk<0 ){ + sqlite3_bind_int(pCells, 1, 1); + }else{ + sqlite3_bind_int(pCells, 1, 0); + } + sqlite3_bind_int(pCells, 3, pTab->iPk); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + int iPgno = sqlite3_column_int(pPages, 0); + sqlite3_bind_int(pCells, 2, iPgno); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int nField = sqlite3_column_int(pCells, 0); + int iMin = sqlite3_column_int(pCells, 2); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + + RecoverTable *pTab2 = pTab; + if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ + if( pOrphan==0 ){ + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); + } + pTab2 = pOrphan; + if( pTab2==0 ) break; + } + + nField = nField+1; + if( pTab2==pOrphan ){ + raw_printf(pState->out, + "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", + pTab2->zQuoted, iRoot, iPgno, nField, + iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] + ); + }else{ + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab2->zQuoted, pTab2->azlCol[nField], zVal + ); + } + } + shellReset(&rc, pCells); + } + shellReset(&rc, pPages); + if( pTab!=pOrphan ) recoverFreeTable(pTab); + } + shellFinalize(&rc, pLoop); + shellFinalize(&rc, pPages); + shellFinalize(&rc, pCells); + recoverFreeTable(pOrphan); + + /* The rest of the schema */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + shellPrepare(pState->db, &rc, + "SELECT sql, name FROM recovery.schema " + "WHERE sql NOT LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + char *zPrint = shellMPrintf(&rc, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + ); + raw_printf(pState->out, "%s;\n", zPrint); + sqlite3_free(zPrint); + }else{ + raw_printf(pState->out, "%s;\n", zSql); + } + } + shellFinalize(&rc, pStmt); + } + + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); + raw_printf(pState->out, "COMMIT;\n"); + } + sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; +} +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + + +/* +** If an input line begins with "." then invoke this routine to +** process that line. +** +** Return 1 on error, 2 to exit, and 0 otherwise. +*/ +static int do_meta_command(char *zLine, ShellState *p){ + int h = 1; + int nArg = 0; + int n, c; + int rc = 0; + char *azArg[52]; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( p->expert.pExpert ){ + expertFinish(p, 1, 0); + } +#endif + + /* Parse the input line into tokens. + */ + while( zLine[h] && nArgdb, shellAuth, p); + }else{ + sqlite3_set_authorizer(p->db, 0, 0); + } + }else +#endif + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) + if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ + open_db(p, 0); + rc = arDotCommand(p, 0, azArg, nArg); + }else +#endif + + if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) + || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) + ){ + const char *zDestFile = 0; + const char *zDb = 0; + sqlite3 *pDest; + sqlite3_backup *pBackup; + int j; + int bAsync = 0; + const char *zVfs = 0; + for(j=1; jdb, zDb); + if( pBackup==0 ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + close_db(pDest); + return 1; + } + while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK ){} + sqlite3_backup_finish(pBackup); + if( rc==SQLITE_DONE ){ + rc = 0; + }else{ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + rc = 1; + } + close_db(pDest); + }else + + if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ + if( nArg==2 ){ + bail_on_error = booleanValue(azArg[1]); + }else{ + raw_printf(stderr, "Usage: .bail on|off\n"); + rc = 1; + } + }else + + if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + if( nArg==2 ){ + if( booleanValue(azArg[1]) ){ + setBinaryMode(p->out, 1); + }else{ + setTextMode(p->out, 1); + } + }else{ + raw_printf(stderr, "Usage: .binary on|off\n"); + rc = 1; + } + }else + + if( c=='c' && strcmp(azArg[0],"cd")==0 ){ + if( nArg==2 ){ +#if defined(_WIN32) || defined(WIN32) + wchar_t *z = sqlite3_win32_utf8_to_unicode(azArg[1]); + rc = !SetCurrentDirectoryW(z); + sqlite3_free(z); +#else + rc = chdir(azArg[1]); +#endif + if( rc ){ + utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]); + rc = 1; + } + }else{ + raw_printf(stderr, "Usage: .cd DIRECTORY\n"); + rc = 1; + } + }else + + /* The undocumented ".breakpoint" command causes a call to the no-op + ** routine named test_breakpoint(). + */ + if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + test_breakpoint(); + }else + + if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ + if( nArg==2 ){ + setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .changes on|off\n"); + rc = 1; + } + }else + + /* Cancel output redirection, if it is currently set (by .testcase) + ** Then read the content of the testcase-out.txt file and compare against + ** azArg[1]. If there are differences, report an error and exit. + */ + if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + char *zRes = 0; + output_reset(p); + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + rc = 2; + }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ + raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); + rc = 2; + }else if( testcase_glob(azArg[1],zRes)==0 ){ + utf8_printf(stderr, + "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); + rc = 1; + }else{ + utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + p->nCheck++; + } + sqlite3_free(zRes); + }else + + if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ + if( nArg==2 ){ + tryToClone(p, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .clone FILENAME\n"); + rc = 1; + } + }else + + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + char **azName = 0; + int nName = 0; + sqlite3_stmt *pStmt; + int i; + open_db(p, 0); + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + rc = 1; + }else{ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zSchema = (const char *)sqlite3_column_text(pStmt,1); + const char *zFile = (const char*)sqlite3_column_text(pStmt,2); + azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*)); + if( azName==0 ){ shell_out_of_memory(); /* Does not return */ } + azName[nName*2] = strdup(zSchema); + azName[nName*2+1] = strdup(zFile); + nName++; + } + } + sqlite3_finalize(pStmt); + for(i=0; idb, azName[i*2]); + int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); + const char *z = azName[i*2+1]; + utf8_printf(p->out, "%s: %s %s%s\n", + azName[i*2], + z && z[0] ? z : "\"\"", + bRdonly ? "r/o" : "r/w", + eTxn==SQLITE_TXN_NONE ? "" : + eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); + free(azName[i*2]); + free(azName[i*2+1]); + } + sqlite3_free(azName); + }else + + if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + static const struct DbConfigChoices { + const char *zName; + int op; + } aDbConfig[] = { + { "defensive", SQLITE_DBCONFIG_DEFENSIVE }, + { "dqs_ddl", SQLITE_DBCONFIG_DQS_DDL }, + { "dqs_dml", SQLITE_DBCONFIG_DQS_DML }, + { "enable_fkey", SQLITE_DBCONFIG_ENABLE_FKEY }, + { "enable_qpsg", SQLITE_DBCONFIG_ENABLE_QPSG }, + { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, + { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, + { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, + { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, + { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, + { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, + { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, + { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, + }; + int ii, v; + open_db(p, 0); + for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>=3 ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } + sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); + utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + if( nArg>1 ) break; + } + if( nArg>1 && ii==ArraySize(aDbConfig) ){ + utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); + utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); + } + }else + + if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ + rc = shell_dbinfo_command(p, nArg, azArg); + }else + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + open_db(p, 0); + rc = recoverDatabaseCmd(p, nArg, azArg); + }else +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + char *zLike = 0; + char *zSql; + int i; + int savedShowHeader = p->showHeader; + int savedShellFlags = p->shellFlgs; + ShellClearFlag(p, + SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo + |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + for(i=1; ishellFlgs & SHFLG_DumpDataOnly)==0 ){ + /* When playing back a "dump", the content might appear in an order + ** which causes immediate foreign key constraints to be violated. + ** So disable foreign-key constraint enforcement to prevent problems. */ + raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); + raw_printf(p->out, "BEGIN TRANSACTION;\n"); + } + p->writableSchema = 0; + p->showHeader = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_schema table is + ** corrupt. */ + sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); + p->nErr = 0; + if( zLike==0 ) zLike = sqlite3_mprintf("true"); + zSql = sqlite3_mprintf( + "SELECT name, type, sql FROM sqlite_schema " + "WHERE (%s) AND type=='table'" + " AND sql NOT NULL" + " ORDER BY tbl_name='sqlite_sequence', rowid", + zLike + ); + run_schema_dump_query(p,zSql); + sqlite3_free(zSql); + if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ + zSql = sqlite3_mprintf( + "SELECT sql FROM sqlite_schema " + "WHERE (%s) AND sql NOT NULL" + " AND type IN ('index','trigger','view')", + zLike + ); + run_table_dump_query(p, zSql); + sqlite3_free(zSql); + } + sqlite3_free(zLike); + if( p->writableSchema ){ + raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); + p->writableSchema = 0; + } + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); + if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ + raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); + } + p->showHeader = savedShowHeader; + p->shellFlgs = savedShellFlags; + }else + + if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ + if( nArg==2 ){ + setOrClearFlag(p, SHFLG_Echo, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .echo on|off\n"); + rc = 1; + } + }else + + if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ + if( nArg==2 ){ + p->autoEQPtest = 0; + if( p->autoEQPtrace ){ + if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); + p->autoEQPtrace = 0; + } + if( strcmp(azArg[1],"full")==0 ){ + p->autoEQP = AUTOEQP_full; + }else if( strcmp(azArg[1],"trigger")==0 ){ + p->autoEQP = AUTOEQP_trigger; +#ifdef SQLITE_DEBUG + }else if( strcmp(azArg[1],"test")==0 ){ + p->autoEQP = AUTOEQP_on; + p->autoEQPtest = 1; + }else if( strcmp(azArg[1],"trace")==0 ){ + p->autoEQP = AUTOEQP_full; + p->autoEQPtrace = 1; + open_db(p, 0); + sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); +#endif + }else{ + p->autoEQP = (u8)booleanValue(azArg[1]); + } + }else{ + raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n"); + rc = 1; + } + }else + + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); + rc = 2; + }else + + /* The ".explain" command is automatic now. It is largely pointless. It + ** retained purely for backwards compatibility */ + if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + int val = 1; + if( nArg>=2 ){ + if( strcmp(azArg[1],"auto")==0 ){ + val = 99; + }else{ + val = booleanValue(azArg[1]); + } + } + if( val==1 && p->mode!=MODE_Explain ){ + p->normalMode = p->mode; + p->mode = MODE_Explain; + p->autoExplain = 0; + }else if( val==0 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 0; + }else if( val==99 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 1; + } + }else + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){ + open_db(p, 0); + expertDotCommand(p, azArg, nArg); + }else +#endif + + if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ + static const struct { + const char *zCtrlName; /* Name of a test-control option */ + int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ + } aCtrl[] = { + { "chunk_size", SQLITE_FCNTL_CHUNK_SIZE, "SIZE" }, + { "data_version", SQLITE_FCNTL_DATA_VERSION, "" }, + { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, + { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, + { "persist_wal", SQLITE_FCNTL_PERSIST_WAL, "[BOOLEAN]" }, + /* { "pragma", SQLITE_FCNTL_PRAGMA, "NAME ARG" },*/ + { "psow", SQLITE_FCNTL_POWERSAFE_OVERWRITE, "[BOOLEAN]" }, + { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, + { "size_limit", SQLITE_FCNTL_SIZE_LIMIT, "[LIMIT]" }, + { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, + /* { "win32_av_retry", SQLITE_FCNTL_WIN32_AV_RETRY, "COUNT DELAY" },*/ + }; + int filectrl = -1; + int iCtrl = -1; + sqlite3_int64 iRes = 0; /* Integer result to display if rc2==1 */ + int isOk = 0; /* 0: usage 1: %lld 2: no-result */ + int n2, i; + const char *zCmd = 0; + const char *zSchema = 0; + + open_db(p, 0); + zCmd = nArg>=2 ? azArg[1] : "help"; + + if( zCmd[0]=='-' + && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && nArg>=4 + ){ + zSchema = azArg[2]; + for(i=3; iout, "Available file-controls:\n"); + for(i=0; iout, " .filectrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); + } + rc = 1; + goto meta_command_exit; + } + + /* convert filectrl text option to value. allow any unique prefix + ** of the option name, or a numerical value. */ + n2 = strlen30(zCmd); + for(i=0; idb, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes); + isOk = 1; + break; + } + case SQLITE_FCNTL_LOCK_TIMEOUT: + case SQLITE_FCNTL_CHUNK_SIZE: { + int x; + if( nArg!=3 ) break; + x = (int)integerValue(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + isOk = 2; + break; + } + case SQLITE_FCNTL_PERSIST_WAL: + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + int x; + if( nArg!=2 && nArg!=3 ) break; + x = nArg==3 ? booleanValue(azArg[2]) : -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + iRes = x; + isOk = 1; + break; + } + case SQLITE_FCNTL_DATA_VERSION: + case SQLITE_FCNTL_HAS_MOVED: { + int x; + if( nArg!=2 ) break; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + iRes = x; + isOk = 1; + break; + } + case SQLITE_FCNTL_TEMPFILENAME: { + char *z = 0; + if( nArg!=2 ) break; + sqlite3_file_control(p->db, zSchema, filectrl, &z); + if( z ){ + utf8_printf(p->out, "%s\n", z); + sqlite3_free(z); + } + isOk = 2; + break; + } + case SQLITE_FCNTL_RESERVE_BYTES: { + int x; + if( nArg>=3 ){ + x = atoi(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + } + x = -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + utf8_printf(p->out,"%d\n", x); + isOk = 2; + break; + } + } + } + if( isOk==0 && iCtrl>=0 ){ + utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + char zBuf[100]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); + raw_printf(p->out, "%s\n", zBuf); + } + }else + + if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + ShellState data; + char *zErrMsg = 0; + int doStats = 0; + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + if( nArg==2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg = 1; + } + if( nArg!=1 ){ + raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = sqlite3_exec(p->db, + "SELECT sql FROM" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " FROM sqlite_schema UNION ALL" + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " + "ORDER BY rowid", + callback, &data, &zErrMsg + ); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare_v2(p->db, + "SELECT rowid FROM sqlite_schema" + " WHERE name GLOB 'sqlite_stat[134]'", + -1, &pStmt, 0); + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } + if( doStats==0 ){ + raw_printf(p->out, "/* No STAT tables available */\n"); + }else{ + raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_schema'", + callback, &data, &zErrMsg); + data.cMode = data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", &zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", &zErrMsg); + raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + } + }else + + if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){ + if( nArg==2 ){ + p->showHeader = booleanValue(azArg[1]); + p->shellFlgs |= SHFLG_HeaderSet; + }else{ + raw_printf(stderr, "Usage: .headers on|off\n"); + rc = 1; + } + }else + + if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + if( nArg>=2 ){ + n = showHelp(p->out, azArg[1]); + if( n==0 ){ + utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + } + }else{ + showHelp(p->out, 0); + } + }else + + if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ + char *zTable = 0; /* Insert data into this table */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + int nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + int nSep; /* Number of bytes in p->colSeparator[] */ + char *zSql; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + int nSkip = 0; /* Initial lines to skip */ + int useOutputMode = 1; /* Use output mode to determine separators */ + + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + for(i=1; iout, "ERROR: extra argument: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + }else if( strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( strcmp(z,"-skip")==0 && iout, "ERROR: unknown option: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + } + if( zTable==0 ){ + utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + seenInterrupt = 0; + open_db(p, 0); + if( useOutputMode ){ + /* If neither the --csv or --ascii options are specified, then set + ** the column and row separator characters from the output mode. */ + nSep = strlen30(p->colSeparator); + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null column separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep>1 ){ + raw_printf(stderr, + "Error: multi-character column separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } + nSep = strlen30(p->rowSeparator); + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null row separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + /* When importing CSV (only), if the row separator is set to the + ** default output row separator, change it to the default input + ** row separator. This avoids having to maintain different input + ** and output row separators. */ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + nSep = strlen30(p->rowSeparator); + } + if( nSep>1 ){ + raw_printf(stderr, "Error: multi-character row separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } + sCtx.cColSep = p->colSeparator[0]; + sCtx.cRowSep = p->rowSeparator[0]; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + goto meta_command_exit; +#else + sCtx.in = popen(sCtx.zFile+1, "r"); + sCtx.zFile = ""; + sCtx.xCloser = pclose; +#endif + }else{ + sCtx.in = fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + rc = 1; + goto meta_command_exit; + } + if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + utf8_printf(p->out, "Column separator "); + output_c_string(p->out, zSep); + utf8_printf(p->out, ", row separator "); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + utf8_printf(p->out, "\n"); + } + while( (nSkip--)>0 ){ + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + zSql = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + nByte = strlen30(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + char *zCreate = sqlite3_mprintf("CREATE TABLE \"%w\"", zTable); + char cSep = '('; + while( xRead(&sCtx) ){ + zCreate = sqlite3_mprintf("%z%c\n \"%w\" TEXT", zCreate, cSep, sCtx.z); + cSep = ','; + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + if( cSep=='(' ){ + sqlite3_free(zCreate); + import_cleanup(&sCtx); + utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); + rc = 1; + goto meta_command_exit; + } + zCreate = sqlite3_mprintf("%z\n)", zCreate); + if( eVerbose>=1 ){ + utf8_printf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc ){ + utf8_printf(stderr, "CREATE TABLE \"%s\"(...) failed: %s\n", zTable, + sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + nCol = sqlite3_column_count(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + j = strlen30(zSql); + for(i=1; i=2 ){ + utf8_printf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; imode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, + startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); + + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + utf8_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + } + }else + +#ifndef SQLITE_UNTESTABLE + if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + char *zSql; + char *zCollist = 0; + sqlite3_stmt *pStmt; + int tnum = 0; + int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ + int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ + int i; + if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ + utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); + /* Also allowed, but not documented: + ** + ** .imposter TABLE IMPOSTER + ** + ** where TABLE is a WITHOUT ROWID table. In that case, the + ** imposter is another WITHOUT ROWID table with the columns in + ** storage order. */ + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( nArg==2 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + goto meta_command_exit; + } + zSql = sqlite3_mprintf( + "SELECT rootpage, 0 FROM sqlite_schema" + " WHERE name='%q' AND type='index'" + "UNION ALL " + "SELECT rootpage, 1 FROM sqlite_schema" + " WHERE name='%q' AND type='table'" + " AND sql LIKE '%%without%%rowid%%'", + azArg[1], azArg[1] + ); + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + tnum = sqlite3_column_int(pStmt, 0); + isWO = sqlite3_column_int(pStmt, 1); + } + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char zLabel[20]; + const char *zCol = (const char*)sqlite3_column_text(pStmt,2); + i++; + if( zCol==0 ){ + if( sqlite3_column_int(pStmt,1)==-1 ){ + zCol = "_ROWID_"; + }else{ + sqlite3_snprintf(sizeof(zLabel),zLabel,"expr%d",i); + zCol = zLabel; + } + } + if( isWO && lenPK==0 && sqlite3_column_int(pStmt,5)==0 && zCollist ){ + lenPK = (int)strlen(zCollist); + } + if( zCollist==0 ){ + zCollist = sqlite3_mprintf("\"%w\"", zCol); + }else{ + zCollist = sqlite3_mprintf("%z,\"%w\"", zCollist, zCol); + } + } + sqlite3_finalize(pStmt); + if( i==0 || tnum==0 ){ + utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); + rc = 1; + sqlite3_free(zCollist); + goto meta_command_exit; + } + if( lenPK==0 ) lenPK = 100000; + zSql = sqlite3_mprintf( + "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID", + azArg[2], zCollist, lenPK, zCollist); + sqlite3_free(zCollist); + rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + if( rc ){ + utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); + }else{ + utf8_printf(stdout, "%s;\n", zSql); + raw_printf(stdout, + "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n", + azArg[1], isWO ? "table" : "index" + ); + } + }else{ + raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + rc = 1; + } + sqlite3_free(zSql); + }else +#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + +#ifdef SQLITE_ENABLE_IOTRACE + if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); + if( iotrace && iotrace!=stdout ) fclose(iotrace); + iotrace = 0; + if( nArg<2 ){ + sqlite3IoTrace = 0; + }else if( strcmp(azArg[1], "-")==0 ){ + sqlite3IoTrace = iotracePrintf; + iotrace = stdout; + }else{ + iotrace = fopen(azArg[1], "w"); + if( iotrace==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + sqlite3IoTrace = 0; + rc = 1; + }else{ + sqlite3IoTrace = iotracePrintf; + } + } + }else +#endif + + if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + static const struct { + const char *zLimitName; /* Name of a limit */ + int limitCode; /* Integer code for that limit */ + } aLimit[] = { + { "length", SQLITE_LIMIT_LENGTH }, + { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, + { "column", SQLITE_LIMIT_COLUMN }, + { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, + { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, + { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, + { "attached", SQLITE_LIMIT_ATTACHED }, + { "like_pattern_length", SQLITE_LIMIT_LIKE_PATTERN_LENGTH }, + { "variable_number", SQLITE_LIMIT_VARIABLE_NUMBER }, + { "trigger_depth", SQLITE_LIMIT_TRIGGER_DEPTH }, + { "worker_threads", SQLITE_LIMIT_WORKER_THREADS }, + }; + int i, n2; + open_db(p, 0); + if( nArg==1 ){ + for(i=0; idb, aLimit[i].limitCode, -1)); + } + }else if( nArg>3 ){ + raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n"); + rc = 1; + goto meta_command_exit; + }else{ + int iLimit = -1; + n2 = strlen30(azArg[1]); + for(i=0; idb, aLimit[iLimit].limitCode, + (int)integerValue(azArg[2])); + } + printf("%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); + } + }else + + if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + open_db(p, 0); + lintDotCommand(p, azArg, nArg); + }else + +#ifndef SQLITE_OMIT_LOAD_EXTENSION + if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ + const char *zFile, *zProc; + char *zErrMsg = 0; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); + rc = 1; + goto meta_command_exit; + } + zFile = azArg[1]; + zProc = nArg>=3 ? azArg[2] : 0; + open_db(p, 0); + rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + rc = 1; + } + }else +#endif + + if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .log FILENAME\n"); + rc = 1; + }else{ + const char *zFile = azArg[1]; + output_file_close(p->pLog); + p->pLog = output_file_open(zFile, 0); + } + }else + + if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){ + const char *zMode = nArg>=2 ? azArg[1] : ""; + int n2 = strlen30(zMode); + int c2 = zMode[0]; + if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ + p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ + p->mode = MODE_Column; + if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ + p->showHeader = 1; + } + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ + p->mode = MODE_Html; + }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ + p->mode = MODE_Tcl; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); + }else if( c2=='i' && strncmp(azArg[1],"insert",n2)==0 ){ + p->mode = MODE_Insert; + set_table_name(p, nArg>=3 ? azArg[2] : "table"); + }else if( c2=='q' && strncmp(azArg[1],"quote",n2)==0 ){ + p->mode = MODE_Quote; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='a' && strncmp(azArg[1],"ascii",n2)==0 ){ + p->mode = MODE_Ascii; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); + }else if( c2=='m' && strncmp(azArg[1],"markdown",n2)==0 ){ + p->mode = MODE_Markdown; + }else if( c2=='t' && strncmp(azArg[1],"table",n2)==0 ){ + p->mode = MODE_Table; + }else if( c2=='b' && strncmp(azArg[1],"box",n2)==0 ){ + p->mode = MODE_Box; + }else if( c2=='j' && strncmp(azArg[1],"json",n2)==0 ){ + p->mode = MODE_Json; + }else if( nArg==1 ){ + raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]); + }else{ + raw_printf(stderr, "Error: mode should be one of: " + "ascii box column csv html insert json line list markdown " + "quote table tabs tcl\n"); + rc = 1; + } + p->cMode = p->mode; + }else + + if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ + if( nArg==2 ){ + sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, + "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .nullvalue STRING\n"); + rc = 1; + } + }else + +#ifdef SQLITE_DEBUG + if( c=='o' && strcmp(azArg[0],"oom")==0 ){ + int i; + for(i=1; iout, "missing argument on \"%s\"\n", azArg[i]); + rc = 1; + }else{ + oomRepeat = (int)integerValue(azArg[++i]); + } + }else if( IsDigit(z[0]) ){ + oomCounter = (int)integerValue(azArg[i]); + }else{ + raw_printf(p->out, "unknown argument: \"%s\"\n", azArg[i]); + raw_printf(p->out, "Usage: .oom [--repeat N] [M]\n"); + rc = 1; + } + } + if( rc==0 ){ + raw_printf(p->out, "oomCounter = %d\n", oomCounter); + raw_printf(p->out, "oomRepeat = %d\n", oomRepeat); + } + }else +#endif /* SQLITE_DEBUG */ + + if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + char *zNewFilename = 0; /* Name of the database file to open */ + int iName = 1; /* Index in azArg[] of the filename */ + int newFlag = 0; /* True to delete file before opening */ + /* Close the existing database */ + session_close_all(p); + close_db(p->db); + p->db = 0; + p->zDbFilename = 0; + sqlite3_free(p->zFreeOnClose); + p->zFreeOnClose = 0; + p->openMode = SHELL_OPEN_UNSPEC; + p->openFlags = 0; + p->szMax = 0; + /* Check for command-line arguments */ + for(iName=1; iNameopenMode = SHELL_OPEN_ZIPFILE; +#endif + }else if( optionMatch(z, "append") ){ + p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; + }else if( optionMatch(z, "nofollow") ){ + p->openFlags |= SQLITE_OPEN_NOFOLLOW; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( optionMatch(z, "deserialize") ){ + p->openMode = SHELL_OPEN_DESERIALIZE; + }else if( optionMatch(z, "hexdb") ){ + p->openMode = SHELL_OPEN_HEXDB; + }else if( optionMatch(z, "maxsize") && iName+1szMax = integerValue(azArg[++iName]); +#endif /* SQLITE_ENABLE_DESERIALIZE */ + }else if( z[0]=='-' ){ + utf8_printf(stderr, "unknown option: %s\n", z); + rc = 1; + goto meta_command_exit; + }else if( zNewFilename ){ + utf8_printf(stderr, "extra argument: \"%s\"\n", z); + rc = 1; + goto meta_command_exit; + }else{ + zNewFilename = sqlite3_mprintf("%s", z); + } + } + /* If a filename is specified, try to open it first */ + if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){ + if( newFlag ) shellDeleteFile(zNewFilename); + p->zDbFilename = zNewFilename; + open_db(p, OPEN_DB_KEEPALIVE); + if( p->db==0 ){ + utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + sqlite3_free(zNewFilename); + }else{ + p->zFreeOnClose = zNewFilename; + } + } + if( p->db==0 ){ + /* As a fall-back open a TEMP database */ + p->zDbFilename = 0; + open_db(p, 0); + } + }else + + if( (c=='o' + && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + ){ + char *zFile = 0; + int bTxtMode = 0; + int i; + int eMode = 0; + int bBOM = 0; + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ + + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zFile==0 && eMode!='e' && eMode!='x' ){ + zFile = sqlite3_mprintf("%s", z); + if( zFile[0]=='|' ){ + while( i+1out,"ERROR: extra parameter: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + sqlite3_free(zFile); + goto meta_command_exit; + } + } + if( zFile==0 ) zFile = sqlite3_mprintf("stdout"); + if( bOnce ){ + p->outCount = 2; + }else{ + p->outCount = 0; + } + output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' ){ + p->doXdgOpen = 1; + outputModePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + ShellClearFlag(p, SHFLG_Echo); + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + bTxtMode = 1; + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + p->out = stdout; +#else + p->out = popen(zFile + 1, "w"); + if( p->out==0 ){ + utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + p->out = stdout; + rc = 1; + }else{ + if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + p->out = output_file_open(zFile, bTxtMode); + if( p->out==0 ){ + if( strcmp(zFile,"off")!=0 ){ + utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + p->out = stdout; + rc = 1; + } else { + if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); + }else + + if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + open_db(p,0); + if( nArg<=1 ) goto parameter_syntax_error; + + /* .parameter clear + ** Clear all bind parameters by dropping the TEMP table that holds them. + */ + if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", + 0, 0, 0); + }else + + /* .parameter list + ** List all bind parameters. + */ + if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + sqlite3_stmt *pStmt = 0; + int rx; + int len = 0; + rx = sqlite3_prepare_v2(p->db, + "SELECT max(length(key)) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + if( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + len = sqlite3_column_int(pStmt, 0); + if( len>40 ) len = 40; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( len ){ + rx = sqlite3_prepare_v2(p->db, + "SELECT key, quote(value) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), + sqlite3_column_text(pStmt,1)); + } + sqlite3_finalize(pStmt); + } + }else + + /* .parameter init + ** Make sure the TEMP table used to hold bind parameters exists. + ** Create it if necessary. + */ + if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + bind_table_init(p); + }else + + /* .parameter set NAME VALUE + ** Set or reset a bind parameter. NAME should be the full parameter + ** name exactly as it appears in the query. (ex: $abc, @def). The + ** VALUE can be in either SQL literal notation, or if not it will be + ** understood to be a text string. + */ + if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + int rx; + char *zSql; + sqlite3_stmt *pStmt; + const char *zKey = azArg[2]; + const char *zValue = azArg[3]; + bind_table_init(p); + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%s);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + pStmt = 0; + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%Q);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + pStmt = 0; + rc = 1; + } + } + sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + }else + + /* .parameter unset NAME + ** Remove the NAME binding from the parameter binding table, if it + ** exists. + */ + if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + char *zSql = sqlite3_mprintf( + "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); + if( zSql==0 ) shell_out_of_memory(); + sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + }else + /* If no command name matches, show a syntax error */ + parameter_syntax_error: + showHelp(p->out, "parameter"); + }else + + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + int i; + for(i=1; i1 ) raw_printf(p->out, " "); + utf8_printf(p->out, "%s", azArg[i]); + } + raw_printf(p->out, "\n"); + }else + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + int i; + int nn = 0; + p->flgProgress = 0; + p->mxProgress = 0; + p->nProgress = 0; + for(i=1; iflgProgress |= SHELL_PROGRESS_QUIET; + continue; + } + if( strcmp(z,"reset")==0 ){ + p->flgProgress |= SHELL_PROGRESS_RESET; + continue; + } + if( strcmp(z,"once")==0 ){ + p->flgProgress |= SHELL_PROGRESS_ONCE; + continue; + } + if( strcmp(z,"limit")==0 ){ + if( i+1>=nArg ){ + utf8_printf(stderr, "Error: missing argument on --limit\n"); + rc = 1; + goto meta_command_exit; + }else{ + p->mxProgress = (int)integerValue(azArg[++i]); + } + continue; + } + utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]); + rc = 1; + goto meta_command_exit; + }else{ + nn = (int)integerValue(z); + } + } + open_db(p, 0); + sqlite3_progress_handler(p->db, nn, progress_handler, p); + }else +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ + if( nArg >= 2) { + strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); + } + if( nArg >= 3) { + strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1); + } + }else + + if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ + rc = 2; + }else + + if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ + FILE *inSaved = p->in; + int savedLineno = p->lineno; + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .read FILE\n"); + rc = 1; + goto meta_command_exit; + } + if( azArg[1][0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + p->out = stdout; +#else + p->in = popen(azArg[1]+1, "r"); + if( p->in==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + rc = 1; + }else{ + rc = process_input(p); + pclose(p->in); + } +#endif + }else if( notNormalFile(azArg[1]) || (p->in = fopen(azArg[1], "rb"))==0 ){ + utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + rc = 1; + }else{ + rc = process_input(p); + fclose(p->in); + } + p->in = inSaved; + p->lineno = savedLineno; + }else + + if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ + const char *zSrcFile; + const char *zDb; + sqlite3 *pSrc; + sqlite3_backup *pBackup; + int nTimeout = 0; + + if( nArg==2 ){ + zSrcFile = azArg[1]; + zDb = "main"; + }else if( nArg==3 ){ + zSrcFile = azArg[2]; + zDb = azArg[1]; + }else{ + raw_printf(stderr, "Usage: .restore ?DB? FILE\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_open(zSrcFile, &pSrc); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile); + close_db(pSrc); + return 1; + } + open_db(p, 0); + pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); + if( pBackup==0 ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + close_db(pSrc); + return 1; + } + while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK + || rc==SQLITE_BUSY ){ + if( rc==SQLITE_BUSY ){ + if( nTimeout++ >= 3 ) break; + sqlite3_sleep(100); + } + } + sqlite3_backup_finish(pBackup); + if( rc==SQLITE_DONE ){ + rc = 0; + }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){ + raw_printf(stderr, "Error: source database is busy\n"); + rc = 1; + }else{ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + rc = 1; + } + close_db(pSrc); + }else + + if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( nArg==2 ){ + p->scanstatsOn = (u8)booleanValue(azArg[1]); +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); +#endif + }else{ + raw_printf(stderr, "Usage: .scanstats on|off\n"); + rc = 1; + } + }else + + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + ShellText sSelect; + ShellState data; + char *zErrMsg = 0; + const char *zDiv = "("; + const char *zName = 0; + int iSchema = 0; + int bDebug = 0; + int bNoSystemTabs = 0; + int ii; + + open_db(p, 0); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + initText(&sSelect); + for(ii=1; iidb, "SELECT name FROM pragma_database_list", + -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + rc = 1; + goto meta_command_exit; + } + appendText(&sSelect, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + appendText(&sSelect, zDiv, 0); + zDiv = " UNION ALL "; + appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); + if( sqlite3_stricmp(zDb, "main")!=0 ){ + appendText(&sSelect, zDb, '\''); + }else{ + appendText(&sSelect, "NULL", 0); + } + appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); + appendText(&sSelect, zScNum, 0); + appendText(&sSelect, " AS snum, ", 0); + appendText(&sSelect, zDb, '\''); + appendText(&sSelect, " AS sname FROM ", 0); + appendText(&sSelect, zDb, quoteChar(zDb)); + appendText(&sSelect, ".sqlite_schema", 0); + } + sqlite3_finalize(pStmt); +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + if( zName ){ + appendText(&sSelect, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", + 0); + } +#endif + appendText(&sSelect, ") WHERE ", 0); + if( zName ){ + char *zQarg = sqlite3_mprintf("%Q", zName); + int bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); + }else{ + appendText(&sSelect, "lower(tbl_name)", 0); + } + appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); + appendText(&sSelect, zQarg, 0); + if( !bGlob ){ + appendText(&sSelect, " ESCAPE '\\' ", 0); + } + appendText(&sSelect, " AND ", 0); + sqlite3_free(zQarg); + } + if( bNoSystemTabs ){ + appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0); + } + appendText(&sSelect, "sql IS NOT NULL" + " ORDER BY snum, rowid", 0); + if( bDebug ){ + utf8_printf(p->out, "SQL: %s;\n", sSelect.z); + }else{ + rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg); + } + freeText(&sSelect); + } + if( zErrMsg ){ + utf8_printf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + rc = 1; + }else if( rc != SQLITE_OK ){ + raw_printf(stderr,"Error: querying schema information\n"); + rc = 1; + }else{ + rc = 0; + } + }else + + if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ + unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); + }else + +#if defined(SQLITE_ENABLE_SESSION) + if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + OpenSession *pSession = &p->aSession[0]; + char **azCmd = &azArg[1]; + int iSes = 0; + int nCmd = nArg - 1; + int i; + if( nArg<=1 ) goto session_syntax_error; + open_db(p, 0); + if( nArg>=3 ){ + for(iSes=0; iSesnSession; iSes++){ + if( strcmp(p->aSession[iSes].zName, azArg[1])==0 ) break; + } + if( iSesnSession ){ + pSession = &p->aSession[iSes]; + azCmd++; + nCmd--; + }else{ + pSession = &p->aSession[0]; + iSes = 0; + } + } + + /* .session attach TABLE + ** Invoke the sqlite3session_attach() interface to attach a particular + ** table so that it is never filtered. + */ + if( strcmp(azCmd[0],"attach")==0 ){ + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ){ + session_not_open: + raw_printf(stderr, "ERROR: No sessions are open\n"); + }else{ + rc = sqlite3session_attach(pSession->p, azCmd[1]); + if( rc ){ + raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + rc = 0; + } + } + }else + + /* .session changeset FILE + ** .session patchset FILE + ** Write a changeset or patchset into a file. The file is overwritten. + */ + if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + FILE *out = 0; + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ) goto session_not_open; + out = fopen(azCmd[1], "wb"); + if( out==0 ){ + utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", + azCmd[1]); + }else{ + int szChng; + void *pChng; + if( azCmd[0][0]=='c' ){ + rc = sqlite3session_changeset(pSession->p, &szChng, &pChng); + }else{ + rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); + } + if( rc ){ + printf("Error: error code %d\n", rc); + rc = 0; + } + if( pChng + && fwrite(pChng, szChng, 1, out)!=1 ){ + raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", + szChng); + } + sqlite3_free(pChng); + fclose(out); + } + }else + + /* .session close + ** Close the identified session + */ + if( strcmp(azCmd[0], "close")==0 ){ + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + session_close(pSession); + p->aSession[iSes] = p->aSession[--p->nSession]; + } + }else + + /* .session enable ?BOOLEAN? + ** Query or set the enable flag + */ + if( strcmp(azCmd[0], "enable")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_enable(pSession->p, ii); + utf8_printf(p->out, "session %s enable flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session filter GLOB .... + ** Set a list of GLOB patterns of table names to be excluded. + */ + if( strcmp(azCmd[0], "filter")==0 ){ + int ii, nByte; + if( nCmd<2 ) goto session_syntax_error; + if( p->nSession ){ + for(ii=0; iinFilter; ii++){ + sqlite3_free(pSession->azFilter[ii]); + } + sqlite3_free(pSession->azFilter); + nByte = sizeof(pSession->azFilter[0])*(nCmd-1); + pSession->azFilter = sqlite3_malloc( nByte ); + if( pSession->azFilter==0 ){ + raw_printf(stderr, "Error: out or memory\n"); + exit(1); + } + for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + } + pSession->nFilter = ii-1; + } + }else + + /* .session indirect ?BOOLEAN? + ** Query or set the indirect flag + */ + if( strcmp(azCmd[0], "indirect")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_indirect(pSession->p, ii); + utf8_printf(p->out, "session %s indirect flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session isempty + ** Determine if the session is empty + */ + if( strcmp(azCmd[0], "isempty")==0 ){ + int ii; + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + ii = sqlite3session_isempty(pSession->p); + utf8_printf(p->out, "session %s isempty flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session list + ** List all currently open sessions + */ + if( strcmp(azCmd[0],"list")==0 ){ + for(i=0; inSession; i++){ + utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName); + } + }else + + /* .session open DB NAME + ** Open a new session called NAME on the attached database DB. + ** DB is normally "main". + */ + if( strcmp(azCmd[0],"open")==0 ){ + char *zName; + if( nCmd!=3 ) goto session_syntax_error; + zName = azCmd[2]; + if( zName[0]==0 ) goto session_syntax_error; + for(i=0; inSession; i++){ + if( strcmp(p->aSession[i].zName,zName)==0 ){ + utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + goto meta_command_exit; + } + } + if( p->nSession>=ArraySize(p->aSession) ){ + raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession)); + goto meta_command_exit; + } + pSession = &p->aSession[p->nSession]; + rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); + if( rc ){ + raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + rc = 0; + goto meta_command_exit; + } + pSession->nFilter = 0; + sqlite3session_table_filter(pSession->p, session_filter, pSession); + p->nSession++; + pSession->zName = sqlite3_mprintf("%s", zName); + }else + /* If no command name matches, show a syntax error */ + session_syntax_error: + showHelp(p->out, "session"); + }else +#endif + +#ifdef SQLITE_DEBUG + /* Undocumented commands for internal testing. Subject to change + ** without notice. */ + if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ + if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + int i, v; + for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); + } + } + if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + int i; sqlite3_int64 v; + for(i=1; iout, "%s", zBuf); + } + } + }else +#endif + + if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + int bIsInit = 0; /* True to initialize the SELFTEST table */ + int bVerbose = 0; /* Verbose output */ + int bSelftestExists; /* True if SELFTEST already exists */ + int i, k; /* Loop counters */ + int nTest = 0; /* Number of tests runs */ + int nErr = 0; /* Number of errors seen */ + ShellText str; /* Answer for a query */ + sqlite3_stmt *pStmt = 0; /* Query against the SELFTEST table */ + + open_db(p,0); + for(i=1; idb,"main","selftest",0,0,0,0,0,0) + != SQLITE_OK ){ + bSelftestExists = 0; + }else{ + bSelftestExists = 1; + } + if( bIsInit ){ + createSelftestTable(p); + bSelftestExists = 1; + } + initText(&str); + appendText(&str, "x", 0); + for(k=bSelftestExists; k>=0; k--){ + if( k==1 ){ + rc = sqlite3_prepare_v2(p->db, + "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno", + -1, &pStmt, 0); + }else{ + rc = sqlite3_prepare_v2(p->db, + "VALUES(0,'memo','Missing SELFTEST table - default checks only','')," + " (1,'run','PRAGMA integrity_check','ok')", + -1, &pStmt, 0); + } + if( rc ){ + raw_printf(stderr, "Error querying the selftest table\n"); + rc = 1; + sqlite3_finalize(pStmt); + goto meta_command_exit; + } + for(i=1; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int tno = sqlite3_column_int(pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(pStmt, 1); + const char *zSql = (const char*)sqlite3_column_text(pStmt, 2); + const char *zAns = (const char*)sqlite3_column_text(pStmt, 3); + + k = 0; + if( bVerbose>0 ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + utf8_printf(p->out, "%s\n", zSql); + }else + if( strcmp(zOp,"run")==0 ){ + char *zErrMsg = 0; + str.n = 0; + str.z[0] = 0; + rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); + nTest++; + if( bVerbose ){ + utf8_printf(p->out, "Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + } + }else + { + utf8_printf(stderr, + "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + rc = 1; + break; + } + } /* End loop over rows of content from SELFTEST */ + sqlite3_finalize(pStmt); + } /* End loop over k */ + freeText(&str); + utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + }else + + if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ + if( nArg<2 || nArg>3 ){ + raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); + rc = 1; + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, + "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); + } + if( nArg>=3 ){ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, + "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); + } + }else + + if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + const char *zLike = 0; /* Which table to checksum. 0 means everything */ + int i; /* Loop counter */ + int bSchema = 0; /* Also hash the schema */ + int bSeparate = 0; /* Hash each table separately */ + int iSize = 224; /* Hash algorithm to use */ + int bDebug = 0; /* Only show the query that would have run */ + sqlite3_stmt *pStmt; /* For querying tables names */ + char *zSql; /* SQL to be run */ + char *zSep; /* Separator */ + ShellText sSql; /* Complete SQL for the query to run the hash */ + ShellText sQuery; /* Set of queries used to read all content */ + open_db(p, 0); + for(i=1; iout, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zLike ){ + raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); + rc = 1; + goto meta_command_exit; + }else{ + zLike = z; + bSeparate = 1; + if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1; + } + } + if( bSchema ){ + zSql = "SELECT lower(name) FROM sqlite_schema" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " UNION ALL SELECT 'sqlite_schema'" + " ORDER BY 1 collate nocase"; + }else{ + zSql = "SELECT lower(name) FROM sqlite_schema" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " AND name NOT LIKE 'sqlite_%'" + " ORDER BY 1 collate nocase"; + } + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + initText(&sQuery); + initText(&sSql); + appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); + zSep = "VALUES("; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; + if( strncmp(zTab, "sqlite_",7)!=0 ){ + appendText(&sQuery,"SELECT * FROM ", 0); + appendText(&sQuery,zTab,'"'); + appendText(&sQuery," NOT INDEXED;", 0); + }else if( strcmp(zTab, "sqlite_schema")==0 ){ + appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); + }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); + } + appendText(&sSql, zSep, 0); + appendText(&sSql, sQuery.z, '\''); + sQuery.n = 0; + appendText(&sSql, ",", 0); + appendText(&sSql, zTab, '\''); + zSep = "),("; + } + sqlite3_finalize(pStmt); + if( bSeparate ){ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" + " FROM [sha3sum$query]", + sSql.z, iSize); + }else{ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" + " FROM [sha3sum$query]", + sSql.z, iSize); + } + freeText(&sQuery); + freeText(&sSql); + if( bDebug ){ + utf8_printf(p->out, "%s\n", zSql); + }else{ + shell_exec(p, zSql, 0); + } + sqlite3_free(zSql); + }else + +#ifndef SQLITE_NOHAVE_SYSTEM + if( c=='s' + && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) + ){ + char *zCmd; + int i, x; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .system COMMAND\n"); + rc = 1; + goto meta_command_exit; + } + zCmd = sqlite3_mprintf(strchr(azArg[1],' ')==0?"%s":"\"%s\"", azArg[1]); + for(i=2; iout, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); + utf8_printf(p->out, "%12.12s: %s\n","explain", + p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); + utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); + utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); + utf8_printf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->nullValue); + raw_printf(p->out, "\n"); + utf8_printf(p->out,"%12.12s: %s\n","output", + strlen30(p->outfile) ? p->outfile : "stdout"); + utf8_printf(p->out,"%12.12s: ", "colseparator"); + output_c_string(p->out, p->colSeparator); + raw_printf(p->out, "\n"); + utf8_printf(p->out,"%12.12s: ", "rowseparator"); + output_c_string(p->out, p->rowSeparator); + raw_printf(p->out, "\n"); + switch( p->statsOn ){ + case 0: zOut = "off"; break; + default: zOut = "on"; break; + case 2: zOut = "stmt"; break; + case 3: zOut = "vmstep"; break; + } + utf8_printf(p->out, "%12.12s: %s\n","stats", zOut); + utf8_printf(p->out, "%12.12s: ", "width"); + for (i=0;inWidth;i++) { + raw_printf(p->out, "%d ", p->colWidth[i]); + } + raw_printf(p->out, "\n"); + utf8_printf(p->out, "%12.12s: %s\n", "filename", + p->zDbFilename ? p->zDbFilename : ""); + }else + + if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ + if( nArg==2 ){ + if( strcmp(azArg[1],"stmt")==0 ){ + p->statsOn = 2; + }else if( strcmp(azArg[1],"vmstep")==0 ){ + p->statsOn = 3; + }else{ + p->statsOn = (u8)booleanValue(azArg[1]); + } + }else if( nArg==1 ){ + display_stats(p->db, p, 0); + }else{ + raw_printf(stderr, "Usage: .stats ?on|off|stmt|vmstep?\n"); + rc = 1; + } + }else + + if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (strncmp(azArg[0], "indices", n)==0 + || strncmp(azArg[0], "indexes", n)==0) ) + ){ + sqlite3_stmt *pStmt; + char **azResult; + int nRow, nAlloc; + int ii; + ShellText s; + initText(&s); + open_db(p, 0); + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return shellDatabaseError(p->db); + } + + if( nArg>2 && c=='i' ){ + /* It is an historical accident that the .indexes command shows an error + ** when called with the wrong number of arguments whereas the .tables + ** command does not. */ + raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); + rc = 1; + sqlite3_finalize(pStmt); + goto meta_command_exit; + } + for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ + const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); + if( zDbName==0 ) continue; + if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0); + if( sqlite3_stricmp(zDbName, "main")==0 ){ + appendText(&s, "SELECT name FROM ", 0); + }else{ + appendText(&s, "SELECT ", 0); + appendText(&s, zDbName, '\''); + appendText(&s, "||'.'||name FROM ", 0); + } + appendText(&s, zDbName, '"'); + appendText(&s, ".sqlite_schema ", 0); + if( c=='t' ){ + appendText(&s," WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%'" + " AND name LIKE ?1", 0); + }else{ + appendText(&s," WHERE type='index'" + " AND tbl_name LIKE ?1", 0); + } + } + rc = sqlite3_finalize(pStmt); + appendText(&s, " ORDER BY 1", 0); + rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0); + freeText(&s); + if( rc ) return shellDatabaseError(p->db); + + /* Run the SQL statement prepared by the above block. Store the results + ** as an array of nul-terminated strings in azResult[]. */ + nRow = nAlloc = 0; + azResult = 0; + if( nArg>1 ){ + sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nRow>=nAlloc ){ + char **azNew; + int n2 = nAlloc*2 + 10; + azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); + if( azNew==0 ) shell_out_of_memory(); + nAlloc = n2; + azResult = azNew; + } + azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + if( 0==azResult[nRow] ) shell_out_of_memory(); + nRow++; + } + if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ + rc = shellDatabaseError(p->db); + } + + /* Pretty-print the contents of array azResult[] to the output */ + if( rc==0 && nRow>0 ){ + int len, maxlen = 0; + int i, j; + int nPrintCol, nPrintRow; + for(i=0; imaxlen ) maxlen = len; + } + nPrintCol = 80/(maxlen+2); + if( nPrintCol<1 ) nPrintCol = 1; + nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; + for(i=0; iout, "%s%-*s", zSp, maxlen, + azResult[j] ? azResult[j]:""); + } + raw_printf(p->out, "\n"); + } + } + + for(ii=0; iiout = output_file_open("testcase-out.txt", 0); + if( p->out==0 ){ + raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); + } + }else + +#ifndef SQLITE_UNTESTABLE + if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ + static const struct { + const char *zCtrlName; /* Name of a test-control option */ + int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ + } aCtrl[] = { + { "always", SQLITE_TESTCTRL_ALWAYS, "BOOLEAN" }, + { "assert", SQLITE_TESTCTRL_ASSERT, "BOOLEAN" }, + /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, "" },*/ + /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, "" },*/ + { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, + { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,"BOOLEAN" }, + /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" },*/ + { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, + { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "" }, + { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, + { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, + { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, +#ifdef YYCOVERAGE + { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE, "" }, +#endif + { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, + { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, + { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, + { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" }, + { "seek_count", SQLITE_TESTCTRL_SEEK_COUNT, "" }, + }; + int testctrl = -1; + int iCtrl = -1; + int rc2 = 0; /* 0: usage. 1: %d 2: %x 3: no-output */ + int isOk = 0; + int i, n2; + const char *zCmd = 0; + + open_db(p, 0); + zCmd = nArg>=2 ? azArg[1] : "help"; + + /* The argument can optionally begin with "-" or "--" */ + if( zCmd[0]=='-' && zCmd[1] ){ + zCmd++; + if( zCmd[0]=='-' && zCmd[1] ) zCmd++; + } + + /* --help lists all test-controls */ + if( strcmp(zCmd,"help")==0 ){ + utf8_printf(p->out, "Available test-controls:\n"); + for(i=0; iout, " .testctrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); + } + rc = 1; + goto meta_command_exit; + } + + /* convert testctrl text option to value. allow any unique prefix + ** of the option name, or a numerical value. */ + n2 = strlen30(zCmd); + for(i=0; idb, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(int) */ + case SQLITE_TESTCTRL_PRNG_SAVE: + case SQLITE_TESTCTRL_PRNG_RESTORE: + case SQLITE_TESTCTRL_BYTEORDER: + if( nArg==2 ){ + rc2 = sqlite3_test_control(testctrl); + isOk = testctrl==SQLITE_TESTCTRL_BYTEORDER ? 1 : 3; + } + break; + + /* sqlite3_test_control(int, uint) */ + case SQLITE_TESTCTRL_PENDING_BYTE: + if( nArg==3 ){ + unsigned int opt = (unsigned int)integerValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(int, int, sqlite3*) */ + case SQLITE_TESTCTRL_PRNG_SEED: + if( nArg==3 || nArg==4 ){ + int ii = (int)integerValue(azArg[2]); + sqlite3 *db; + if( ii==0 && strcmp(azArg[2],"random")==0 ){ + sqlite3_randomness(sizeof(ii),&ii); + printf("-- random seed: %d\n", ii); + } + if( nArg==3 ){ + db = 0; + }else{ + db = p->db; + /* Make sure the schema has been loaded */ + sqlite3_table_column_metadata(db, 0, "x", 0, 0, 0, 0, 0, 0); + } + rc2 = sqlite3_test_control(testctrl, ii, db); + isOk = 3; + } + break; + + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_ASSERT: + case SQLITE_TESTCTRL_ALWAYS: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 1; + } + break; + + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: + case SQLITE_TESTCTRL_NEVER_CORRUPT: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(sqlite3*) */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: + rc2 = sqlite3_test_control(testctrl, p->db); + isOk = 3; + break; + + case SQLITE_TESTCTRL_IMPOSTER: + if( nArg==5 ){ + rc2 = sqlite3_test_control(testctrl, p->db, + azArg[2], + integerValue(azArg[3]), + integerValue(azArg[4])); + isOk = 3; + } + break; + + case SQLITE_TESTCTRL_SEEK_COUNT: { + u64 x = 0; + rc2 = sqlite3_test_control(testctrl, p->db, &x); + utf8_printf(p->out, "%llu\n", x); + isOk = 3; + break; + } + +#ifdef YYCOVERAGE + case SQLITE_TESTCTRL_PARSER_COVERAGE: + if( nArg==2 ){ + sqlite3_test_control(testctrl, p->out); + isOk = 3; + } +#endif + } + } + if( isOk==0 && iCtrl>=0 ){ + utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + raw_printf(p->out, "%d\n", rc2); + }else if( isOk==2 ){ + raw_printf(p->out, "0x%08x\n", rc2); + } + }else +#endif /* !defined(SQLITE_UNTESTABLE) */ + + if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ + open_db(p, 0); + sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); + }else + + if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ + if( nArg==2 ){ + enableTimer = booleanValue(azArg[1]); + if( enableTimer && !HAS_TIMER ){ + raw_printf(stderr, "Error: timer not available on this system.\n"); + enableTimer = 0; + } + }else{ + raw_printf(stderr, "Usage: .timer on|off\n"); + rc = 1; + } + }else + +#ifndef SQLITE_OMIT_TRACE + if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; + open_db(p, 0); + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } + } + if( p->traceOut==0 ){ + sqlite3_trace_v2(p->db, 0, 0, 0); + }else{ + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); + } + }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) + if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ + int ii; + int lenOpt; + char *zOpt; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + zOpt = azArg[1]; + if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; + lenOpt = (int)strlen(zOpt); + if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ + assert( azArg[nArg]==0 ); + sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); + }else{ + for(ii=1; iidb, azArg[ii], 0, 0); + } + } + }else +#endif + +#if SQLITE_USER_AUTHENTICATION + if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( nArg<2 ){ + raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( strcmp(azArg[1],"login")==0 ){ + if( nArg!=4 ){ + raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], + strlen30(azArg[3])); + if( rc ){ + utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); + rc = 1; + } + }else if( strcmp(azArg[1],"add")==0 ){ + if( nArg!=5 ){ + raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + raw_printf(stderr, "User-Add failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"edit")==0 ){ + if( nArg!=5 ){ + raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + raw_printf(stderr, "User-Edit failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"delete")==0 ){ + if( nArg!=3 ){ + raw_printf(stderr, "Usage: .user delete USER\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_delete(p->db, azArg[2]); + if( rc ){ + raw_printf(stderr, "User-Delete failed: %d\n", rc); + rc = 1; + } + }else{ + raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); + rc = 1; + goto meta_command_exit; + } + }else +#endif /* SQLITE_USER_AUTHENTICATION */ + + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + sqlite3_libversion(), sqlite3_sourceid()); +#if SQLITE_HAVE_ZLIB + utf8_printf(p->out, "zlib version %s\n", zlibVersion()); +#endif +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) +#if defined(__clang__) && defined(__clang_major__) + utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__) "\n"); +#elif defined(_MSC_VER) + utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n"); +#elif defined(__GNUC__) && defined(__VERSION__) + utf8_printf(p->out, "gcc-" __VERSION__ "\n"); +#endif + }else + + if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ + const char *zDbName = nArg==2 ? azArg[1] : "main"; + sqlite3_vfs *pVfs = 0; + if( p->db ){ + sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); + if( pVfs ){ + utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + } + } + }else + + if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + sqlite3_vfs *pVfs; + sqlite3_vfs *pCurrent = 0; + if( p->db ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); + } + for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ + utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + if( pVfs->pNext ){ + raw_printf(p->out, "-----------------------------------\n"); + } + } + }else + + if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + const char *zDbName = nArg==2 ? azArg[1] : "main"; + char *zVfsName = 0; + if( p->db ){ + sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); + if( zVfsName ){ + utf8_printf(p->out, "%s\n", zVfsName); + sqlite3_free(zVfsName); + } + } + }else + + if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); + }else + + if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + int j; + assert( nArg<=ArraySize(azArg) ); + p->nWidth = nArg-1; + p->colWidth = realloc(p->colWidth, p->nWidth*sizeof(int)*2); + if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); + if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; + for(j=1; jcolWidth[j-1] = (int)integerValue(azArg[j]); + } + }else + + { + utf8_printf(stderr, "Error: unknown command or invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[0]); + rc = 1; + } + +meta_command_exit: + if( p->outCount ){ + p->outCount--; + if( p->outCount==0 ) output_reset(p); + } + return rc; +} + +/* +** Return TRUE if a semicolon occurs anywhere in the first N characters +** of string z[]. +*/ +static int line_contains_semicolon(const char *z, int N){ + int i; + for(i=0; iflgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; + BEGIN_TIMER; + rc = shell_exec(p, zSql, &zErrMsg); + END_TIMER; + if( rc || zErrMsg ){ + char zPrefix[100]; + if( in!=0 || !stdin_is_interactive ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "Error: near line %d:", startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); + } + if( zErrMsg!=0 ){ + utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg); + sqlite3_free(zErrMsg); + zErrMsg = 0; + }else{ + utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); + } + return 1; + }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ + raw_printf(p->out, "changes: %3d total_changes: %d\n", + sqlite3_changes(p->db), sqlite3_total_changes(p->db)); + } + return 0; +} + + +/* +** Read input from *in and process it. If *in==0 then input +** is interactive - the user is typing it it. Otherwise, input +** is coming from a file or device. A prompt is issued and history +** is saved only if input is interactive. An interrupt signal will +** cause this routine to exit immediately, unless input is interactive. +** +** Return the number of errors. +*/ +static int process_input(ShellState *p){ + char *zLine = 0; /* A single input line */ + char *zSql = 0; /* Accumulated SQL text */ + int nLine; /* Length of current line */ + int nSql = 0; /* Bytes of zSql[] used */ + int nAlloc = 0; /* Allocated zSql[] space */ + int nSqlPrior = 0; /* Bytes of zSql[] used by prior line */ + int rc; /* Error code */ + int errCnt = 0; /* Number of errors seen */ + int startline = 0; /* Line number for start of current input */ + + p->lineno = 0; + while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ + fflush(p->out); + zLine = one_input_line(p->in, zLine, nSql>0); + if( zLine==0 ){ + /* End of input */ + if( p->in==0 && stdin_is_interactive ) printf("\n"); + break; + } + if( seenInterrupt ){ + if( p->in!=0 ) break; + seenInterrupt = 0; + } + p->lineno++; + if( nSql==0 && _all_whitespace(zLine) ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); + continue; + } + if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); + if( zLine[0]=='.' ){ + rc = do_meta_command(zLine, p); + if( rc==2 ){ /* exit requested */ + break; + }else if( rc ){ + errCnt++; + } + } + continue; + } + if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){ + memcpy(zLine,";",2); + } + nLine = strlen30(zLine); + if( nSql+nLine+2>=nAlloc ){ + nAlloc = nSql+nLine+100; + zSql = realloc(zSql, nAlloc); + if( zSql==0 ) shell_out_of_memory(); + } + nSqlPrior = nSql; + if( nSql==0 ){ + int i; + for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} + assert( nAlloc>0 && zSql!=0 ); + memcpy(zSql, zLine+i, nLine+1-i); + startline = p->lineno; + nSql = nLine-i; + }else{ + zSql[nSql++] = '\n'; + memcpy(zSql+nSql, zLine, nLine+1); + nSql += nLine; + } + if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior) + && sqlite3_complete(zSql) ){ + errCnt += runOneSqlLine(p, zSql, p->in, startline); + nSql = 0; + if( p->outCount ){ + output_reset(p); + p->outCount = 0; + }else{ + clearTempFile(p); + } + }else if( nSql && _all_whitespace(zSql) ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); + nSql = 0; + } + } + if( nSql && !_all_whitespace(zSql) ){ + errCnt += runOneSqlLine(p, zSql, p->in, startline); + } + free(zSql); + free(zLine); + return errCnt>0; +} + +/* +** Return a pathname which is the user's home directory. A +** 0 return indicates an error of some kind. +*/ +static char *find_home_dir(int clearFlag){ + static char *home_dir = NULL; + if( clearFlag ){ + free(home_dir); + home_dir = 0; + return 0; + } + if( home_dir ) return home_dir; + +#if 0 +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) + { + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != NULL) { + home_dir = pwent->pw_dir; + } + } +#endif +#endif + +#if defined(_WIN32_WCE) + /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() + */ + home_dir = "/"; +#else + + if (IsWindows() && !home_dir) { + home_dir = getenv("USERPROFILE"); + } + + if (!home_dir) { + home_dir = getenv("HOME"); + } + + if (IsWindows() && !home_dir) { + char *zDrive, *zPath; + int n; + zDrive = getenv("HOMEDRIVE"); + zPath = getenv("HOMEPATH"); + if( zDrive && zPath ){ + n = strlen30(zDrive) + strlen30(zPath) + 1; + home_dir = malloc( n ); + if( home_dir==0 ) return 0; + sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath); + return home_dir; + } + home_dir = "c:\\"; + } + +#endif /* !_WIN32_WCE */ + + if( home_dir ){ + int n = strlen30(home_dir) + 1; + char *z = malloc( n ); + if( z ) memcpy(z, home_dir, n); + home_dir = z; + } + + return home_dir; +} + +/* +** Read input from the file given by sqliterc_override. Or if that +** parameter is NULL, take input from ~/.sqliterc +** +** Returns the number of errors. +*/ +static void process_sqliterc( + ShellState *p, /* Configuration data */ + const char *sqliterc_override /* Name of config file. NULL to use default */ +){ + char *home_dir = NULL; + const char *sqliterc = sqliterc_override; + char *zBuf = 0; + FILE *inSaved = p->in; + int savedLineno = p->lineno; + + if (sqliterc == NULL) { + home_dir = find_home_dir(0); + if( home_dir==0 ){ + raw_printf(stderr, "-- warning: cannot find home directory;" + " cannot read ~/.sqliterc\n"); + return; + } + zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); + sqliterc = zBuf; + } + p->in = fopen(sqliterc,"rb"); + if( p->in ){ + if( stdin_is_interactive ){ + utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc); + } + if( process_input(p) && bail_on_error ) exit(1); + fclose(p->in); + }else if( sqliterc_override!=0 ){ + utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) exit(1); + } + p->in = inSaved; + p->lineno = savedLineno; + sqlite3_free(zBuf); +} + +/* +** Show available command line options +*/ +static const char zOptions[] = +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + " -A ARGS... run \".archive ARGS\" and exit\n" +#endif + " -append append the database to the end of the file\n" + " -ascii set output mode to 'ascii'\n" + " -bail stop after hitting an error\n" + " -batch force batch I/O\n" + " -box set output mode to 'box'\n" + " -column set output mode to 'column'\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" + " -csv set output mode to 'csv'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -deserialize open the database using sqlite3_deserialize()\n" +#endif + " -echo print commands before execution\n" + " -init FILENAME read/process named file\n" + " -[no]header turn headers on or off\n" +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + " -heap SIZE Size of heap for memsys3 or memsys5\n" +#endif + " -help show this message\n" + " -html set output mode to HTML\n" + " -interactive force interactive I/O\n" + " -json set output mode to 'json'\n" + " -line set output mode to 'line'\n" + " -list set output mode to 'list'\n" + " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" + " -markdown set output mode to 'markdown'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -maxsize N maximum size for a --deserialize database\n" +#endif + " -memtrace trace all memory allocations and deallocations\n" + " -mmap N default mmap size set to N\n" +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif + " -newline SEP set output row separator. Default: '\\n'\n" + " -nofollow refuse to open symbolic links to database files\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -quote set output mode to 'quote'\n" + " -readonly open the database read-only\n" + " -separator SEP set output column separator. Default: '|'\n" +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + " -sorterref SIZE sorter references threshold size\n" +#endif + " -stats print memory stats before each finalize\n" + " -table set output mode to 'table'\n" + " -tabs set output mode to 'tabs'\n" + " -version show SQLite version\n" + " -vfs NAME use NAME as the default VFS\n" +#ifdef SQLITE_ENABLE_VFSTRACE + " -vfstrace enable tracing of all VFS calls\n" +#endif +#ifdef SQLITE_HAVE_ZLIB + " -zip open the file as a ZIP Archive\n" +#endif +; +static void usage(int showDetail){ + utf8_printf(stderr, + "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "FILENAME is the name of an SQLite database. A new database is created\n" + "if the file does not previously exist.\n", Argv0); + if( showDetail ){ + utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); + }else{ + raw_printf(stderr, "Use the -help option for additional information\n"); + } + exit(1); +} + +/* +** Internal check: Verify that the SQLite is uninitialized. Print a +** error message if it is initialized. +*/ +static void verify_uninitialized(void){ + if( sqlite3_config(-1)==SQLITE_MISUSE ){ + utf8_printf(stdout, "WARNING: attempt to configure SQLite after" + " initialization.\n"); + } +} + +/* +** Initialize the state information in data +*/ +static void main_init(ShellState *data) { + memset(data, 0, sizeof(*data)); + data->normalMode = data->cMode = data->mode = MODE_List; + data->autoExplain = 1; + memcpy(data->colSeparator,SEP_Column, 2); + memcpy(data->rowSeparator,SEP_Row, 2); + data->showHeader = 0; + data->shellFlgs = SHFLG_Lookaside; + verify_uninitialized(); + sqlite3_config(SQLITE_CONFIG_URI, 1); + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); + sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); +} + +/* +** Output text to the console in a font that attracts extra attention. +*/ +#ifdef _WIN32 +static void printBold(const char *zText){ +#if !SQLITE_OS_WINRT + HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; + GetConsoleScreenBufferInfo(out, &defaultScreenInfo); + SetConsoleTextAttribute(out, + FOREGROUND_RED|FOREGROUND_INTENSITY + ); +#endif + printf("%s", zText); +#if !SQLITE_OS_WINRT + SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); +#endif +} +#else +static void printBold(const char *zText){ + printf("\033[1m%s\033[0m", zText); +} +#endif + +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static char *cmdline_option_value(int argc, char **argv, int i){ + if( i==argc ){ + utf8_printf(stderr, "%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + +#ifndef SQLITE_SHELL_IS_UTF8 +# if (defined(_WIN32) || defined(WIN32)) \ + && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) +# define SQLITE_SHELL_IS_UTF8 (0) +# else +# define SQLITE_SHELL_IS_UTF8 (1) +# endif +#endif + +#if SQLITE_SHELL_IS_UTF8 +int SQLITE_CDECL main(int argc, char **argv){ +#else +int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +#endif + char *zErrMsg = 0; + ShellState data; + const char *zInitFile = 0; + int i; + int rc = 0; + int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; + const char *zVfs = 0; /* Value of -vfs command-line option */ +#if !SQLITE_SHELL_IS_UTF8 + char **argvToFree = 0; + int argcToFree = 0; +#endif + + setBinaryMode(stdin, 0); + setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + setvbuf(stdin, (char *)NULL, _IONBF, BUFSIZ); + setvbuf(stdout, (char *)NULL, _IONBF, BUFSIZ); + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); + +#ifdef SQLITE_DEBUG + registerOomSimulator(); +#endif + +#if !defined(_WIN32_WCE) + if( getenv("SQLITE_DEBUG_BREAK") ){ + if( isatty(0) && isatty(2) ){ + fprintf(stderr, + "attach debugger to process %d and press any key to continue.\n", + GETPID()); + fgetc(stdin); + }else{ +#if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT + __debugbreak(); +#else + DebugBreak(); +#endif +#elif defined(SIGTRAP) + raise(SIGTRAP); +#endif + } + } +#endif + +#if USE_SYSTEM_SQLITE+0!=1 + if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ + utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); + exit(1); + } +#endif + main_init(&data); + + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ +#if !SQLITE_SHELL_IS_UTF8 + sqlite3_initialize(); + argvToFree = malloc(sizeof(argv[0])*argc*2); + argcToFree = argc; + argv = argvToFree + argc; + if( argv==0 ) shell_out_of_memory(); + for(i=0; i=1 && argv && argv[0] ); + Argv0 = argv[0]; + + /* Make sure we have a valid signal handler early, before anything + ** else is done. + */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); +#endif + +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif + + /* Do an initial pass through the command-line argument to locate + ** the name of the database file, the name of the initialization file, + ** the size of the alternative malloc heap, + ** and the first command to execute. + */ + verify_uninitialized(); + for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; + sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); +#else + (void)cmdline_option_value(argc, argv, ++i); +#endif + }else if( strcmp(z,"-pagecache")==0 ){ + sqlite3_int64 n, sz; + sz = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>70000 ) sz = 70000; + if( sz<0 ) sz = 0; + n = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>0 && n>0 && 0xffffffffffffLL/sz0 && sz>0) ? malloc(n*sz) : 0, sz, n); + data.shellFlgs |= SHFLG_Pagecache; + }else if( strcmp(z,"-lookaside")==0 ){ + int n, sz; + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( sz<0 ) sz = 0; + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( n<0 ) n = 0; + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); + if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; +#ifdef SQLITE_ENABLE_VFSTRACE + }else if( strcmp(z,"-vfstrace")==0 ){ + extern int vfstrace_register( + const char *zTraceName, + const char *zOldVfsName, + int (*xOut)(const char*,void*), + void *pOutArg, + int makeDefault + ); + vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(z,"-multiplex")==0 ){ + extern int sqlite3_multiple_initialize(const char*,int); + sqlite3_multiplex_initialize(0, 1); +#endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); +#endif + }else if( strcmp(z,"-vfs")==0 ){ + zVfs = cmdline_option_value(argc, argv, ++i); +#ifdef SQLITE_HAVE_ZLIB + }else if( strcmp(z,"-zip")==0 ){ + data.openMode = SHELL_OPEN_ZIPFILE; +#endif + }else if( strcmp(z,"-append")==0 ){ + data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; + }else if( strcmp(z,"-maxsize")==0 && i+10 ){ + utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" + " with \"%s\"\n", z); + return 1; + } + open_db(&data, OPEN_DB_ZIPFILE); + if( z[2] ){ + argv[i] = &z[2]; + arDotCommand(&data, 1, argv+(i-1), argc-(i-1)); + }else{ + arDotCommand(&data, 1, argv+i, argc-i); + } + readStdin = 0; + break; +#endif + }else{ + utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + raw_printf(stderr,"Use -help for a list of options.\n"); + return 1; + } + data.cMode = data.mode; + } + + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. + */ + for(i=0; i +#include +#include +#include +#include "sqlite3.h" +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; +typedef unsigned char u8; +#if SQLITE_USER_AUTHENTICATION +# include "sqlite3userauth.inc" +#endif +#include +#include + +#if !defined(_WIN32) && !defined(WIN32) +# include +# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# include +# endif +#endif +#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__) +# include +# include +# define GETPID getpid +# if defined(__MINGW32__) +# define DIRENT dirent +# ifndef S_ISLNK +# define S_ISLNK(mode) (0) +# endif +# endif +#else +# define GETPID (int)GetCurrentProcessId +#endif +#include +#include + +#if HAVE_READLINE +# include +# include +#endif + +#if HAVE_EDITLINE +# include +#endif + +#if HAVE_EDITLINE || HAVE_READLINE + +# define shell_add_history(X) add_history(X) +# define shell_read_history(X) read_history(X) +# define shell_write_history(X) write_history(X) +# define shell_stifle_history(X) stifle_history(X) +# define shell_readline(X) readline(X) + +#elif HAVE_LINENOISE + +# include "linenoise.inc" +# define shell_add_history(X) linenoiseHistoryAdd(X) +# define shell_read_history(X) linenoiseHistoryLoad(X) +# define shell_write_history(X) linenoiseHistorySave(X) +# define shell_stifle_history(X) linenoiseHistorySetMaxLen(X) +# define shell_readline(X) linenoise(X) + +#else + +# define shell_read_history(X) +# define shell_write_history(X) +# define shell_stifle_history(X) + +# define SHELL_USE_LOCAL_GETLINE 1 +#endif + + +#if defined(_WIN32) || defined(WIN32) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef popen +# define popen _popen +# undef pclose +# define pclose _pclose +# endif +#else + /* Make sure isatty() has a prototype. */ + extern int isatty(int); + +# if !defined(__RTP__) && !defined(_WRS_KERNEL) + /* popen and pclose are not C89 functions and so are + ** sometimes omitted from the header */ + extern FILE *popen(const char*,const char*); + extern int pclose(FILE*); +# else +# define SQLITE_OMIT_POPEN 1 +# endif +#endif + +#if defined(_WIN32_WCE) +/* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() + * thus we always assume that we have a console. That can be + * overridden with the -batch command line option. + */ +#define isatty(x) 1 +#endif + +/* ctype macros that work with signed characters */ +#define IsSpace(X) isspace((unsigned char)X) +#define IsDigit(X) isdigit((unsigned char)X) +#define ToLower(X) (char)tolower((unsigned char)X) + +#if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include +#endif +#include + +/* string conversion routines only needed on Win32 */ +extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); +extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); +extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); +extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); +#endif + +/* On Windows, we normally run with output mode of TEXT so that \n characters +** are automatically translated into \r\n. However, this behavior needs +** to be disabled in some cases (ex: when generating CSV output and when +** rendering quoted strings that contain \n characters). The following +** routines take care of that. +*/ +#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT +static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); +} +static void setTextMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_TEXT); +} +#else +# define setBinaryMode(X,Y) +# define setTextMode(X,Y) +#endif + + +/* True if the timer is enabled */ +static int enableTimer = 0; + +/* Return the current wall-clock time */ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include +#include + +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. +*/ +static void endTimer(void){ + if( enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + printf("Run Time: real %.3f user %f sys %f\n", + (iEnd - iBegin)*0.001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { +#if !SQLITE_OS_WINRT + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } +#endif + } + return 0; +} + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer && getProcessTimesAddr ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +/* +** Print the timing results. +*/ +static void endTimer(void){ + if( enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + printf("Run Time: real %.3f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER +#define END_TIMER +#define HAS_TIMER 0 +#endif + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* +** If the following flag is set, then command execution stops +** at an error if we are not interactive. +*/ +static int bail_on_error = 0; + +/* +** Threat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; + +/* +** On Windows systems we have to know if standard output is a console +** in order to translate UTF-8 into MBCS. The following variable is +** true if translation is required. +*/ +static int stdout_is_console = 1; + +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *globalDb = 0; + +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; + +#ifdef SQLITE_DEBUG +/* +** Out-of-memory simulator variables +*/ +static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ +static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ +static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ +#endif /* SQLITE_DEBUG */ + +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; + +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ +static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ + +/* +** Render output like fprintf(). Except, if the output is going to the +** console and if this is running on a Windows machine, translate the +** output from UTF-8 into MBCS. +*/ +#if defined(_WIN32) || defined(WIN32) +void utf8_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + if( stdout_is_console && (out==stdout || out==stderr) ){ + char *z1 = sqlite3_vmprintf(zFormat, ap); + char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); + sqlite3_free(z1); + fputs(z2, out); + sqlite3_free(z2); + }else{ + vfprintf(out, zFormat, ap); + } + va_end(ap); +} +#elif !defined(utf8_printf) +# define utf8_printf fprintf +#endif + +/* +** Render output like fprintf(). This should not be used on anything that +** includes string formatting (e.g. "%s"). +*/ +#if !defined(raw_printf) +# define raw_printf fprintf +#endif + +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + raw_printf(stderr,"Error: out of memory\n"); + exit(1); +} + +#ifdef SQLITE_DEBUG +/* This routine is called when a simulated OOM occurs. It is broken +** out as a separate routine to make it easy to set a breakpoint on +** the OOM +*/ +void shellOomFault(void){ + if( oomRepeat>0 ){ + oomRepeat--; + }else{ + oomCounter--; + } +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* This routine is a replacement malloc() that is used to simulate +** Out-Of-Memory (OOM) errors for testing purposes. +*/ +static void *oomMalloc(int nByte){ + if( oomCounter ){ + if( oomCounter==1 ){ + shellOomFault(); + return 0; + }else{ + oomCounter--; + } + } + return defaultMalloc(nByte); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* Register the OOM simulator. This must occur before any memory +** allocations */ +static void registerOomSimulator(void){ + sqlite3_mem_methods mem; + sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem); + defaultMalloc = mem.xMalloc; + mem.xMalloc = oomMalloc; + sqlite3_config(SQLITE_CONFIG_MALLOC, &mem); +} +#endif + +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + utf8_printf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif + +/* +** Output string zUtf to stream pOut as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. +*/ +static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ + int i; + int n; + int aw = w<0 ? -w : w; + for(i=n=0; zUtf[i]; i++){ + if( (zUtf[i]&0xc0)!=0x80 ){ + n++; + if( n==aw ){ + do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + break; + } + } + } + if( n>=aw ){ + utf8_printf(pOut, "%.*s", i, zUtf); + }else if( w<0 ){ + utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + }else{ + utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + } +} + + +/* +** Determines if a string is a number of not. +*/ +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !IsDigit(*z) ){ + return 0; + } + z++; + if( realnum ) *realnum = 0; + while( IsDigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + return *z==0; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0xc0&*(z++))!=0x80 ) n++; + } + return n; +} + +/* +** Return true if zFile does not exist or if it is not an ordinary file. +*/ +#ifdef _WIN32 +# define notNormalFile(X) 0 +#else +static int notNormalFile(const char *zFile){ + struct stat x; + int rc; + memset(&x, 0, sizeof(x)); + rc = stat(zFile, &x); + return rc || !S_ISREG(x.st_mode); +} +#endif + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. +*/ +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; + + while( 1 ){ + if( n+100>nLine ){ + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + if( zLine==0 ) shell_out_of_memory(); + } + if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } + } +#if defined(_WIN32) || defined(WIN32) + /* For interactive input on Windows systems, translate the + ** multi-byte characterset characters into UTF-8. */ + if( stdin_is_interactive && in==stdin ){ + char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); + if( zTrans ){ + int nTrans = strlen30(zTrans)+1; + if( nTrans>nLine ){ + zLine = realloc(zLine, nTrans); + if( zLine==0 ) shell_out_of_memory(); + } + memcpy(zLine, zTrans, nTrans); + sqlite3_free(zTrans); + } + } +#endif /* defined(_WIN32) || defined(WIN32) */ + return zLine; +} + +/* +** Retrieve a single line of input text. +** +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. +** +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. +** +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. +*/ +static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + if( in!=0 ){ + zResult = local_getline(zPrior, in); + }else{ + zPrompt = isContinuation ? continuePrompt : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + printf("%s", zPrompt); + fflush(stdout); + zResult = local_getline(zPrior, stdin); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + if( zResult && *zResult ) shell_add_history(zResult); +#endif + } + return zResult; +} + + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; iz); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ) shell_out_of_memory(); + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + +/* +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +*/ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; + + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.z = 0; + } + return s.z; +} + +/* +** SQL function: shell_module_schema(X) +** +** Return a fake schema for the table-valued function or eponymous virtual +** table X. +*/ +static void shellModuleSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zName = (const char*)sqlite3_value_text(apVal[0]); + char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName); + UNUSED_PARAMETER(nVal); + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} + +/* +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. +*/ +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){ + int n = strlen30(aPrefix[i]); + if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){ + char *z = 0; + char *zFake = 0; + if( zSchema ){ + char cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){ + z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8); + }else{ + z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8); + } + } + if( zName + && aPrefix[i][0]=='V' + && (zFake = shellFakeSchema(db, zSchema, zName))!=0 + ){ + if( z==0 ){ + z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake); + }else{ + z = sqlite3_mprintf("%z\n/* %s */", z, zFake); + } + free(zFake); + } + if( z ){ + sqlite3_result_text(pCtx, z, -1, sqlite3_free); + return; + } + } + } + } + sqlite3_result_value(pCtx, apVal[0]); +} + +/* +** The source code for several run-time loadable extensions is inserted +** below by the ../tool/mkshellc.tcl script. Before processing that included +** code, we need to override some macros to make the included program code +** work here in the middle of this regular program. +*/ +#define SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT2(X) (void)(X) + +#if defined(_WIN32) && defined(_MSC_VER) +INCLUDE test_windirent.h +INCLUDE test_windirent.c +#define dirent DIRENT +#endif +INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/fileio.c +INCLUDE ../ext/misc/completion.c +INCLUDE ../ext/misc/appendvfs.c +INCLUDE ../ext/misc/memtrace.c +INCLUDE ../ext/misc/uint.c +INCLUDE ../ext/misc/decimal.c +INCLUDE ../ext/misc/ieee754.c +INCLUDE ../ext/misc/series.c +#ifdef SQLITE_HAVE_ZLIB +INCLUDE ../ext/misc/zipfile.c +INCLUDE ../ext/misc/sqlar.c +#endif +INCLUDE ../ext/expert/sqlite3expert.h +INCLUDE ../ext/expert/sqlite3expert.c + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +INCLUDE ../ext/misc/dbdata.c +#endif + +#if defined(SQLITE_ENABLE_SESSION) +/* +** State information for a single open session +*/ +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ +}; +#endif + +typedef struct ExpertInfo ExpertInfo; +struct ExpertInfo { + sqlite3expert *pExpert; + int bVerbose; +}; + +/* A single line in the EQP output */ +typedef struct EQPGraphRow EQPGraphRow; +struct EQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + EQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct EQPGraph EQPGraph; +struct EQPGraph { + EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + EQPGraphRow *pLast; /* Last element of the pRow list */ + char zPrefix[100]; /* Graph prefix */ +}; + +/* +** State information about the database connection is contained in an +** instance of the following structure. +*/ +typedef struct ShellState ShellState; +struct ShellState { + sqlite3 *db; /* The database */ + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ + u8 autoEQPtest; /* autoEQP is in test mode */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ + u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ + u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ + unsigned statsOn; /* True to display memory stats before each finalize */ + unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ + int outCount; /* Revert to stdout when reaching zero */ + int cnt; /* Number of records displayed so far */ + int lineno; /* Line number of last line read from in */ + int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ + FILE *in; /* Read commands from this stream */ + FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite3_trace() */ + int nErr; /* Number of errors seen */ + int mode; /* An output mode setting */ + int modePrior; /* Saved mode */ + int cMode; /* temporary output mode for the current query */ + int normalMode; /* Output mode before ".explain on" */ + int writableSchema; /* True if PRAGMA writable_schema=ON */ + int showHeader; /* True to show column names in List or Column mode */ + int nCheck; /* Number of ".check" commands run */ + unsigned nProgress; /* Number of progress callbacks encountered */ + unsigned mxProgress; /* Maximum progress callbacks before failing */ + unsigned flgProgress; /* Flags for the progress callback */ + unsigned shellFlgs; /* Various flags */ + unsigned priorShFlgs; /* Saved copy of flags */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ + char *zDestTable; /* Name of destination table when MODE_Insert */ + char *zTempFile; /* Temporary file that might need deleting */ + char zTestcase[30]; /* Name of current test case */ + char colSeparator[20]; /* Column separator character for several modes */ + char rowSeparator[20]; /* Row separator character for MODE_Ascii */ + char colSepPrior[20]; /* Saved column separator */ + char rowSepPrior[20]; /* Saved row separator */ + int *colWidth; /* Requested width of each column in columnar modes */ + int *actualWidth; /* Actual width of each column */ + int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ + char nullValue[20]; /* The text to print when a NULL comes back from + ** the database */ + char outfile[FILENAME_MAX]; /* Filename for *out */ + const char *zDbFilename; /* name of the database file */ + char *zFreeOnClose; /* Filename to free when closing */ + const char *zVfs; /* Name of VFS to use */ + sqlite3_stmt *pStmt; /* Current statement if any. */ + FILE *pLog; /* Write log output here */ + int *aiIndent; /* Array of indents used in MODE_Explain */ + int nIndent; /* Size of array aiIndent[] */ + int iIndent; /* Index of current op in aiIndent[] */ + EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif + ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +}; + + +/* Allowed values for ShellState.autoEQP +*/ +#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ +#define AUTOEQP_on 1 /* Automatic EQP is on */ +#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ +#define AUTOEQP_full 3 /* Show full EXPLAIN */ + +/* Allowed values for ShellState.openMode +*/ +#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ +#define SHELL_OPEN_NORMAL 1 /* Normal database file */ +#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ +#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ +#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ + +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + +/* Bits in the ShellState.flgProgress variable */ +#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progres + ** callback limit is reached, and for each + ** top-level SQL statement */ +#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ + +/* +** These are the allowed shellFlgs values +*/ +#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ +#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ +#define SHFLG_HeaderSet 0x00000080 /* .header has been used */ +#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ +#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ + +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) + +/* +** These are the allowed modes. +*/ +#define MODE_Line 0 /* One column per line. Blank line between records */ +#define MODE_Column 1 /* One record per line in neat columns */ +#define MODE_List 2 /* One record per line with a separator */ +#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ +#define MODE_Html 4 /* Generate an XHTML table */ +#define MODE_Insert 5 /* Generate SQL "insert" statements */ +#define MODE_Quote 6 /* Quote values as for SQL */ +#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ +#define MODE_Csv 8 /* Quote strings, numbers are plain */ +#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ +#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Pretty 11 /* Pretty-print schemas */ +#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ +#define MODE_Json 13 /* Output JSON */ +#define MODE_Markdown 14 /* Markdown formatting */ +#define MODE_Table 15 /* MySQL-style table formatting */ +#define MODE_Box 16 /* Unicode box-drawing characters */ + +static const char *modeDescr[] = { + "line", + "column", + "list", + "semi", + "html", + "insert", + "quote", + "tcl", + "csv", + "explain", + "ascii", + "prettyprint", + "eqp", + "json", + "markdown", + "table", + "box" +}; + +/* +** These are the column/row/line separators used by the various +** import/export modes. +*/ +#define SEP_Column "|" +#define SEP_Row "\n" +#define SEP_Tab "\t" +#define SEP_Space " " +#define SEP_Comma "," +#define SEP_CrLf "\r\n" +#define SEP_Unit "\x1F" +#define SEP_Record "\x1E" + +/* +** A callback for the sqlite3_log() interface. +*/ +static void shellLog(void *pArg, int iErrCode, const char *zMsg){ + ShellState *p = (ShellState*)pArg; + if( p->pLog==0 ) return; + utf8_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + fflush(p->pLog); +} + +/* +** SQL function: shell_putsnl(X) +** +** Write the text X to the screen (or whatever output is being directed) +** adding a newline at the end, and then return X. +*/ +static void shellPutsFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + (void)nVal; + utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + sqlite3_result_value(pCtx, apVal[0]); +} + +/* +** SQL function: edit(VALUE) +** edit(VALUE,EDITOR) +** +** These steps: +** +** (1) Write VALUE into a temporary file. +** (2) Run program EDITOR on that temporary file. +** (3) Read the temporary file back and return its content as the result. +** (4) Delete the temporary file +** +** If the EDITOR argument is omitted, use the value in the VISUAL +** environment variable. If still there is no EDITOR, through an error. +** +** Also throw an error if the EDITOR program returns a non-zero exit code. +*/ +#ifndef SQLITE_NOHAVE_SYSTEM +static void editFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zEditor; + char *zTempFile = 0; + sqlite3 *db; + char *zCmd = 0; + int bBin; + int rc; + int hasCRNL = 0; + FILE *f = 0; + sqlite3_int64 sz; + sqlite3_int64 x; + unsigned char *p = 0; + + if( argc==2 ){ + zEditor = (const char*)sqlite3_value_text(argv[1]); + }else{ + zEditor = getenv("VISUAL"); + } + if( zEditor==0 ){ + sqlite3_result_error(context, "no editor for edit()", -1); + return; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + sqlite3_result_error(context, "NULL input to edit()", -1); + return; + } + db = sqlite3_context_db_handle(context); + zTempFile = 0; + sqlite3_file_control(db, 0, SQLITE_FCNTL_TEMPFILENAME, &zTempFile); + if( zTempFile==0 ){ + sqlite3_uint64 r = 0; + sqlite3_randomness(sizeof(r), &r); + zTempFile = sqlite3_mprintf("temp%llx", r); + if( zTempFile==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB; + /* When writing the file to be edited, do \n to \r\n conversions on systems + ** that want \r\n line endings */ + f = fopen(zTempFile, bBin ? "wb" : "w"); + if( f==0 ){ + sqlite3_result_error(context, "edit() cannot open temp file", -1); + goto edit_func_end; + } + sz = sqlite3_value_bytes(argv[0]); + if( bBin ){ + x = fwrite(sqlite3_value_blob(argv[0]), 1, (size_t)sz, f); + }else{ + const char *z = (const char*)sqlite3_value_text(argv[0]); + /* Remember whether or not the value originally contained \r\n */ + if( z && strstr(z,"\r\n")!=0 ) hasCRNL = 1; + x = fwrite(sqlite3_value_text(argv[0]), 1, (size_t)sz, f); + } + fclose(f); + f = 0; + if( x!=sz ){ + sqlite3_result_error(context, "edit() could not write the whole file", -1); + goto edit_func_end; + } + zCmd = sqlite3_mprintf("%s \"%s\"", zEditor, zTempFile); + if( zCmd==0 ){ + sqlite3_result_error_nomem(context); + goto edit_func_end; + } + rc = system(zCmd); + sqlite3_free(zCmd); + if( rc ){ + sqlite3_result_error(context, "EDITOR returned non-zero", -1); + goto edit_func_end; + } + f = fopen(zTempFile, "rb"); + if( f==0 ){ + sqlite3_result_error(context, + "edit() cannot reopen temp file after edit", -1); + goto edit_func_end; + } + fseek(f, 0, SEEK_END); + sz = ftell(f); + rewind(f); + p = sqlite3_malloc64( sz+1 ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + goto edit_func_end; + } + x = fread(p, 1, (size_t)sz, f); + fclose(f); + f = 0; + if( x!=sz ){ + sqlite3_result_error(context, "could not read back the whole file", -1); + goto edit_func_end; + } + if( bBin ){ + sqlite3_result_blob64(context, p, sz, sqlite3_free); + }else{ + sqlite3_int64 i, j; + if( hasCRNL ){ + /* If the original contains \r\n then do no conversions back to \n */ + j = sz; + }else{ + /* If the file did not originally contain \r\n then convert any new + ** \r\n back into \n */ + for(i=j=0; imodePrior = p->mode; + p->priorShFlgs = p->shellFlgs; + memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); + memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); +} +static void outputModePop(ShellState *p){ + p->mode = p->modePrior; + p->shellFlgs = p->priorShFlgs; + memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); + memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); +} + +/* +** Output the given string as a hex-encoded blob (eg. X'1234' ) +*/ +static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ + int i; + char *zBlob = (char *)pBlob; + raw_printf(out,"X'"); + for(i=0; i0 ){ + utf8_printf(out,"%.*s",i,z); + } + if( z[i]=='<' ){ + raw_printf(out,"<"); + }else if( z[i]=='&' ){ + raw_printf(out,"&"); + }else if( z[i]=='>' ){ + raw_printf(out,">"); + }else if( z[i]=='\"' ){ + raw_printf(out,"""); + }else if( z[i]=='\'' ){ + raw_printf(out,"'"); + }else{ + break; + } + z += i + 1; + } +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char needCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Output a single term of CSV. Actually, p->colSeparator is used for +** the separator, which may or may not be a comma. p->nullValue is +** the null value. Strings are quoted if necessary. The separator +** is only issued if bSep is true. +*/ +static void output_csv(ShellState *p, const char *z, int bSep){ + FILE *out = p->out; + if( z==0 ){ + utf8_printf(out,"%s",p->nullValue); + }else{ + int i; + int nSep = strlen30(p->colSeparator); + for(i=0; z[i]; i++){ + if( needCsvQuote[((unsigned char*)z)[i]] + || (z[i]==p->colSeparator[0] && + (nSep==1 || memcmp(z, p->colSeparator, nSep)==0)) ){ + i = 0; + break; + } + } + if( i==0 ){ + char *zQuoted = sqlite3_mprintf("\"%w\"", z); + utf8_printf(out, "%s", zQuoted); + sqlite3_free(zQuoted); + }else{ + utf8_printf(out, "%s", z); + } + } + if( bSep ){ + utf8_printf(p->out, "%s", p->colSeparator); + } +} + +/* +** This routine runs when the user presses Ctrl-C +*/ +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + seenInterrupt++; + if( seenInterrupt>2 ) exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); +} + +#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) +/* +** This routine runs for console events (e.g. Ctrl-C) on Win32 +*/ +static BOOL WINAPI ConsoleCtrlHandler( + DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ +){ + if( dwCtrlType==CTRL_C_EVENT ){ + interrupt_handler(0); + return TRUE; + } + return FALSE; +} +#endif + +#ifndef SQLITE_OMIT_AUTHORIZATION +/* +** When the ".auth ON" is set, the following authorizer callback is +** invoked. It always returns SQLITE_OK. +*/ +static int shellAuth( + void *pClientData, + int op, + const char *zA1, + const char *zA2, + const char *zA3, + const char *zA4 +){ + ShellState *p = (ShellState*)pClientData; + static const char *azAction[] = { 0, + "CREATE_INDEX", "CREATE_TABLE", "CREATE_TEMP_INDEX", + "CREATE_TEMP_TABLE", "CREATE_TEMP_TRIGGER", "CREATE_TEMP_VIEW", + "CREATE_TRIGGER", "CREATE_VIEW", "DELETE", + "DROP_INDEX", "DROP_TABLE", "DROP_TEMP_INDEX", + "DROP_TEMP_TABLE", "DROP_TEMP_TRIGGER", "DROP_TEMP_VIEW", + "DROP_TRIGGER", "DROP_VIEW", "INSERT", + "PRAGMA", "READ", "SELECT", + "TRANSACTION", "UPDATE", "ATTACH", + "DETACH", "ALTER_TABLE", "REINDEX", + "ANALYZE", "CREATE_VTABLE", "DROP_VTABLE", + "FUNCTION", "SAVEPOINT", "RECURSIVE" + }; + int i; + const char *az[4]; + az[0] = zA1; + az[1] = zA2; + az[2] = zA3; + az[3] = zA4; + utf8_printf(p->out, "authorizer: %s", azAction[op]); + for(i=0; i<4; i++){ + raw_printf(p->out, " "); + if( az[i] ){ + output_c_string(p->out, az[i]); + }else{ + raw_printf(p->out, "NULL"); + } + } + raw_printf(p->out, "\n"); + return SQLITE_OK; +} +#endif + +/* +** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** +** This routine converts some CREATE TABLE statements for shadow tables +** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +*/ +static void printSchemaLine(FILE *out, const char *z, const char *zTail){ + if( z==0 ) return; + if( zTail==0 ) return; + if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ + utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + }else{ + utf8_printf(out, "%s%s", z, zTail); + } +} +static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ + char c = z[n]; + z[n] = 0; + printSchemaLine(out, z, zTail); + z[n] = c; +} + +/* +** Return true if string z[] has nothing but whitespace and comments to the +** end of the first line. +*/ +static int wsToEol(const char *z){ + int i; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) return 1; + if( IsSpace(z[i]) ) continue; + if( z[i]=='-' && z[i+1]=='-' ) return 1; + return 0; + } + return 1; +} + +/* +** Add a new entry to the EXPLAIN QUERY PLAN data +*/ +static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ + EQPGraphRow *pNew; + int nText = strlen30(zText); + if( p->autoEQPtest ){ + utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + } + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ) shell_out_of_memory(); + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->sGraph.pLast ){ + p->sGraph.pLast->pNext = pNew; + }else{ + p->sGraph.pRow = pNew; + } + p->sGraph.pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->sGraph. +*/ +static void eqp_reset(ShellState *p){ + EQPGraphRow *pRow, *pNext; + for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + memset(&p->sGraph, 0, sizeof(p->sGraph)); +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ + EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void eqp_render_level(ShellState *p, int iEqpId){ + EQPGraphRow *pRow, *pNext; + int n = strlen30(p->sGraph.zPrefix); + char *z; + for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = eqp_next_row(p, iEqpId, pRow); + z = pRow->zText; + utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, + pNext ? "|--" : "`--", z); + if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); + eqp_render_level(p, pRow->iEqpId); + p->sGraph.zPrefix[n] = 0; + } + } +} + +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void eqp_render(ShellState *p){ + EQPGraphRow *pRow = p->sGraph.pRow; + if( pRow ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + eqp_reset(p); + return; + } + utf8_printf(p->out, "%s\n", pRow->zText+3); + p->sGraph.pRow = pRow->pNext; + sqlite3_free(pRow); + }else{ + utf8_printf(p->out, "QUERY PLAN\n"); + } + p->sGraph.zPrefix[0] = 0; + eqp_render_level(p, 0); + eqp_reset(p); + } +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** Progress handler callback. +*/ +static int progress_handler(void *pClientData) { + ShellState *p = (ShellState*)pClientData; + p->nProgress++; + if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ + raw_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; + if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; + return 1; + } + if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ + raw_printf(p->out, "Progress %u\n", p->nProgress); + } + return 0; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + +/* +** Print N dashes +*/ +static void print_dashes(FILE *out, int N){ + const char zDash[] = "--------------------------------------------------"; + const int nDash = sizeof(zDash) - 1; + while( N>nDash ){ + fputs(zDash, out); + N -= nDash; + } + raw_printf(out, "%.*s", N, zDash); +} + +/* +** Print a markdown or table-style row separator using ascii-art +*/ +static void print_row_separator( + ShellState *p, + int nArg, + const char *zSep +){ + int i; + if( nArg>0 ){ + fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[0]+2); + for(i=1; iout); + print_dashes(p->out, p->actualWidth[i]+2); + } + fputs(zSep, p->out); + } + fputs("\n", p->out); +} + +/* +** This is the callback routine that the shell +** invokes for each row of a query result. +*/ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types. Might be NULL */ +){ + int i; + ShellState *p = (ShellState*)pArg; + + if( azArg==0 ) return 0; + switch( p->cMode ){ + case MODE_Line: { + int w = 5; + if( azArg==0 ) break; + for(i=0; iw ) w = len; + } + if( p->cnt++>0 ) utf8_printf(p->out, "%s", p->rowSeparator); + for(i=0; iout,"%*s = %s%s", w, azCol[i], + azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + } + break; + } + case MODE_Explain: { + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + if( nArg>ArraySize(aExplainWidth) ){ + nArg = ArraySize(aExplainWidth); + } + if( p->cnt++==0 ){ + for(i=0; iout, w, azCol[i]); + fputs(i==nArg-1 ? "\n" : " ", p->out); + } + for(i=0; iout, w); + fputs(i==nArg-1 ? "\n" : " ", p->out); + } + } + if( azArg==0 ) break; + for(i=0; iw ){ + w = strlenChar(azArg[i]); + } + if( i==1 && p->aiIndent && p->pStmt ){ + if( p->iIndentnIndent ){ + utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + } + p->iIndent++; + } + utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); + fputs(i==nArg-1 ? "\n" : " ", p->out); + } + break; + } + case MODE_Semi: { /* .schema and .fullschema output */ + printSchemaLine(p->out, azArg[0], ";\n"); + break; + } + case MODE_Pretty: { /* .schema and .fullschema with --indent */ + char *z; + int j; + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + assert( nArg==1 ); + if( azArg[0]==0 ) break; + if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 + ){ + utf8_printf(p->out, "%s;\n", azArg[0]); + break; + } + z = sqlite3_mprintf("%s", azArg[0]); + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 ){ + printSchemaLineN(p->out, z, j, "\n"); + j = 0; + } + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + ){ + if( c=='\n' ) j--; + printSchemaLineN(p->out, z, j, "\n "); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } + } + } + z[j] = 0; + } + printSchemaLine(p->out, z, ";\n"); + sqlite3_free(z); + break; + } + case MODE_List: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,"%s%s",azCol[i], + i==nArg-1 ? p->rowSeparator : p->colSeparator); + } + } + if( azArg==0 ) break; + for(i=0; inullValue; + utf8_printf(p->out, "%s", z); + if( iout, "%s", p->colSeparator); + }else{ + utf8_printf(p->out, "%s", p->rowSeparator); + } + } + break; + } + case MODE_Html: { + if( p->cnt++==0 && p->showHeader ){ + raw_printf(p->out,""); + for(i=0; iout,"\n"); + } + raw_printf(p->out,"\n"); + } + if( azArg==0 ) break; + raw_printf(p->out,""); + for(i=0; iout,"\n"); + } + raw_printf(p->out,"\n"); + break; + } + case MODE_Tcl: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,azCol[i] ? azCol[i] : ""); + if(iout, "%s", p->colSeparator); + } + utf8_printf(p->out, "%s", p->rowSeparator); + } + if( azArg==0 ) break; + for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); + if(iout, "%s", p->colSeparator); + } + utf8_printf(p->out, "%s", p->rowSeparator); + break; + } + case MODE_Csv: { + setBinaryMode(p->out, 1); + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout, "%s", p->rowSeparator); + } + if( nArg>0 ){ + for(i=0; iout, "%s", p->rowSeparator); + } + setTextMode(p->out, 1); + break; + } + case MODE_Insert: { + if( azArg==0 ) break; + utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + raw_printf(p->out,"("); + for(i=0; i0 ) raw_printf(p->out, ","); + if( quoteChar(azCol[i]) ){ + char *z = sqlite3_mprintf("\"%w\"", azCol[i]); + utf8_printf(p->out, "%s", z); + sqlite3_free(z); + }else{ + raw_printf(p->out, "%s", azCol[i]); + } + } + raw_printf(p->out,")"); + } + p->cnt++; + for(i=0; iout, i>0 ? "," : " VALUES("); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + utf8_printf(p->out,"NULL"); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p->out, azArg[i]); + }else{ + output_quoted_escaped_string(p->out, azArg[i]); + } + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + raw_printf(p->out, "1e999"); + }else if( ur==0xfff0000000000000LL ){ + raw_printf(p->out, "-1e999"); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p->out, azArg[i]); + }else{ + output_quoted_escaped_string(p->out, azArg[i]); + } + } + raw_printf(p->out,");\n"); + break; + } + case MODE_Json: { + if( azArg==0 ) break; + if( p->cnt==0 ){ + fputs("[{", p->out); + }else{ + fputs(",\n{", p->out); + } + p->cnt++; + for(i=0; iout, azCol[i], -1); + putc(':', p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + fputs("null",p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + raw_printf(p->out, "1e999"); + }else if( ur==0xfff0000000000000LL ){ + raw_printf(p->out, "-1e999"); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_json_string(p->out, pBlob, nBlob); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_json_string(p->out, azArg[i], -1); + }else{ + utf8_printf(p->out,"%s", azArg[i]); + } + if( iout); + } + } + putc('}', p->out); + break; + } + case MODE_Quote: { + if( azArg==0 ) break; + if( p->cnt==0 && p->showHeader ){ + for(i=0; i0 ) fputs(p->colSeparator, p->out); + output_quoted_string(p->out, azCol[i]); + } + fputs(p->rowSeparator, p->out); + } + p->cnt++; + for(i=0; i0 ) fputs(p->colSeparator, p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + utf8_printf(p->out,"NULL"); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_quoted_string(p->out, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + utf8_printf(p->out,"%s", azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s", z); + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + utf8_printf(p->out,"%s", azArg[i]); + }else{ + output_quoted_string(p->out, azArg[i]); + } + } + fputs(p->rowSeparator, p->out); + break; + } + case MODE_Ascii: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); + utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : ""); + } + utf8_printf(p->out, "%s", p->rowSeparator); + } + if( azArg==0 ) break; + for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); + utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue); + } + utf8_printf(p->out, "%s", p->rowSeparator); + break; + } + case MODE_EQP: { + eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); + break; + } + } + return 0; +} + +/* +** This is the callback routine that the SQLite library +** invokes for each row of a query result. +*/ +static int callback(void *pArg, int nArg, char **azArg, char **azCol){ + /* since we don't have type info, call the shell_callback with a NULL value */ + return shell_callback(pArg, nArg, azArg, azCol, NULL); +} + +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + UNUSED_PARAMETER(az); + if( azArg==0 ) return 0; + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_schema ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_schema\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO [_shell$self]\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); +} + + +/* +** Set the destination table field of the ShellState structure to +** the name of the table given. Escape any quote characters in the +** table name. +*/ +static void set_table_name(ShellState *p, const char *zName){ + int i, n; + char cQuote; + char *z; + + if( p->zDestTable ){ + free(p->zDestTable); + p->zDestTable = 0; + } + if( zName==0 ) return; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += n+2; + z = p->zDestTable = malloc( n+1 ); + if( z==0 ) shell_out_of_memory(); + n = 0; + if( cQuote ) z[n++] = cQuote; + for(i=0; zName[i]; i++){ + z[n++] = zName[i]; + if( zName[i]==cQuote ) z[n++] = cQuote; + } + if( cQuote ) z[n++] = cQuote; + z[n] = 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static int run_table_dump_query( + ShellState *p, /* Query context */ + const char *zSelect /* SELECT statement to extract content */ +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); + if( rc!=SQLITE_OK || !pSelect ){ + utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + return rc; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + utf8_printf(p->out, "%s", z); + for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + raw_printf(p->out, "\n;\n"); + }else{ + raw_printf(p->out, ";\n"); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } + return rc; +} + +/* +** Allocate space and save off current error string. +*/ +static char *save_err_msg( + sqlite3 *db /* Database to query */ +){ + int nErrMsg = 1+strlen30(sqlite3_errmsg(db)); + char *zErrMsg = sqlite3_malloc64(nErrMsg); + if( zErrMsg ){ + memcpy(zErrMsg, sqlite3_errmsg(db), nErrMsg); + } + return zErrMsg; +} + +#ifdef __linux__ +/* +** Attempt to display I/O stats on Linux using /proc/PID/io +*/ +static void displayLinuxIoStats(FILE *out){ + FILE *in; + char z[200]; + sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); + in = fopen(z, "rb"); + if( in==0 ) return; + while( fgets(z, sizeof(z), in)!=0 ){ + static const struct { + const char *zPattern; + const char *zDesc; + } aTrans[] = { + { "rchar: ", "Bytes received by read():" }, + { "wchar: ", "Bytes sent to write():" }, + { "syscr: ", "Read() system calls:" }, + { "syscw: ", "Write() system calls:" }, + { "read_bytes: ", "Bytes read from storage:" }, + { "write_bytes: ", "Bytes written to storage:" }, + { "cancelled_write_bytes: ", "Cancelled write bytes:" }, + }; + int i; + for(i=0; i1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + raw_printf(p->out, "%-36s %s\n", zLabel, zLine); +} + +/* +** Display memory stats. +*/ +static int display_stats( + sqlite3 *db, /* Database to query */ + ShellState *pArg, /* Pointer to ShellState */ + int bReset /* True to reset the stats */ +){ + int iCur; + int iHiwtr; + FILE *out; + if( pArg==0 || pArg->out==0 ) return 0; + out = pArg->out; + + if( pArg->pStmt && pArg->statsOn==2 ){ + int nCol, i, x; + sqlite3_stmt *pStmt = pArg->pStmt; + char z[100]; + nCol = sqlite3_column_count(pStmt); + raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + for(i=0; istatsOn==3 ){ + if( pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + raw_printf(pArg->out, "VM-steps: %d\n", iCur); + } + return 0; + } + + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); + if( pArg->shellFlgs & SHFLG_Pagecache ){ + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); + } + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); +#ifdef YYTRACKMAXSTACKDEPTH + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); +#endif + + if( db ){ + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, + "Lookaside Slots Used: %d (max %d)\n", + iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Successful lookaside attempts: %d\n", + iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Lookaside failures due to size: %d\n", + iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Lookaside failures due to OOM: %d\n", + iHiwtr); + } + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Pager Heap Usage: %d bytes\n", + iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache hits: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); + raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", + iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); + raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", + iCur); + } + + if( pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); + raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); + raw_printf(pArg->out, "Sort Operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); + raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + } + +#ifdef __linux__ + displayLinuxIoStats(pArg->out); +#endif + + /* Do not remove this machine readable comment: extra-stats-output-here */ + + return 0; +} + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + int i, k, n, mx; + raw_printf(pArg->out, "-------- scanstats --------\n"); + mx = 0; + for(k=0; k<=mx; k++){ + double rEstLoop = 1.0; + for(i=n=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nLoop, nVisit; + double rEst; + int iSid; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>mx ) mx = iSid; + if( iSid!=k ) continue; + if( n==0 ){ + rEstLoop = (double)nLoop; + if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k); + } + n++; + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); + rEstLoop *= rEst; + raw_printf(pArg->out, + " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + ); + } + } + raw_printf(pArg->out, "---------------------------\n"); +#endif +} + +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int str_in_array(const char *zStr, const char **azArray){ + int i; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** If compiled statement pSql appears to be an EXPLAIN statement, allocate +** and populate the ShellState.aiIndent[] array with the number of +** spaces each opcode should be indented before it is output. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ + const char *zSql; /* The text of the SQL statement */ + const char *z; /* Used to check if this is an EXPLAIN */ + int *abYield = 0; /* True if op is an OP_Yield */ + int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ + int iOp; /* Index of operation in p->aiIndent[] */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* Try to figure out if this is really an EXPLAIN statement. If this + ** cannot be verified, return early. */ + if( sqlite3_column_count(pSql)!=8 ){ + p->cMode = p->mode; + return; + } + zSql = sqlite3_sql(pSql); + if( zSql==0 ) return; + for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); + if( sqlite3_strnicmp(z, "explain", 7) ){ + p->cMode = p->mode; + return; + } + + for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ + int i; + int iAddr = sqlite3_column_int(pSql, 0); + const char *zOp = (const char*)sqlite3_column_text(pSql, 1); + + /* Set p2 to the P2 field of the current opcode. Then, assuming that + ** p2 is an instruction address, set variable p2op to the index of that + ** instruction in the aiIndent[] array. p2 and p2op may be different if + ** the current instruction is part of a sub-program generated by an + ** SQL trigger or foreign key. */ + int p2 = sqlite3_column_int(pSql, 3); + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the p->aiIndent array as required */ + if( iOp>=nAlloc ){ + if( iOp==0 ){ + /* Do further verfication that this is explain output. Abort if + ** it is not */ + static const char *explainCols[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; + int jj; + for(jj=0; jjcMode = p->mode; + sqlite3_reset(pSql); + return; + } + } + } + nAlloc += 100; + p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + if( p->aiIndent==0 ) shell_out_of_memory(); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( abYield==0 ) shell_out_of_memory(); + } + abYield[iOp] = str_in_array(zOp, azYield); + p->aiIndent[iOp] = 0; + p->nIndent = iOp+1; + + if( str_in_array(zOp, azNext) ){ + for(i=p2op; iaiIndent[i] += 2; + } + if( str_in_array(zOp, azGoto) && p2opnIndent + && (abYield[p2op] || sqlite3_column_int(pSql, 2)) + ){ + for(i=p2op; iaiIndent[i] += 2; + } + } + + p->iIndent = 0; + sqlite3_free(abYield); + sqlite3_reset(pSql); +} + +/* +** Free the array allocated by explain_data_prepare(). +*/ +static void explain_data_delete(ShellState *p){ + sqlite3_free(p->aiIndent); + p->aiIndent = 0; + p->nIndent = 0; + p->iIndent = 0; +} + +/* +** Disable and restore .wheretrace and .selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +} +static void restore_debug_trace_modes(void){ + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +} + +/* Create the TEMP table used to store parameter bindings */ +static void bind_table_init(ShellState *p){ + int wrSchema = 0; + int defensiveMode = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &defensiveMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, + "CREATE TABLE IF NOT EXISTS temp.sqlite_parameters(\n" + " key TEXT PRIMARY KEY,\n" + " value ANY\n" + ") WITHOUT ROWID;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, defensiveMode, 0); +} + +/* +** Bind parameters on a prepared statement. +** +** Parameter bindings are taken from a TEMP table of the form: +** +** CREATE TEMP TABLE sqlite_parameters(key TEXT PRIMARY KEY, value) +** WITHOUT ROWID; +** +** No bindings occur if this table does not exist. The name of the table +** begins with "sqlite_" so that it will not collide with ordinary application +** tables. The table must be in the TEMP schema. +*/ +static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ + int nVar; + int i; + int rc; + sqlite3_stmt *pQ = 0; + + nVar = sqlite3_bind_parameter_count(pStmt); + if( nVar==0 ) return; /* Nothing to do */ + if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", + "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ + return; /* Parameter table does not exist */ + } + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); + if( rc || pQ==0 ) return; + for(i=1; i<=nVar; i++){ + char zNum[30]; + const char *zVar = sqlite3_bind_parameter_name(pStmt, i); + if( zVar==0 ){ + sqlite3_snprintf(sizeof(zNum),zNum,"?%d",i); + zVar = zNum; + } + sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); + if( sqlite3_step(pQ)==SQLITE_ROW ){ + sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); + }else{ + sqlite3_bind_null(pStmt, i); + } + sqlite3_reset(pQ); + } + sqlite3_finalize(pQ); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void print_box_line(FILE *out, int N){ + const char zDash[] = + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; + const int nDash = sizeof(zDash) - 1; + N *= 3; + while( N>nDash ){ + utf8_printf(out, zDash); + N -= nDash; + } + utf8_printf(out, "%.*s", N, zDash); +} + +/* +** Draw a horizontal separator for a MODE_Box table. +*/ +static void print_box_row_separator( + ShellState *p, + int nArg, + const char *zSep1, + const char *zSep2, + const char *zSep3 +){ + int i; + if( nArg>0 ){ + utf8_printf(p->out, "%s", zSep1); + print_box_line(p->out, p->actualWidth[0]+2); + for(i=1; iout, "%s", zSep2); + print_box_line(p->out, p->actualWidth[i]+2); + } + utf8_printf(p->out, "%s", zSep3); + } + fputs("\n", p->out); +} + + + +/* +** Run a prepared statement and output the result in one of the +** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, +** or MODE_Box. +** +** This is different from ordinary exec_prepared_stmt() in that +** it has to run the entire query and gather the results into memory +** first, in order to determine column widths, before providing +** any output. +*/ +static void exec_prepared_stmt_columnar( + ShellState *p, /* Pointer to ShellState */ + sqlite3_stmt *pStmt /* Statment to run */ +){ + sqlite3_int64 nRow = 0; + int nColumn = 0; + char **azData = 0; + sqlite3_int64 nAlloc = 0; + const char *z; + int rc; + sqlite3_int64 i, nData; + int j, nTotal, w, n; + const char *colSep = 0; + const char *rowSep = 0; + + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ) return; + nColumn = sqlite3_column_count(pStmt); + nAlloc = nColumn*4; + if( nAlloc<=0 ) nAlloc = 1; + azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); + if( azData==0 ) shell_out_of_memory(); + for(i=0; i= nAlloc ){ + nAlloc *= 2; + azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); + if( azData==0 ) shell_out_of_memory(); + } + nRow++; + for(i=0; ip->nWidth ){ + p->colWidth = realloc(p->colWidth, nColumn*2*sizeof(int)); + if( p->colWidth==0 ) shell_out_of_memory(); + for(i=p->nWidth; icolWidth[i] = 0; + p->nWidth = nColumn; + p->actualWidth = &p->colWidth[nColumn]; + } + memset(p->actualWidth, 0, nColumn*sizeof(int)); + for(i=0; icolWidth[i]; + if( w<0 ) w = -w; + p->actualWidth[i] = w; + } + nTotal = nColumn*(nRow+1); + for(i=0; inullValue; + n = strlenChar(z); + j = i%nColumn; + if( n>p->actualWidth[j] ) p->actualWidth[j] = n; + } + if( seenInterrupt ) goto columnar_end; + if( nColumn==0 ) goto columnar_end; + switch( p->cMode ){ + case MODE_Column: { + colSep = " "; + rowSep = "\n"; + if( p->showHeader ){ + for(i=0; iactualWidth[i]; + if( p->colWidth[i]<0 ) w = -w; + utf8_width_print(p->out, w, azData[i]); + fputs(i==nColumn-1?"\n":" ", p->out); + } + for(i=0; iout, p->actualWidth[i]); + fputs(i==nColumn-1?"\n":" ", p->out); + } + } + break; + } + case MODE_Table: { + colSep = " | "; + rowSep = " |\n"; + print_row_separator(p, nColumn, "+"); + fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "+"); + break; + } + case MODE_Markdown: { + colSep = " | "; + rowSep = " |\n"; + fputs("| ", p->out); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "|"); + break; + } + case MODE_Box: { + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); + utf8_printf(p->out, BOX_13 " "); + for(i=0; iactualWidth[i]; + n = strlenChar(azData[i]); + utf8_printf(p->out, "%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + } + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + break; + } + } + for(i=nColumn, j=0; icMode!=MODE_Column ){ + utf8_printf(p->out, "%s", p->cMode==MODE_Box?BOX_13" ":"| "); + } + z = azData[i]; + if( z==0 ) z = p->nullValue; + w = p->actualWidth[j]; + if( p->colWidth[j]<0 ) w = -w; + utf8_width_print(p->out, w, z); + if( j==nColumn-1 ){ + utf8_printf(p->out, "%s", rowSep); + j = -1; + if( seenInterrupt ) goto columnar_end; + }else{ + utf8_printf(p->out, "%s", colSep); + } + } + if( p->cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); + } +columnar_end: + if( seenInterrupt ){ + utf8_printf(p->out, "Interrupt\n"); + } + nData = (nRow+1)*nColumn; + for(i=0; icMode==MODE_Column + || pArg->cMode==MODE_Table + || pArg->cMode==MODE_Box + || pArg->cMode==MODE_Markdown + ){ + exec_prepared_stmt_columnar(pArg, pStmt); + return; + } + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + rc = SQLITE_NOMEM; + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; icMode==MODE_Insert ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + if( pArg->cMode==MODE_Json ){ + fputs("]\n", pArg->out); + } + } + } +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** This function is called to process SQL if the previous shell command +** was ".expert". It passes the SQL in the second argument directly to +** the sqlite3expert object. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error +** code. In this case, (*pzErr) may be set to point to a buffer containing +** an English language error message. It is the responsibility of the +** caller to eventually free this buffer using sqlite3_free(). +*/ +static int expertHandleSQL( + ShellState *pState, + const char *zSql, + char **pzErr +){ + assert( pState->expert.pExpert ); + assert( pzErr==0 || *pzErr==0 ); + return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr); +} + +/* +** This function is called either to silently clean up the object +** created by the ".expert" command (if bCancel==1), or to generate a +** report from it and then clean it up (if bCancel==0). +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error +** code. In this case, (*pzErr) may be set to point to a buffer containing +** an English language error message. It is the responsibility of the +** caller to eventually free this buffer using sqlite3_free(). +*/ +static int expertFinish( + ShellState *pState, + int bCancel, + char **pzErr +){ + int rc = SQLITE_OK; + sqlite3expert *p = pState->expert.pExpert; + assert( p ); + assert( bCancel || pzErr==0 || *pzErr==0 ); + if( bCancel==0 ){ + FILE *out = pState->out; + int bVerbose = pState->expert.bVerbose; + + rc = sqlite3_expert_analyze(p, pzErr); + if( rc==SQLITE_OK ){ + int nQuery = sqlite3_expert_count(p); + int i; + + if( bVerbose ){ + const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); + raw_printf(out, "-- Candidates -----------------------------\n"); + raw_printf(out, "%s\n", zCand); + } + for(i=0; iexpert.pExpert = 0; + return rc; +} + +/* +** Implementation of ".expert" dot command. +*/ +static int expertDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int rc = SQLITE_OK; + char *zErr = 0; + int i; + int iSample = 0; + + assert( pState->expert.pExpert==0 ); + memset(&pState->expert, 0, sizeof(ExpertInfo)); + + for(i=1; rc==SQLITE_OK && i=2 && 0==strncmp(z, "-verbose", n) ){ + pState->expert.bVerbose = 1; + } + else if( n>=2 && 0==strncmp(z, "-sample", n) ){ + if( i==(nArg-1) ){ + raw_printf(stderr, "option requires an argument: %s\n", z); + rc = SQLITE_ERROR; + }else{ + iSample = (int)integerValue(azArg[++i]); + if( iSample<0 || iSample>100 ){ + raw_printf(stderr, "value out of range: %s\n", azArg[i]); + rc = SQLITE_ERROR; + } + } + } + else{ + raw_printf(stderr, "unknown option: %s\n", z); + rc = SQLITE_ERROR; + } + } + + if( rc==SQLITE_OK ){ + pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); + if( pState->expert.pExpert==0 ){ + raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr); + rc = SQLITE_ERROR; + }else{ + sqlite3_expert_config( + pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample + ); + } + } + + return rc; +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Execute a statement or set of statements. Print +** any result rows/columns depending on the current mode +** set via the supplied callback. +** +** This is very similar to SQLite's built-in sqlite3_exec() +** function except it takes a slightly different callback +** and callback data argument. +*/ +static int shell_exec( + ShellState *pArg, /* Pointer to ShellState */ + const char *zSql, /* SQL to be evaluated */ + char **pzErrMsg /* Error msg written here */ +){ + sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ + int rc = SQLITE_OK; /* Return Code */ + int rc2; + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3 *db = pArg->db; + + if( pzErrMsg ){ + *pzErrMsg = NULL; + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pArg->expert.pExpert ){ + rc = expertHandleSQL(pArg, zSql, pzErrMsg); + return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg); + } +#endif + + while( zSql[0] && (SQLITE_OK == rc) ){ + static const char *zStmtSql; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); + if( SQLITE_OK != rc ){ + if( pzErrMsg ){ + *pzErrMsg = save_err_msg(db); + } + }else{ + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + while( IsSpace(zSql[0]) ) zSql++; + continue; + } + zStmtSql = sqlite3_sql(pStmt); + if( zStmtSql==0 ) zStmtSql = ""; + while( IsSpace(zStmtSql[0]) ) zStmtSql++; + + /* save off the prepared statment handle and reset row count */ + if( pArg ){ + pArg->pStmt = pStmt; + pArg->cnt = 0; + } + + /* echo the sql statement if echo on */ + if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ + utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); + } + + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ + if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ + sqlite3_stmt *pExplain; + char *zEQP; + int triggerEQP = 0; + disable_debug_trace_modes(); + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); + if( pArg->autoEQP>=AUTOEQP_trigger ){ + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); + } + zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pExplain)==SQLITE_ROW ){ + const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); + int iEqpId = sqlite3_column_int(pExplain, 0); + int iParentId = sqlite3_column_int(pExplain, 1); + if( zEQPLine==0 ) zEQPLine = ""; + if( zEQPLine[0]=='-' ) eqp_render(pArg); + eqp_append(pArg, iEqpId, iParentId, zEQPLine); + } + eqp_render(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + if( pArg->autoEQP>=AUTOEQP_full ){ + /* Also do an EXPLAIN for ".eqp full" mode */ + zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + pArg->cMode = MODE_Explain; + explain_data_prepare(pArg, pExplain); + exec_prepared_stmt(pArg, pExplain); + explain_data_delete(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + } + if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); + /* Reprepare pStmt before reactiving trace modes */ + sqlite3_finalize(pStmt); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( pArg ) pArg->pStmt = pStmt; + } + restore_debug_trace_modes(); + } + + if( pArg ){ + pArg->cMode = pArg->mode; + if( pArg->autoExplain ){ + if( sqlite3_stmt_isexplain(pStmt)==1 ){ + pArg->cMode = MODE_Explain; + } + if( sqlite3_stmt_isexplain(pStmt)==2 ){ + pArg->cMode = MODE_EQP; + } + } + + /* If the shell is currently in ".explain" mode, gather the extra + ** data required to add indents to the output.*/ + if( pArg->cMode==MODE_Explain ){ + explain_data_prepare(pArg, pStmt); + } + } + + bind_prepared_stmt(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + eqp_render(pArg); + + /* print usage stats if stats on */ + if( pArg && pArg->statsOn ){ + display_stats(db, pArg, 0); + } + + /* print loop-counters if required */ + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); + } + + /* Finalize the statement just executed. If this fails, save a + ** copy of the error message. Otherwise, set zSql to point to the + ** next statement to execute. */ + rc2 = sqlite3_finalize(pStmt); + if( rc!=SQLITE_NOMEM ) rc = rc2; + if( rc==SQLITE_OK ){ + zSql = zLeftover; + while( IsSpace(zSql[0]) ) zSql++; + }else if( pzErrMsg ){ + *pzErrMsg = save_err_msg(db); + } + + /* clear saved stmt handle */ + if( pArg ){ + pArg->pStmt = NULL; + } + } + } /* end while */ + + return rc; +} + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ) shell_out_of_memory(); + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + if( azCol==0 ) return 0; + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + +/* +** Toggle the reverse_unordered_selects setting. +*/ +static void toggleSelectOrder(sqlite3 *db){ + sqlite3_stmt *pStmt = 0; + int iSetting = 0; + char zStmt[100]; + sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iSetting = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + sqlite3_snprintf(sizeof(zStmt), zStmt, + "PRAGMA reverse_unordered_selects(%d)", !iSetting); + sqlite3_exec(db, zStmt, 0, 0, 0); +} + +/* +** This is a different callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + ShellState *p = (ShellState *)pArg; + int dataOnly; + int noSys; + + UNUSED_PARAMETER(azNotUsed); + if( nArg!=3 || azArg==0 ) return 0; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0; + noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; + + if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ + if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ + if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( dataOnly ){ + /* no-op */ + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + char *zIns; + if( !p->writableSchema ){ + raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); + p->writableSchema = 1; + } + zIns = sqlite3_mprintf( + "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + utf8_printf(p->out, "%s\n", zIns); + sqlite3_free(zIns); + return 0; + }else{ + printSchemaLine(p->out, zSql, ";\n"); + } + + if( strcmp(zType, "table")==0 ){ + ShellText sSelect; + ShellText sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; + + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; + } + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); + } + appendText(&sTable, ")", 0); + } + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p, sSelect.z, 0); + if( (rc&0xff)==SQLITE_CORRUPT ){ + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + toggleSelectOrder(p->db); + shell_exec(p, sSelect.z, 0); + toggleSelectOrder(p->db); + } + p->zDestTable = savedDestTable; + p->mode = savedMode; + freeText(&sTable); + freeText(&sSelect); + if( rc ) p->nErr++; + } + return 0; +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static int run_schema_dump_query( + ShellState *p, + const char *zQuery +){ + int rc; + char *zErr = 0; + rc = sqlite3_exec(p->db, zQuery, dump_callback, p, &zErr); + if( rc==SQLITE_CORRUPT ){ + char *zQ2; + int len = strlen30(zQuery); + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + if( zErr ){ + utf8_printf(p->out, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + zErr = 0; + } + zQ2 = malloc( len+100 ); + if( zQ2==0 ) return rc; + sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); + rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); + if( rc ){ + utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr); + }else{ + rc = SQLITE_CORRUPT; + } + sqlite3_free(zErr); + free(zQ2); + } + return rc; +} + +/* +** Text of help messages. +** +** The help text for each individual command begins with a line that starts +** with ".". Subsequent lines are supplimental information. +** +** There must be two or more spaces between the end of the command and the +** start of the description of what that command does. +*/ +static const char *(azHelp[]) = { +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + ".archive ... Manage SQL archives", + " Each command must have exactly one of the following options:", + " -c, --create Create a new archive", + " -u, --update Add or update files with changed mtime", + " -i, --insert Like -u but always add even if unchanged", + " -t, --list List contents of archive", + " -x, --extract Extract files from archive", + " Optional arguments:", + " -v, --verbose Print each filename as it is processed", + " -f FILE, --file FILE Use archive FILE (default is current db)", + " -a FILE, --append FILE Open FILE using the apndvfs VFS", + " -C DIR, --directory DIR Read/extract files from directory DIR", + " -n, --dryrun Show the SQL that would have occurred", + " Examples:", + " .ar -cf ARCHIVE foo bar # Create ARCHIVE from files foo and bar", + " .ar -tf ARCHIVE # List members of ARCHIVE", + " .ar -xvf ARCHIVE # Verbosely extract files from ARCHIVE", + " See also:", + " http://sqlite.org/cli.html#sqlar_archive_support", +#endif +#ifndef SQLITE_OMIT_AUTHORIZATION + ".auth ON|OFF Show authorizer callbacks", +#endif + ".backup ?DB? FILE Backup DB (default \"main\") to FILE", + " --append Use the appendvfs", + " --async Write to FILE without journal and fsync()", + ".bail on|off Stop after hitting an error. Default OFF", + ".binary on|off Turn binary output on or off. Default OFF", + ".cd DIRECTORY Change the working directory to DIRECTORY", + ".changes on|off Show number of rows changed by SQL", + ".check GLOB Fail if output since .testcase does not match", + ".clone NEWDB Clone data into NEWDB from the existing database", + ".databases List names and files of attached databases", + ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", + ".dbinfo ?DB? Show status information about the database", + ".dump ?OBJECTS? Render database content as SQL", + " Options:", + " --data-only Output only INSERT statements", + " --newlines Allow unescaped newline characters in output", + " --nosys Omit system tables (ex: \"sqlite_stat1\")", + " --preserve-rowids Include ROWID values in the output", + " OBJECTS is a LIKE pattern for tables, indexes, triggers or views to dump", + " Additional LIKE patterns can be given in subsequent arguments", + ".echo on|off Turn command echo on or off", + ".eqp on|off|full|... Enable or disable automatic EXPLAIN QUERY PLAN", + " Other Modes:", +#ifdef SQLITE_DEBUG + " test Show raw EXPLAIN QUERY PLAN output", + " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", +#endif + " trigger Like \"full\" but also show trigger bytecode", + ".excel Display the output of next command in spreadsheet", + " --bom Put a UTF8 byte-order mark on intermediate file", + ".exit ?CODE? Exit this program with return-code CODE", + ".expert EXPERIMENTAL. Suggest indexes for queries", + ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", + ".filectrl CMD ... Run various sqlite3_file_control() operations", + " --schema SCHEMA Use SCHEMA instead of \"main\"", + " --help Show CMD details", + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", + ".headers on|off Turn display of headers on or off", + ".help ?-all? ?PATTERN? Show help text for PATTERN", + ".import FILE TABLE Import data from FILE into TABLE", + " Options:", + " --ascii Use \\037 and \\036 as column and row separators", + " --csv Use , and \\n as column and row separators", + " --skip N Skip the first N rows of input", + " -v \"Verbose\" - increase auxiliary output", + " Notes:", + " * If TABLE does not exist, it is created. The first row of input", + " determines the column names.", + " * If neither --csv or --ascii are used, the input mode is derived", + " from the \".mode\" output mode", + " * If FILE begins with \"|\" then it is a command that generates the", + " input text.", +#ifndef SQLITE_OMIT_TEST_CONTROL + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", +#endif + ".indexes ?TABLE? Show names of indexes", + " If TABLE is specified, only show indexes for", + " tables matching TABLE using the LIKE operator.", +#ifdef SQLITE_ENABLE_IOTRACE + ".iotrace FILE Enable I/O diagnostic logging to FILE", +#endif + ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", + ".lint OPTIONS Report potential schema issues.", + " Options:", + " fkey-indexes Find missing foreign key indexes", +#ifndef SQLITE_OMIT_LOAD_EXTENSION + ".load FILE ?ENTRY? Load an extension library", +#endif + ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", + ".mode MODE ?TABLE? Set output mode", + " MODE is one of:", + " ascii Columns/rows delimited by 0x1F and 0x1E", + " box Tables using unicode box-drawing characters", + " csv Comma-separated values", + " column Output in columns. (See .width)", + " html HTML
"); + output_html_string(p->out, azCol[i]); + raw_printf(p->out,"
"); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + raw_printf(p->out,"
code", + " insert SQL insert statements for TABLE", + " json Results in a JSON array", + " line One value per line", + " list Values delimited by \"|\"", + " markdown Markdown table format", + " quote Escape answers as for SQL", + " table ASCII-art table", + " tabs Tab-separated values", + " tcl TCL list elements", + ".nullvalue STRING Use STRING in place of NULL values", + ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", + " If FILE begins with '|' then open as a pipe", + " --bom Put a UTF8 byte-order mark at the beginning", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet (same as \".excel\")", +#ifdef SQLITE_DEBUG + ".oom ?--repeat M? ?N? Simulate an OOM error on the N-th allocation", +#endif + ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", + " Options:", + " --append Use appendvfs to append database to the end of FILE", +#ifdef SQLITE_ENABLE_DESERIALIZE + " --deserialize Load into memory useing sqlite3_deserialize()", + " --hexdb Load the output of \"dbtotxt\" as an in-memory db", + " --maxsize N Maximum size for --hexdb or --deserialized database", +#endif + " --new Initialize FILE to an empty database", + " --nofollow Do not follow symbolic links", + " --readonly Open FILE readonly", + " --zip FILE is a ZIP archive", + ".output ?FILE? Send output to FILE or stdout if FILE is omitted", + " If FILE begins with '|' then open it as a pipe.", + " Options:", + " --bom Prefix output with a UTF8 byte-order mark", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet", + ".parameter CMD ... Manage SQL parameter bindings", + " clear Erase all bindings", + " init Initialize the TEMP table that holds bindings", + " list List the current parameter bindings", + " set PARAMETER VALUE Given SQL parameter PARAMETER a value of VALUE", + " PARAMETER should start with one of: $ : @ ?", + " unset PARAMETER Remove PARAMETER from the binding table", + ".print STRING... Print literal STRING", +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + ".progress N Invoke progress handler after every N opcodes", + " --limit N Interrupt after N progress callbacks", + " --once Do no more than one progress interrupt", + " --quiet|-q No output except at interrupts", + " --reset Reset the count for each input and interrupt", +#endif + ".prompt MAIN CONTINUE Replace the standard prompts", + ".quit Exit this program", + ".read FILE Read input from FILE", +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + ".recover Recover as much data as possible from corrupt db.", + " --freelist-corrupt Assume the freelist is corrupt", + " --recovery-db NAME Store recovery metadata in database file NAME", + " --lost-and-found TABLE Alternative name for the lost-and-found table", + " --no-rowids Do not attempt to recover rowid values", + " that are not also INTEGER PRIMARY KEYs", +#endif + ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", + ".save FILE Write in-memory database into FILE", + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", + ".schema ?PATTERN? Show the CREATE statements matching PATTERN", + " Options:", + " --indent Try to pretty-print the schema", + " --nosys Omit objects whose names start with \"sqlite_\"", + ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + " Options:", + " --init Create a new SELFTEST table", + " -v Verbose output", + ".separator COL ?ROW? Change the column and row separators", +#if defined(SQLITE_ENABLE_SESSION) + ".session ?NAME? CMD ... Create or control sessions", + " Subcommands:", + " attach TABLE Attach TABLE", + " changeset FILE Write a changeset into FILE", + " close Close one session", + " enable ?BOOLEAN? Set or query the enable bit", + " filter GLOB... Reject tables matching GLOBs", + " indirect ?BOOLEAN? Mark or query the indirect status", + " isempty Query whether the session is empty", + " list List currently open session names", + " open DB NAME Open a new session on DB", + " patchset FILE Write a patchset into FILE", + " If ?NAME? is omitted, the first defined session is used.", +#endif + ".sha3sum ... Compute a SHA3 hash of database content", + " Options:", + " --schema Also hash the sqlite_schema table", + " --sha3-224 Use the sha3-224 algorithm", + " --sha3-256 Use the sha3-256 algorithm (default)", + " --sha3-384 Use the sha3-384 algorithm", + " --sha3-512 Use the sha3-512 algorithm", + " Any other argument is a LIKE pattern for tables to hash", +#ifndef SQLITE_NOHAVE_SYSTEM + ".shell CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".show Show the current values for various settings", + ".stats ?ARG? Show stats or turn stats on or off", + " off Turn off automatic stat display", + " on Turn on automatic stat display", + " stmt Show statement stats", + " vmstep Show the virtual machine step count only", +#ifndef SQLITE_NOHAVE_SYSTEM + ".system CMD ARGS... Run CMD ARGS... in a system shell", +#endif + ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", + ".testcase NAME Begin redirecting output to 'testcase-out.txt'", + ".testctrl CMD ... Run various sqlite3_test_control() operations", + " Run \".testctrl\" with no arguments for details", + ".timeout MS Try opening locked tables for MS milliseconds", + ".timer on|off Turn SQL timer on or off", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ +#ifdef SQLITE_DEBUG + ".unmodule NAME ... Unregister virtual table modules", + " --allexcept Unregister everything except those named", +#endif + ".vfsinfo ?AUX? Information about the top-level VFS", + ".vfslist List all available VFSes", + ".vfsname ?AUX? Print the name of the VFS stack", + ".width NUM1 NUM2 ... Set minimum column widths for columnar output", + " Negative values right-justify", +}; + +/* +** Output help text. +** +** zPattern describes the set of commands for which help text is provided. +** If zPattern is NULL, then show all commands, but only give a one-line +** description of each. +** +** Return the number of matches. +*/ +static int showHelp(FILE *out, const char *zPattern){ + int i = 0; + int j = 0; + int n = 0; + char *zPat; + if( zPattern==0 + || zPattern[0]=='0' + || strcmp(zPattern,"-a")==0 + || strcmp(zPattern,"-all")==0 + || strcmp(zPattern,"--all")==0 + ){ + /* Show all commands, but only one line per command */ + if( zPattern==0 ) zPattern = ""; + for(i=0; ip); + sqlite3_free(pSession->zName); + for(i=0; inFilter; i++){ + sqlite3_free(pSession->azFilter[i]); + } + sqlite3_free(pSession->azFilter); + memset(pSession, 0, sizeof(OpenSession)); +} +#endif + +/* +** Close all OpenSession objects and release all associated resources. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static void session_close_all(ShellState *p){ + int i; + for(i=0; inSession; i++){ + session_close(&p->aSession[i]); + } + p->nSession = 0; +} +#else +# define session_close_all(X) +#endif + +/* +** Implementation of the xFilter function for an open session. Omit +** any tables named by ".session filter" but let all other table through. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static int session_filter(void *pCtx, const char *zTab){ + OpenSession *pSession = (OpenSession*)pCtx; + int i; + for(i=0; inFilter; i++){ + if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0; + } + return 1; +} +#endif + +/* +** Try to deduce the type of file for zName based on its content. Return +** one of the SHELL_OPEN_* constants. +** +** If the file does not exist or is empty but its name looks like a ZIP +** archive and the dfltZip flag is true, then assume it is a ZIP archive. +** Otherwise, assume an ordinary database regardless of the filename if +** the type cannot be determined from content. +*/ +int deduceDatabaseType(const char *zName, int dfltZip){ + FILE *f = fopen(zName, "rb"); + size_t n; + int rc = SHELL_OPEN_UNSPEC; + char zBuf[100]; + if( f==0 ){ + if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + return SHELL_OPEN_ZIPFILE; + }else{ + return SHELL_OPEN_NORMAL; + } + } + n = fread(zBuf, 16, 1, f); + if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){ + fclose(f); + return SHELL_OPEN_NORMAL; + } + fseek(f, -25, SEEK_END); + n = fread(zBuf, 25, 1, f); + if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){ + rc = SHELL_OPEN_APPENDVFS; + }else{ + fseek(f, -22, SEEK_END); + n = fread(zBuf, 22, 1, f); + if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05 + && zBuf[3]==0x06 ){ + rc = SHELL_OPEN_ZIPFILE; + }else if( n==0 && dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + rc = SHELL_OPEN_ZIPFILE; + } + } + fclose(f); + return rc; +} + +#ifdef SQLITE_ENABLE_DESERIALIZE +/* +** Reconstruct an in-memory database using the output from the "dbtotxt" +** program. Read content from the file in p->zDbFilename. If p->zDbFilename +** is 0, then read from standard input. +*/ +static unsigned char *readHexDb(ShellState *p, int *pnData){ + unsigned char *a = 0; + int nLine; + int n = 0; + int pgsz = 0; + int iOffset = 0; + int j, k; + int rc; + FILE *in; + unsigned int x[16]; + char zLine[1000]; + if( p->zDbFilename ){ + in = fopen(p->zDbFilename, "r"); + if( in==0 ){ + utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename); + return 0; + } + nLine = 0; + }else{ + in = p->in; + nLine = p->lineno; + if( in==0 ) in = stdin; + } + *pnData = 0; + nLine++; + if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); + if( rc!=2 ) goto readHexDb_error; + if( n<0 ) goto readHexDb_error; + if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error; + n = (n+pgsz-1)&~(pgsz-1); /* Round n up to the next multiple of pgsz */ + a = sqlite3_malloc( n ? n : 1 ); + if( a==0 ){ + utf8_printf(stderr, "Out of memory!\n"); + goto readHexDb_error; + } + memset(a, 0, n); + if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ + utf8_printf(stderr, "invalid pagesize\n"); + goto readHexDb_error; + } + for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + rc = sscanf(zLine, "| page %d offset %d", &j, &k); + if( rc==2 ){ + iOffset = k; + continue; + } + if( strncmp(zLine, "| end ", 6)==0 ){ + break; + } + rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", + &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], + &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); + if( rc==17 ){ + k = iOffset+j; + if( k+16<=n ){ + int ii; + for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; + } + } + } + *pnData = n; + if( in!=p->in ){ + fclose(in); + }else{ + p->lineno = nLine; + } + return a; + +readHexDb_error: + if( in!=p->in ){ + fclose(in); + }else{ + while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ + nLine++; + if(strncmp(zLine, "| end ", 6)==0 ) break; + } + p->lineno = nLine; + } + sqlite3_free(a); + utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); + return 0; +} +#endif /* SQLITE_ENABLE_DESERIALIZE */ + +/* +** Scalar function "shell_int32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void shellInt32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + UNUSED_PARAMETER(argc); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + + ((sqlite3_int64)a[1]<<16) + + ((sqlite3_int64)a[2]<< 8) + + ((sqlite3_int64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, +** using "..." with internal double-quote characters doubled. +*/ +static void shellIdQuote( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName = (const char*)sqlite3_value_text(argv[0]); + UNUSED_PARAMETER(argc); + if( zName ){ + char *z = sqlite3_mprintf("\"%w\"", zName); + sqlite3_result_text(context, z, -1, sqlite3_free); + } +} + +/* +** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. +*/ +static void shellUSleepFunc( + sqlite3_context *context, + int argcUnused, + sqlite3_value **argv +){ + int sleep = sqlite3_value_int(argv[0]); + (void)argcUnused; + sqlite3_sleep(sleep/1000); + sqlite3_result_int(context, sleep); +} + +/* +** Scalar function "shell_escape_crnl" used by the .recover command. +** The argument passed to this function is the output of built-in +** function quote(). If the first character of the input is "'", +** indicating that the value passed to quote() was a text value, +** then this function searches the input for "\n" and "\r" characters +** and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void shellEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + UNUSED_PARAMETER(argc); + if( zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + +/* Flags for open_db(). +** +** The default behavior of open_db() is to exit(1) if the database fails to +** open. The OPEN_DB_KEEPALIVE flag changes that so that it prints an error +** but still returns without calling exit. +** +** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a +** ZIP archive if the file does not exist or is empty and its name matches +** the *.zip pattern. +*/ +#define OPEN_DB_KEEPALIVE 0x001 /* Return after error if true */ +#define OPEN_DB_ZIPFILE 0x002 /* Open as ZIP if name matches *.zip */ + +/* +** Make sure the database is open. If it is not, then open it. If +** the database fails to open, print an error message and exit. +*/ +static void open_db(ShellState *p, int openFlags){ + if( p->db==0 ){ + if( p->openMode==SHELL_OPEN_UNSPEC ){ + if( p->zDbFilename==0 || p->zDbFilename[0]==0 ){ + p->openMode = SHELL_OPEN_NORMAL; + }else{ + p->openMode = (u8)deduceDatabaseType(p->zDbFilename, + (openFlags & OPEN_DB_ZIPFILE)!=0); + } + } + switch( p->openMode ){ + case SHELL_OPEN_APPENDVFS: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); + break; + } + case SHELL_OPEN_HEXDB: + case SHELL_OPEN_DESERIALIZE: { + sqlite3_open(0, &p->db); + break; + } + case SHELL_OPEN_ZIPFILE: { + sqlite3_open(":memory:", &p->db); + break; + } + case SHELL_OPEN_READONLY: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READONLY|p->openFlags, 0); + break; + } + case SHELL_OPEN_UNSPEC: + case SHELL_OPEN_NORMAL: { + sqlite3_open_v2(p->zDbFilename, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0); + break; + } + } + globalDb = p->db; + if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ + utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", + p->zDbFilename, sqlite3_errmsg(p->db)); + if( openFlags & OPEN_DB_KEEPALIVE ){ + sqlite3_open(":memory:", &p->db); + return; + } + exit(1); + } +#ifndef SQLITE_OMIT_LOAD_EXTENSION + sqlite3_enable_load_extension(p->db, 1); +#endif + sqlite3_fileio_init(p->db, 0, 0); + sqlite3_shathree_init(p->db, 0, 0); + sqlite3_completion_init(p->db, 0, 0); + sqlite3_uint_init(p->db, 0, 0); + sqlite3_decimal_init(p->db, 0, 0); + sqlite3_ieee_init(p->db, 0, 0); + sqlite3_series_init(p->db, 0, 0); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + sqlite3_dbdata_init(p->db, 0, 0); +#endif +#ifdef SQLITE_HAVE_ZLIB + sqlite3_zipfile_init(p->db, 0, 0); + sqlite3_sqlar_init(p->db, 0, 0); +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, + shellAddSchemaName, 0, 0); + sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, + shellModuleSchema, 0, 0); + sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, + shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, + shellEscapeCrnl, 0, 0); + sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, + shellInt32, 0, 0); + sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, + shellIdQuote, 0, 0); + sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, + shellUSleepFunc, 0, 0); +#ifndef SQLITE_NOHAVE_SYSTEM + sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, + editFunc, 0, 0); + sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, + editFunc, 0, 0); +#endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ + char *zSql = sqlite3_mprintf( + "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename); + sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +#ifdef SQLITE_ENABLE_DESERIALIZE + else + if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){ + int rc; + int nData = 0; + unsigned char *aData; + if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + aData = (unsigned char*)readFile(p->zDbFilename, &nData); + }else{ + aData = readHexDb(p, &nData); + if( aData==0 ){ + return; + } + } + rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + } + if( p->szMax>0 ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); + } + } +#endif + } +} + +/* +** Attempt to close the databaes connection. Report errors. +*/ +void close_db(sqlite3 *db){ + int rc = sqlite3_close(db); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", + rc, sqlite3_errmsg(db)); + } +} + +#if HAVE_READLINE || HAVE_EDITLINE +/* +** Readline completion callbacks +*/ +static char *readline_completion_generator(const char *text, int state){ + static sqlite3_stmt *pStmt = 0; + char *zRet; + if( state==0 ){ + char *zSql; + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" + " FROM completion(%Q) ORDER BY 1", text); + sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + zRet = strdup((const char*)sqlite3_column_text(pStmt, 0)); + }else{ + sqlite3_finalize(pStmt); + pStmt = 0; + zRet = 0; + } + return zRet; +} +static char **readline_completion(const char *zText, int iStart, int iEnd){ + rl_attempted_completion_over = 1; + return rl_completion_matches(zText, readline_completion_generator); +} + +#elif HAVE_LINENOISE +/* +** Linenoise completion callback +*/ +static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ + int nLine = strlen30(zLine); + int i, iStart; + sqlite3_stmt *pStmt = 0; + char *zSql; + char zBuf[1000]; + + if( nLine>sizeof(zBuf)-30 ) return; + if( zLine[0]=='.' || zLine[0]=='#') return; + for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} + if( i==nLine-1 ) return; + iStart = i+1; + memcpy(zBuf, zLine, iStart); + zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" + " FROM completion(%Q,%Q) ORDER BY 1", + &zLine[iStart], zLine); + sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + sqlite3_exec(globalDb, "PRAGMA page_count", 0, 0, 0); /* Load the schema */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCompletion = (const char*)sqlite3_column_text(pStmt, 0); + int nCompletion = sqlite3_column_bytes(pStmt, 0); + if( iStart+nCompletion < sizeof(zBuf)-1 ){ + memcpy(zBuf+iStart, zCompletion, nCompletion+1); + linenoiseAddCompletion(lc, zBuf); + } + } + sqlite3_finalize(pStmt); +} +#endif + +/* +** Do C-language style dequoting. +** +** \a -> alarm +** \b -> backspace +** \t -> tab +** \n -> newline +** \v -> vertical tab +** \f -> form feed +** \r -> carriage return +** \s -> space +** \" -> " +** \' -> ' +** \\ -> backslash +** \NNN -> ascii character NNN in octal +*/ +static void resolve_backslashes(char *z){ + int i, j; + char c; + while( *z && *z!='\\' ) z++; + for(i=j=0; (c = z[i])!=0; i++, j++){ + if( c=='\\' && z[i+1]!=0 ){ + c = z[++i]; + if( c=='a' ){ + c = '\a'; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='"' ){ + c = '"'; + }else if( c=='\'' ){ + c = '\''; + }else if( c=='\\' ){ + c = '\\'; + }else if( c>='0' && c<='7' ){ + c -= '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + } + } + } + } + z[j] = c; + } + if( j=0; i++){} + }else{ + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + } + if( i>0 && zArg[i]==0 ) return (int)(integerValue(zArg) & 0xffffffff); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; + } + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", + zArg); + return 0; +} + +/* +** Set or clear a shell flag according to a boolean value. +*/ +static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ + if( booleanValue(zArg) ){ + ShellSetFlag(p, mFlag); + }else{ + ShellClearFlag(p, mFlag); + } +} + +/* +** Close an output file, assuming it is not stderr or stdout +*/ +static void output_file_close(FILE *f){ + if( f && f!=stdout && f!=stderr ) fclose(f); +} + +/* +** Try to open an output file. The names "stdout" and "stderr" are +** recognized and do the right thing. NULL is returned if the output +** filename is "off". +*/ +static FILE *output_file_open(const char *zFile, int bTextMode){ + FILE *f; + if( strcmp(zFile,"stdout")==0 ){ + f = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + f = stderr; + }else if( strcmp(zFile, "off")==0 ){ + f = 0; + }else{ + f = fopen(zFile, bTextMode ? "w" : "wb"); + if( f==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } + return f; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** A routine for handling output from sqlite3_trace(). +*/ +static int sql_trace_callback( + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ +){ + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } + } + return 0; +} +#endif + +/* +** A no-op routine that runs with the ".breakpoint" doc-command. This is +** a useful spot to set a debugger breakpoint. +*/ +static void test_breakpoint(void){ + static int nCall = 0; + nCall++; +} + +/* +** An object used to read a CSV and other files for import. +*/ +typedef struct ImportCtx ImportCtx; +struct ImportCtx { + const char *zFile; /* Name of the input file */ + FILE *in; /* Read the CSV text from this input stream */ + int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close in */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + int nRow; /* Number of rows imported */ + int nErr; /* Number of errors encountered */ + int bNotFirst; /* True if one or more bytes already read */ + int cTerm; /* Character that terminated the most recent field */ + int cColSep; /* The column separator character. (Usually ",") */ + int cRowSep; /* The row separator character. (Usually "\n") */ +}; + +/* Clean up resourced used by an ImportCtx */ +static void import_cleanup(ImportCtx *p){ + if( p->in!=0 && p->xCloser!=0 ){ + p->xCloser(p->in); + p->in = 0; + } + sqlite3_free(p->z); + p->z = 0; +} + +/* Append a single byte to z[] */ +static void import_append_char(ImportCtx *p, int c){ + if( p->n+1>=p->nAlloc ){ + p->nAlloc += p->nAlloc + 100; + p->z = sqlite3_realloc64(p->z, p->nAlloc); + if( p->z==0 ) shell_out_of_memory(); + } + p->z[p->n++] = (char)c; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Use p->cSep as the column separator. The default is ",". +** + Use p->rSep as the row separator. The default is "\n". +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** + Report syntax errors on stderr +*/ +static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ + int c; + int cSep = p->cColSep; + int rSep = p->cRowSep; + p->n = 0; + c = fgetc(p->in); + if( c==EOF || seenInterrupt ){ + p->cTerm = EOF; + return 0; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + int cQuote = c; + pc = ppc = 0; + while( 1 ){ + c = fgetc(p->in); + if( c==rSep ) p->nLine++; + if( c==cQuote ){ + if( pc==cQuote ){ + pc = 0; + continue; + } + } + if( (c==cSep && pc==cQuote) + || (c==rSep && pc==cQuote) + || (c==rSep && pc=='\r' && ppc==cQuote) + || (c==EOF && pc==cQuote) + ){ + do{ p->n--; }while( p->z[p->n]!=cQuote ); + p->cTerm = c; + break; + } + if( pc==cQuote && c!='\r' ){ + utf8_printf(stderr, "%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); + } + if( c==EOF ){ + utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n", + p->zFile, startLine, cQuote); + p->cTerm = c; + break; + } + import_append_char(p, c); + ppc = pc; + pc = c; + } + }else{ + /* If this is the first field being parsed and it begins with the + ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + if( (c&0xff)==0xef && p->bNotFirst==0 ){ + import_append_char(p, c); + c = fgetc(p->in); + if( (c&0xff)==0xbb ){ + import_append_char(p, c); + c = fgetc(p->in); + if( (c&0xff)==0xbf ){ + p->bNotFirst = 1; + p->n = 0; + return csv_read_one_field(p); + } + } + } + while( c!=EOF && c!=cSep && c!=rSep ){ + import_append_char(p, c); + c = fgetc(p->in); + } + if( c==rSep ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = c; + } + if( p->z ) p->z[p->n] = 0; + p->bNotFirst = 1; + return p->z; +} + +/* Read a single field of ASCII delimited text. +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Use p->cSep as the column separator. The default is "\x1F". +** + Use p->rSep as the row separator. The default is "\x1E". +** + Keep track of the row number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** + Report syntax errors on stderr +*/ +static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ + int c; + int cSep = p->cColSep; + int rSep = p->cRowSep; + p->n = 0; + c = fgetc(p->in); + if( c==EOF || seenInterrupt ){ + p->cTerm = EOF; + return 0; + } + while( c!=EOF && c!=cSep && c!=rSep ){ + import_append_char(p, c); + c = fgetc(p->in); + } + if( c==rSep ){ + p->nLine++; + } + p->cTerm = c; + if( p->z ) p->z[p->n] = 0; + return p->z; +} + +/* +** Try to transfer data for table zTable. If an error is seen while +** moving forward, try to go backwards. The backwards movement won't +** work for WITHOUT ROWID tables. +*/ +static void tryToCloneData( + ShellState *p, + sqlite3 *newDb, + const char *zTable +){ + sqlite3_stmt *pQuery = 0; + sqlite3_stmt *pInsert = 0; + char *zQuery = 0; + char *zInsert = 0; + int rc; + int i, j, n; + int nTable = strlen30(zTable); + int k = 0; + int cnt = 0; + const int spinRate = 10000; + + zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error %d: %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_data_xfer; + } + n = sqlite3_column_count(pQuery); + zInsert = sqlite3_malloc64(200 + nTable + n*3); + if( zInsert==0 ) shell_out_of_memory(); + sqlite3_snprintf(200+nTable,zInsert, + "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable); + i = strlen30(zInsert); + for(j=1; jdb, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable); + break; + } + } /* End for(k=0...) */ + +end_data_xfer: + sqlite3_finalize(pQuery); + sqlite3_finalize(pInsert); + sqlite3_free(zQuery); + sqlite3_free(zInsert); +} + + +/* +** Try to transfer all rows of the schema that match zWhere. For +** each row, invoke xForEach() on the object defined by that row. +** If an error is encountered while moving forward through the +** sqlite_schema table, try again moving backwards. +*/ +static void tryToCloneSchema( + ShellState *p, + sqlite3 *newDb, + const char *zWhere, + void (*xForEach)(ShellState*,sqlite3*,const char*) +){ + sqlite3_stmt *pQuery = 0; + char *zQuery = 0; + int rc; + const unsigned char *zName; + const unsigned char *zSql; + char *zErrMsg = 0; + + zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" + " WHERE %s", zWhere); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_schema_xfer; + } + while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ + zName = sqlite3_column_text(pQuery, 0); + zSql = sqlite3_column_text(pQuery, 1); + printf("%s... ", zName); fflush(stdout); + sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_free(zErrMsg); + zErrMsg = 0; + } + if( xForEach ){ + xForEach(p, newDb, (const char*)zName); + } + printf("done\n"); + } + if( rc!=SQLITE_DONE ){ + sqlite3_finalize(pQuery); + sqlite3_free(zQuery); + zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" + " WHERE %s ORDER BY rowid DESC", zWhere); + rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); + if( rc ){ + utf8_printf(stderr, "Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), + zQuery); + goto end_schema_xfer; + } + while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ + zName = sqlite3_column_text(pQuery, 0); + zSql = sqlite3_column_text(pQuery, 1); + printf("%s... ", zName); fflush(stdout); + sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_free(zErrMsg); + zErrMsg = 0; + } + if( xForEach ){ + xForEach(p, newDb, (const char*)zName); + } + printf("done\n"); + } + } +end_schema_xfer: + sqlite3_finalize(pQuery); + sqlite3_free(zQuery); +} + +/* +** Open a new database file named "zNewDb". Try to recover as much information +** as possible out of the main database (which might be corrupt) and write it +** into zNewDb. +*/ +static void tryToClone(ShellState *p, const char *zNewDb){ + int rc; + sqlite3 *newDb = 0; + if( access(zNewDb,0)==0 ){ + utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb); + return; + } + rc = sqlite3_open(zNewDb, &newDb); + if( rc ){ + utf8_printf(stderr, "Cannot create output database: %s\n", + sqlite3_errmsg(newDb)); + }else{ + sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); + sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0); + tryToCloneSchema(p, newDb, "type='table'", tryToCloneData); + tryToCloneSchema(p, newDb, "type!='table'", 0); + sqlite3_exec(newDb, "COMMIT;", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); + } + close_db(newDb); +} + +/* +** Change the output file back to stdout. +** +** If the p->doXdgOpen flag is set, that means the output was being +** redirected to a temporary file named by p->zTempFile. In that case, +** launch start/open/xdg-open on that temporary file. +*/ +static void output_reset(ShellState *p){ + if( p->outfile[0]=='|' ){ +#ifndef SQLITE_OMIT_POPEN + pclose(p->out); +#endif + }else{ + output_file_close(p->out); +#ifndef SQLITE_NOHAVE_SYSTEM + if( p->doXdgOpen ){ + const char *zXdgOpenCmd = +#if defined(_WIN32) + "start"; +#elif defined(__APPLE__) + "open"; +#else + "xdg-open"; +#endif + char *zCmd; + zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); + if( system(zCmd) ){ + utf8_printf(stderr, "Failed: [%s]\n", zCmd); + }else{ + /* Give the start/open/xdg-open command some time to get + ** going before we continue, and potential delete the + ** p->zTempFile data file out from under it */ + sqlite3_sleep(2000); + } + sqlite3_free(zCmd); + outputModePop(p); + p->doXdgOpen = 0; + } +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ + } + p->outfile[0] = 0; + p->out = stdout; +} + +/* +** Run an SQL command and return the single integer result. +*/ +static int db_int(ShellState *p, const char *zSql){ + sqlite3_stmt *pStmt; + int res = 0; + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + res = sqlite3_column_int(pStmt,0); + } + sqlite3_finalize(pStmt); + return res; +} + +/* +** Convert a 2-byte or 4-byte big-endian integer into a native integer +*/ +static unsigned int get2byteInt(unsigned char *a){ + return (a[0]<<8) + a[1]; +} +static unsigned int get4byteInt(unsigned char *a){ + return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; +} + +/* +** Implementation of the ".dbinfo" command. +** +** Return 1 on error, 2 to exit, and 0 otherwise. +*/ +static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ + static const struct { const char *zName; int ofst; } aField[] = { + { "file change counter:", 24 }, + { "database page count:", 28 }, + { "freelist page count:", 36 }, + { "schema cookie:", 40 }, + { "schema format:", 44 }, + { "default cache size:", 48 }, + { "autovacuum top root:", 52 }, + { "incremental vacuum:", 64 }, + { "text encoding:", 56 }, + { "user version:", 60 }, + { "application id:", 68 }, + { "software version:", 96 }, + }; + static const struct { const char *zName; const char *zSql; } aQuery[] = { + { "number of tables:", + "SELECT count(*) FROM %s WHERE type='table'" }, + { "number of indexes:", + "SELECT count(*) FROM %s WHERE type='index'" }, + { "number of triggers:", + "SELECT count(*) FROM %s WHERE type='trigger'" }, + { "number of views:", + "SELECT count(*) FROM %s WHERE type='view'" }, + { "schema size:", + "SELECT total(length(sql)) FROM %s" }, + }; + int i, rc; + unsigned iDataVersion; + char *zSchemaTab; + char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; + unsigned char aHdr[100]; + open_db(p, 0); + if( p->db==0 ) return 1; + rc = sqlite3_prepare_v2(p->db, + "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + return 1; + } + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ + raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); + return 1; + } + i = get2byteInt(aHdr+16); + if( i==1 ) i = 65536; + utf8_printf(p->out, "%-20s %d\n", "database page size:", i); + utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + for(i=0; iout, "%-20s %u", aField[i].zName, val); + switch( ofst ){ + case 56: { + if( val==1 ) raw_printf(p->out, " (utf8)"); + if( val==2 ) raw_printf(p->out, " (utf16le)"); + if( val==3 ) raw_printf(p->out, " (utf16be)"); + } + } + raw_printf(p->out, "\n"); + } + if( zDb==0 ){ + zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); + }else if( strcmp(zDb,"temp")==0 ){ + zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema"); + }else{ + zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb); + } + for(i=0; iout, "%-20s %d\n", aQuery[i].zName, val); + } + sqlite3_free(zSchemaTab); + sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); + utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); + return 0; +} + +/* +** Print the current sqlite3_errmsg() value to stderr and return 1. +*/ +static int shellDatabaseError(sqlite3 *db){ + const char *zErr = sqlite3_errmsg(db); + utf8_printf(stderr, "Error: %s\n", zErr); + return 1; +} + +/* +** Compare the pattern in zGlob[] against the text in z[]. Return TRUE +** if they match and FALSE (0) if they do not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +** +** ' ' Any span of whitespace matches any other span of +** whitespace. +** +** Extra whitespace at the end of z[] is ignored. +*/ +static int testcase_glob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( IsSpace(c) ){ + if( !IsSpace(*z) ) return 0; + while( IsSpace(*zGlob) ) zGlob++; + while( IsSpace(*z) ) z++; + }else if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && testcase_glob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( testcase_glob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++; + if( !IsDigit(z[0]) ) return 0; + z++; + while( IsDigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + while( IsSpace(*z) ){ z++; } + return *z==0; +} + + +/* +** Compare the string as a command-line option with either one or two +** initial "-" characters. +*/ +static int optionMatch(const char *zStr, const char *zOpt){ + if( zStr[0]!='-' ) return 0; + zStr++; + if( zStr[0]=='-' ) zStr++; + return strcmp(zStr, zOpt)==0; +} + +/* +** Delete a file. +*/ +int shellDeleteFile(const char *zFilename){ + int rc; +#ifdef _WIN32 + wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename); + rc = _wunlink(z); + sqlite3_free(z); +#else + rc = unlink(zFilename); +#endif + return rc; +} + +/* +** Try to delete the temporary file (if there is one) and free the +** memory used to hold the name of the temp file. +*/ +static void clearTempFile(ShellState *p){ + if( p->zTempFile==0 ) return; + if( p->doXdgOpen ) return; + if( shellDeleteFile(p->zTempFile) ) return; + sqlite3_free(p->zTempFile); + p->zTempFile = 0; +} + +/* +** Create a new temp file name with the given suffix. +*/ +static void newTempFile(ShellState *p, const char *zSuffix){ + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + if( p->db ){ + sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); + } + if( p->zTempFile==0 ){ + /* If p->db is an in-memory database then the TEMPFILENAME file-control + ** will not work and we will need to fallback to guessing */ + char *zTemp; + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r), &r); + zTemp = getenv("TEMP"); + if( zTemp==0 ) zTemp = getenv("TMP"); + if( zTemp==0 ){ +#ifdef _WIN32 + zTemp = "\\tmp"; +#else + zTemp = "/tmp"; +#endif + } + p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); + }else{ + p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + } + if( p->zTempFile==0 ){ + raw_printf(stderr, "out of memory\n"); + exit(1); + } +} + + +/* +** The implementation of SQL scalar function fkey_collate_clause(), used +** by the ".lint fkey-indexes" command. This scalar function is always +** called with four arguments - the parent table name, the parent column name, +** the child table name and the child column name. +** +** fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col') +** +** If either of the named tables or columns do not exist, this function +** returns an empty string. An empty string is also returned if both tables +** and columns exist but have the same default collation sequence. Or, +** if both exist but the default collation sequences are different, this +** function returns the string " COLLATE ", where +** is the default collation sequence of the parent column. +*/ +static void shellFkeyCollateClause( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zParent; + const char *zParentCol; + const char *zParentSeq; + const char *zChild; + const char *zChildCol; + const char *zChildSeq = 0; /* Initialize to avoid false-positive warning */ + int rc; + + assert( nVal==4 ); + zParent = (const char*)sqlite3_value_text(apVal[0]); + zParentCol = (const char*)sqlite3_value_text(apVal[1]); + zChild = (const char*)sqlite3_value_text(apVal[2]); + zChildCol = (const char*)sqlite3_value_text(apVal[3]); + + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = sqlite3_table_column_metadata( + db, "main", zParent, zParentCol, 0, &zParentSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_table_column_metadata( + db, "main", zChild, zChildCol, 0, &zChildSeq, 0, 0, 0 + ); + } + + if( rc==SQLITE_OK && sqlite3_stricmp(zParentSeq, zChildSeq) ){ + char *z = sqlite3_mprintf(" COLLATE %s", zParentSeq); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); + sqlite3_free(z); + } +} + + +/* +** The implementation of dot-command ".lint fkey-indexes". +*/ +static int lintFkeyIndexes( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + sqlite3 *db = pState->db; /* Database handle to query "main" db of */ + FILE *out = pState->out; /* Stream to write non-error output to */ + int bVerbose = 0; /* If -verbose is present */ + int bGroupByParent = 0; /* If -groupbyparent is present */ + int i; /* To iterate through azArg[] */ + const char *zIndent = ""; /* How much to indent CREATE INDEX by */ + int rc; /* Return code */ + sqlite3_stmt *pSql = 0; /* Compiled version of SQL statement below */ + + /* + ** This SELECT statement returns one row for each foreign key constraint + ** in the schema of the main database. The column values are: + ** + ** 0. The text of an SQL statement similar to: + ** + ** "EXPLAIN QUERY PLAN SELECT 1 FROM child_table WHERE child_key=?" + ** + ** This SELECT is similar to the one that the foreign keys implementation + ** needs to run internally on child tables. If there is an index that can + ** be used to optimize this query, then it can also be used by the FK + ** implementation to optimize DELETE or UPDATE statements on the parent + ** table. + ** + ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by + ** the EXPLAIN QUERY PLAN command matches this pattern, then the schema + ** contains an index that can be used to optimize the query. + ** + ** 2. Human readable text that describes the child table and columns. e.g. + ** + ** "child_table(child_key1, child_key2)" + ** + ** 3. Human readable text that describes the parent table and columns. e.g. + ** + ** "parent_table(parent_key1, parent_key2)" + ** + ** 4. A full CREATE INDEX statement for an index that could be used to + ** optimize DELETE or UPDATE statements on the parent table. e.g. + ** + ** "CREATE INDEX child_table_child_key ON child_table(child_key)" + ** + ** 5. The name of the parent table. + ** + ** These six values are used by the C logic below to generate the report. + */ + const char *zSql = + "SELECT " + " 'EXPLAIN QUERY PLAN SELECT 1 FROM ' || quote(s.name) || ' WHERE '" + " || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' " + " || fkey_collate_clause(" + " f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')" + ", " + " 'SEARCH TABLE ' || s.name || ' USING COVERING INDEX*('" + " || group_concat('*=?', ' AND ') || ')'" + ", " + " s.name || '(' || group_concat(f.[from], ', ') || ')'" + ", " + " f.[table] || '(' || group_concat(COALESCE(f.[to], p.[name])) || ')'" + ", " + " 'CREATE INDEX ' || quote(s.name ||'_'|| group_concat(f.[from], '_'))" + " || ' ON ' || quote(s.name) || '('" + " || group_concat(quote(f.[from]) ||" + " fkey_collate_clause(" + " f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]), ', ')" + " || ');'" + ", " + " f.[table] " + "FROM sqlite_schema AS s, pragma_foreign_key_list(s.name) AS f " + "LEFT JOIN pragma_table_info AS p ON (pk-1=seq AND p.arg=f.[table]) " + "GROUP BY s.name, f.id " + "ORDER BY (CASE WHEN ? THEN f.[table] ELSE s.name END)" + ; + const char *zGlobIPK = "SEARCH TABLE * USING INTEGER PRIMARY KEY (rowid=?)"; + + for(i=2; i1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){ + bVerbose = 1; + } + else if( n>1 && sqlite3_strnicmp("-groupbyparent", azArg[i], n)==0 ){ + bGroupByParent = 1; + zIndent = " "; + } + else{ + raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", + azArg[0], azArg[1] + ); + return SQLITE_ERROR; + } + } + + /* Register the fkey_collate_clause() SQL function */ + rc = sqlite3_create_function(db, "fkey_collate_clause", 4, SQLITE_UTF8, + 0, shellFkeyCollateClause, 0, 0 + ); + + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pSql, 1, bGroupByParent); + } + + if( rc==SQLITE_OK ){ + int rc2; + char *zPrev = 0; + while( SQLITE_ROW==sqlite3_step(pSql) ){ + int res = -1; + sqlite3_stmt *pExplain = 0; + const char *zEQP = (const char*)sqlite3_column_text(pSql, 0); + const char *zGlob = (const char*)sqlite3_column_text(pSql, 1); + const char *zFrom = (const char*)sqlite3_column_text(pSql, 2); + const char *zTarget = (const char*)sqlite3_column_text(pSql, 3); + const char *zCI = (const char*)sqlite3_column_text(pSql, 4); + const char *zParent = (const char*)sqlite3_column_text(pSql, 5); + + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc!=SQLITE_OK ) break; + if( SQLITE_ROW==sqlite3_step(pExplain) ){ + const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3); + res = ( + 0==sqlite3_strglob(zGlob, zPlan) + || 0==sqlite3_strglob(zGlobIPK, zPlan) + ); + } + rc = sqlite3_finalize(pExplain); + if( rc!=SQLITE_OK ) break; + + if( res<0 ){ + raw_printf(stderr, "Error: internal error"); + break; + }else{ + if( bGroupByParent + && (bVerbose || res==0) + && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) + ){ + raw_printf(out, "-- Parent table %s\n", zParent); + sqlite3_free(zPrev); + zPrev = sqlite3_mprintf("%s", zParent); + } + + if( res==0 ){ + raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + }else if( bVerbose ){ + raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n", + zIndent, zFrom, zTarget + ); + } + } + } + sqlite3_free(zPrev); + + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + rc2 = sqlite3_finalize(pSql); + if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ + rc = rc2; + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + }else{ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + return rc; +} + +/* +** Implementation of ".lint" dot command. +*/ +static int lintDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int n; + n = (nArg>=2 ? strlen30(azArg[1]) : 0); + if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage; + return lintFkeyIndexes(pState, azArg, nArg); + + usage: + raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]); + raw_printf(stderr, "Where sub-commands are:\n"); + raw_printf(stderr, " fkey-indexes\n"); + return SQLITE_ERROR; +} + +#if !defined SQLITE_OMIT_VIRTUALTABLE +static void shellPrepare( + sqlite3 *db, + int *pRc, + const char *zSql, + sqlite3_stmt **ppStmt +){ + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "sql error: %s (%d)\n", + sqlite3_errmsg(db), sqlite3_errcode(db) + ); + *pRc = rc; + } + } +} + +/* +** Create a prepared statement using printf-style arguments for the SQL. +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellPreparePrintf( + sqlite3 *db, + int *pRc, + sqlite3_stmt **ppStmt, + const char *zFmt, + ... +){ + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellPrepare(db, pRc, z, ppStmt); + sqlite3_free(z); + } + } +} + +/* Finalize the prepared statement created using shellPreparePrintf(). +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellFinalize( + int *pRc, + sqlite3_stmt *pStmt +){ + if( pStmt ){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ){ + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + } + *pRc = rc; + } + } +} + +/* Reset the prepared statement created using shellPreparePrintf(). +** +** This routine is could be marked "static". But it is not always used, +** depending on compile-time options. By omitting the "static", we avoid +** nuisance compiler warnings about "defined but not used". +*/ +void shellReset( + int *pRc, + sqlite3_stmt *pStmt +){ + int rc = sqlite3_reset(pStmt); + if( *pRc==SQLITE_OK ){ + if( rc!=SQLITE_OK ){ + sqlite3 *db = sqlite3_db_handle(pStmt); + raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + } + *pRc = rc; + } +} +#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +/****************************************************************************** +** The ".archive" or ".ar" command. +*/ +/* +** Structure representing a single ".ar" command. +*/ +typedef struct ArCommand ArCommand; +struct ArCommand { + u8 eCmd; /* An AR_CMD_* value */ + u8 bVerbose; /* True if --verbose */ + u8 bZip; /* True if the archive is a ZIP */ + u8 bDryRun; /* True if --dry-run */ + u8 bAppend; /* True if --append */ + u8 fromCmdLine; /* Run from -A instead of .archive */ + int nArg; /* Number of command arguments */ + char *zSrcTable; /* "sqlar", "zipfile($file)" or "zip" */ + const char *zFile; /* --file argument, or NULL */ + const char *zDir; /* --directory argument, or NULL */ + char **azArg; /* Array of command arguments */ + ShellState *p; /* Shell state */ + sqlite3 *db; /* Database containing the archive */ +}; + +/* +** Print a usage message for the .ar command to stderr and return SQLITE_ERROR. +*/ +static int arUsage(FILE *f){ + showHelp(f,"archive"); + return SQLITE_ERROR; +} + +/* +** Print an error message for the .ar command to stderr and return +** SQLITE_ERROR. +*/ +static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + utf8_printf(stderr, "Error: %s\n", z); + if( pAr->fromCmdLine ){ + utf8_printf(stderr, "Use \"-A\" for more help\n"); + }else{ + utf8_printf(stderr, "Use \".archive --help\" for more help\n"); + } + sqlite3_free(z); + return SQLITE_ERROR; +} + +/* +** Values for ArCommand.eCmd. +*/ +#define AR_CMD_CREATE 1 +#define AR_CMD_UPDATE 2 +#define AR_CMD_INSERT 3 +#define AR_CMD_EXTRACT 4 +#define AR_CMD_LIST 5 +#define AR_CMD_HELP 6 + +/* +** Other (non-command) switches. +*/ +#define AR_SWITCH_VERBOSE 7 +#define AR_SWITCH_FILE 8 +#define AR_SWITCH_DIRECTORY 9 +#define AR_SWITCH_APPEND 10 +#define AR_SWITCH_DRYRUN 11 + +static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ + switch( eSwitch ){ + case AR_CMD_CREATE: + case AR_CMD_EXTRACT: + case AR_CMD_LIST: + case AR_CMD_UPDATE: + case AR_CMD_INSERT: + case AR_CMD_HELP: + if( pAr->eCmd ){ + return arErrorMsg(pAr, "multiple command options"); + } + pAr->eCmd = eSwitch; + break; + + case AR_SWITCH_DRYRUN: + pAr->bDryRun = 1; + break; + case AR_SWITCH_VERBOSE: + pAr->bVerbose = 1; + break; + case AR_SWITCH_APPEND: + pAr->bAppend = 1; + /* Fall thru into --file */ + case AR_SWITCH_FILE: + pAr->zFile = zArg; + break; + case AR_SWITCH_DIRECTORY: + pAr->zDir = zArg; + break; + } + + return SQLITE_OK; +} + +/* +** Parse the command line for an ".ar" command. The results are written into +** structure (*pAr). SQLITE_OK is returned if the command line is parsed +** successfully, otherwise an error message is written to stderr and +** SQLITE_ERROR returned. +*/ +static int arParseCommand( + char **azArg, /* Array of arguments passed to dot command */ + int nArg, /* Number of entries in azArg[] */ + ArCommand *pAr /* Populate this object */ +){ + struct ArSwitch { + const char *zLong; + char cShort; + u8 eSwitch; + u8 bArg; + } aSwitch[] = { + { "create", 'c', AR_CMD_CREATE, 0 }, + { "extract", 'x', AR_CMD_EXTRACT, 0 }, + { "insert", 'i', AR_CMD_INSERT, 0 }, + { "list", 't', AR_CMD_LIST, 0 }, + { "update", 'u', AR_CMD_UPDATE, 0 }, + { "help", 'h', AR_CMD_HELP, 0 }, + { "verbose", 'v', AR_SWITCH_VERBOSE, 0 }, + { "file", 'f', AR_SWITCH_FILE, 1 }, + { "append", 'a', AR_SWITCH_APPEND, 1 }, + { "directory", 'C', AR_SWITCH_DIRECTORY, 1 }, + { "dryrun", 'n', AR_SWITCH_DRYRUN, 0 }, + }; + int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch); + struct ArSwitch *pEnd = &aSwitch[nSwitch]; + + if( nArg<=1 ){ + utf8_printf(stderr, "Wrong number of arguments. Usage:\n"); + return arUsage(stderr); + }else{ + char *z = azArg[1]; + if( z[0]!='-' ){ + /* Traditional style [tar] invocation */ + int i; + int iArg = 2; + for(i=0; z[i]; i++){ + const char *zArg = 0; + struct ArSwitch *pOpt; + for(pOpt=&aSwitch[0]; pOptcShort ) break; + } + if( pOpt==pEnd ){ + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); + } + if( pOpt->bArg ){ + if( iArg>=nArg ){ + return arErrorMsg(pAr, "option requires an argument: %c",z[i]); + } + zArg = azArg[iArg++]; + } + if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR; + } + pAr->nArg = nArg-iArg; + if( pAr->nArg>0 ){ + pAr->azArg = &azArg[iArg]; + } + }else{ + /* Non-traditional invocation */ + int iArg; + for(iArg=1; iArgazArg = &azArg[iArg]; + pAr->nArg = nArg-iArg; + break; + } + n = strlen30(z); + + if( z[1]!='-' ){ + int i; + /* One or more short options */ + for(i=1; icShort ) break; + } + if( pOpt==pEnd ){ + return arErrorMsg(pAr, "unrecognized option: %c", z[i]); + } + if( pOpt->bArg ){ + if( i<(n-1) ){ + zArg = &z[i+1]; + i = n; + }else{ + if( iArg>=(nArg-1) ){ + return arErrorMsg(pAr, "option requires an argument: %c", + z[i]); + } + zArg = azArg[++iArg]; + } + } + if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR; + } + }else if( z[2]=='\0' ){ + /* A -- option, indicating that all remaining command line words + ** are command arguments. */ + pAr->azArg = &azArg[iArg+1]; + pAr->nArg = nArg-iArg-1; + break; + }else{ + /* A long option */ + const char *zArg = 0; /* Argument for option, if any */ + struct ArSwitch *pMatch = 0; /* Matching option */ + struct ArSwitch *pOpt; /* Iterator */ + for(pOpt=&aSwitch[0]; pOptzLong; + if( (n-2)<=strlen30(zLong) && 0==memcmp(&z[2], zLong, n-2) ){ + if( pMatch ){ + return arErrorMsg(pAr, "ambiguous option: %s",z); + }else{ + pMatch = pOpt; + } + } + } + + if( pMatch==0 ){ + return arErrorMsg(pAr, "unrecognized option: %s", z); + } + if( pMatch->bArg ){ + if( iArg>=(nArg-1) ){ + return arErrorMsg(pAr, "option requires an argument: %s", z); + } + zArg = azArg[++iArg]; + } + if( arProcessSwitch(pAr, pMatch->eSwitch, zArg) ) return SQLITE_ERROR; + } + } + } + } + + return SQLITE_OK; +} + +/* +** This function assumes that all arguments within the ArCommand.azArg[] +** array refer to archive members, as for the --extract or --list commands. +** It checks that each of them are present. If any specified file is not +** present in the archive, an error is printed to stderr and an error +** code returned. Otherwise, if all specified arguments are present in +** the archive, SQLITE_OK is returned. +** +** This function strips any trailing '/' characters from each argument. +** This is consistent with the way the [tar] command seems to work on +** Linux. +*/ +static int arCheckEntries(ArCommand *pAr){ + int rc = SQLITE_OK; + if( pAr->nArg ){ + int i, j; + sqlite3_stmt *pTest = 0; + + shellPreparePrintf(pAr->db, &rc, &pTest, + "SELECT name FROM %s WHERE name=$name", + pAr->zSrcTable + ); + j = sqlite3_bind_parameter_index(pTest, "$name"); + for(i=0; inArg && rc==SQLITE_OK; i++){ + char *z = pAr->azArg[i]; + int n = strlen30(z); + int bOk = 0; + while( n>0 && z[n-1]=='/' ) n--; + z[n] = '\0'; + sqlite3_bind_text(pTest, j, z, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pTest) ){ + bOk = 1; + } + shellReset(&rc, pTest); + if( rc==SQLITE_OK && bOk==0 ){ + utf8_printf(stderr, "not found in archive: %s\n", z); + rc = SQLITE_ERROR; + } + } + shellFinalize(&rc, pTest); + } + return rc; +} + +/* +** Format a WHERE clause that can be used against the "sqlar" table to +** identify all archive members that match the command arguments held +** in (*pAr). Leave this WHERE clause in (*pzWhere) before returning. +** The caller is responsible for eventually calling sqlite3_free() on +** any non-NULL (*pzWhere) value. +*/ +static void arWhereClause( + int *pRc, + ArCommand *pAr, + char **pzWhere /* OUT: New WHERE clause */ +){ + char *zWhere = 0; + if( *pRc==SQLITE_OK ){ + if( pAr->nArg==0 ){ + zWhere = sqlite3_mprintf("1"); + }else{ + int i; + const char *zSep = ""; + for(i=0; inArg; i++){ + const char *z = pAr->azArg[i]; + zWhere = sqlite3_mprintf( + "%z%s name = '%q' OR substr(name,1,%d) = '%q/'", + zWhere, zSep, z, strlen30(z)+1, z + ); + if( zWhere==0 ){ + *pRc = SQLITE_NOMEM; + break; + } + zSep = " OR "; + } + } + } + *pzWhere = zWhere; +} + +/* +** Implementation of .ar "lisT" command. +*/ +static int arListCommand(ArCommand *pAr){ + const char *zSql = "SELECT %s FROM %s WHERE %s"; + const char *azCols[] = { + "name", + "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name" + }; + + char *zWhere = 0; + sqlite3_stmt *pSql = 0; + int rc; + + rc = arCheckEntries(pAr); + arWhereClause(&rc, pAr, &zWhere); + + shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], + pAr->zSrcTable, zWhere); + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( pAr->bVerbose ){ + utf8_printf(pAr->p->out, "%s % 10d %s %s\n", + sqlite3_column_text(pSql, 0), + sqlite3_column_int(pSql, 1), + sqlite3_column_text(pSql, 2), + sqlite3_column_text(pSql, 3) + ); + }else{ + utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + } + } + } + shellFinalize(&rc, pSql); + sqlite3_free(zWhere); + return rc; +} + + +/* +** Implementation of .ar "eXtract" command. +*/ +static int arExtractCommand(ArCommand *pAr){ + const char *zSql1 = + "SELECT " + " ($dir || name)," + " writefile(($dir || name), %s, mode, mtime) " + "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" + " AND name NOT GLOB '*..[/\\]*'"; + + const char *azExtraArg[] = { + "sqlar_uncompress(data, sz)", + "data" + }; + + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + char *zDir = 0; + char *zWhere = 0; + int i, j; + + /* If arguments are specified, check that they actually exist within + ** the archive before proceeding. And formulate a WHERE clause to + ** match them. */ + rc = arCheckEntries(pAr); + arWhereClause(&rc, pAr, &zWhere); + + if( rc==SQLITE_OK ){ + if( pAr->zDir ){ + zDir = sqlite3_mprintf("%s/", pAr->zDir); + }else{ + zDir = sqlite3_mprintf(""); + } + if( zDir==0 ) rc = SQLITE_NOMEM; + } + + shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, + azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere + ); + + if( rc==SQLITE_OK ){ + j = sqlite3_bind_parameter_index(pSql, "$dir"); + sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); + + /* Run the SELECT statement twice. The first time, writefile() is called + ** for all archive members that should be extracted. The second time, + ** only for the directories. This is because the timestamps for + ** extracted directories must be reset after they are populated (as + ** populating them changes the timestamp). */ + for(i=0; i<2; i++){ + j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); + sqlite3_bind_int(pSql, j, i); + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + } + } + } + shellReset(&rc, pSql); + } + shellFinalize(&rc, pSql); + } + + sqlite3_free(zDir); + sqlite3_free(zWhere); + return rc; +} + +/* +** Run the SQL statement in zSql. Or if doing a --dryrun, merely print it out. +*/ +static int arExecSql(ArCommand *pAr, const char *zSql){ + int rc; + if( pAr->bDryRun ){ + utf8_printf(pAr->p->out, "%s\n", zSql); + rc = SQLITE_OK; + }else{ + char *zErr = 0; + rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); + if( zErr ){ + utf8_printf(stdout, "ERROR: %s\n", zErr); + sqlite3_free(zErr); + } + } + return rc; +} + + +/* +** Implementation of .ar "create", "insert", and "update" commands. +** +** create -> Create a new SQL archive +** insert -> Insert or reinsert all files listed +** update -> Insert files that have changed or that were not +** previously in the archive +** +** Create the "sqlar" table in the database if it does not already exist. +** Then add each file in the azFile[] array to the archive. Directories +** are added recursively. If argument bVerbose is non-zero, a message is +** printed on stdout for each file archived. +** +** The create command is the same as update, except that it drops +** any existing "sqlar" table before beginning. The "insert" command +** always overwrites every file named on the command-line, where as +** "update" only overwrites if the size or mtime or mode has changed. +*/ +static int arCreateOrUpdateCommand( + ArCommand *pAr, /* Command arguments and options */ + int bUpdate, /* true for a --create. */ + int bOnlyIfChanged /* Only update if file has changed */ +){ + const char *zCreate = + "CREATE TABLE IF NOT EXISTS sqlar(\n" + " name TEXT PRIMARY KEY, -- name of the file\n" + " mode INT, -- access permissions\n" + " mtime INT, -- last modification time\n" + " sz INT, -- original file size\n" + " data BLOB -- compressed content\n" + ")"; + const char *zDrop = "DROP TABLE IF EXISTS sqlar"; + const char *zInsertFmt[2] = { + "REPLACE INTO %s(name,mode,mtime,sz,data)\n" + " SELECT\n" + " %s,\n" + " mode,\n" + " mtime,\n" + " CASE substr(lsmode(mode),1,1)\n" + " WHEN '-' THEN length(data)\n" + " WHEN 'd' THEN 0\n" + " ELSE -1 END,\n" + " sqlar_compress(data)\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + , + "REPLACE INTO %s(name,mode,mtime,data)\n" + " SELECT\n" + " %s,\n" + " mode,\n" + " mtime,\n" + " data\n" + " FROM fsdir(%Q,%Q) AS disk\n" + " WHERE lsmode(mode) NOT LIKE '?%%'%s;" + }; + int i; /* For iterating through azFile[] */ + int rc; /* Return code */ + const char *zTab = 0; /* SQL table into which to insert */ + char *zSql; + char zTemp[50]; + char *zExists = 0; + + arExecSql(pAr, "PRAGMA page_size=512"); + rc = arExecSql(pAr, "SAVEPOINT ar;"); + if( rc!=SQLITE_OK ) return rc; + zTemp[0] = 0; + if( pAr->bZip ){ + /* Initialize the zipfile virtual table, if necessary */ + if( pAr->zFile ){ + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r),&r); + sqlite3_snprintf(sizeof(zTemp),zTemp,"zip%016llx",r); + zTab = zTemp; + zSql = sqlite3_mprintf( + "CREATE VIRTUAL TABLE temp.%s USING zipfile(%Q)", + zTab, pAr->zFile + ); + rc = arExecSql(pAr, zSql); + sqlite3_free(zSql); + }else{ + zTab = "zip"; + } + }else{ + /* Initialize the table for an SQLAR */ + zTab = "sqlar"; + if( bUpdate==0 ){ + rc = arExecSql(pAr, zDrop); + if( rc!=SQLITE_OK ) goto end_ar_transaction; + } + rc = arExecSql(pAr, zCreate); + } + if( bOnlyIfChanged ){ + zExists = sqlite3_mprintf( + " AND NOT EXISTS(" + "SELECT 1 FROM %s AS mem" + " WHERE mem.name=disk.name" + " AND mem.mtime=disk.mtime" + " AND mem.mode=disk.mode)", zTab); + }else{ + zExists = sqlite3_mprintf(""); + } + if( zExists==0 ) rc = SQLITE_NOMEM; + for(i=0; inArg && rc==SQLITE_OK; i++){ + char *zSql2 = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab, + pAr->bVerbose ? "shell_putsnl(name)" : "name", + pAr->azArg[i], pAr->zDir, zExists); + rc = arExecSql(pAr, zSql2); + sqlite3_free(zSql2); + } +end_ar_transaction: + if( rc!=SQLITE_OK ){ + sqlite3_exec(pAr->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0); + }else{ + rc = arExecSql(pAr, "RELEASE ar;"); + if( pAr->bZip && pAr->zFile ){ + zSql = sqlite3_mprintf("DROP TABLE %s", zTemp); + arExecSql(pAr, zSql); + sqlite3_free(zSql); + } + } + sqlite3_free(zExists); + return rc; +} + +/* +** Implementation of ".ar" dot command. +*/ +static int arDotCommand( + ShellState *pState, /* Current shell tool state */ + int fromCmdLine, /* True if -A command-line option, not .ar cmd */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + ArCommand cmd; + int rc; + memset(&cmd, 0, sizeof(cmd)); + cmd.fromCmdLine = fromCmdLine; + rc = arParseCommand(azArg, nArg, &cmd); + if( rc==SQLITE_OK ){ + int eDbType = SHELL_OPEN_UNSPEC; + cmd.p = pState; + cmd.db = pState->db; + if( cmd.zFile ){ + eDbType = deduceDatabaseType(cmd.zFile, 1); + }else{ + eDbType = pState->openMode; + } + if( eDbType==SHELL_OPEN_ZIPFILE ){ + if( cmd.eCmd==AR_CMD_EXTRACT || cmd.eCmd==AR_CMD_LIST ){ + if( cmd.zFile==0 ){ + cmd.zSrcTable = sqlite3_mprintf("zip"); + }else{ + cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile); + } + } + cmd.bZip = 1; + }else if( cmd.zFile ){ + int flags; + if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS; + if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT + || cmd.eCmd==AR_CMD_UPDATE ){ + flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + }else{ + flags = SQLITE_OPEN_READONLY; + } + cmd.db = 0; + if( cmd.bDryRun ){ + utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile, + eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); + } + rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, + eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "cannot open file: %s (%s)\n", + cmd.zFile, sqlite3_errmsg(cmd.db) + ); + goto end_ar_command; + } + sqlite3_fileio_init(cmd.db, 0, 0); + sqlite3_sqlar_init(cmd.db, 0, 0); + sqlite3_create_function(cmd.db, "shell_putsnl", 1, SQLITE_UTF8, cmd.p, + shellPutsFunc, 0, 0); + + } + if( cmd.zSrcTable==0 && cmd.bZip==0 && cmd.eCmd!=AR_CMD_HELP ){ + if( cmd.eCmd!=AR_CMD_CREATE + && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) + ){ + utf8_printf(stderr, "database does not contain an 'sqlar' table\n"); + rc = SQLITE_ERROR; + goto end_ar_command; + } + cmd.zSrcTable = sqlite3_mprintf("sqlar"); + } + + switch( cmd.eCmd ){ + case AR_CMD_CREATE: + rc = arCreateOrUpdateCommand(&cmd, 0, 0); + break; + + case AR_CMD_EXTRACT: + rc = arExtractCommand(&cmd); + break; + + case AR_CMD_LIST: + rc = arListCommand(&cmd); + break; + + case AR_CMD_HELP: + arUsage(pState->out); + break; + + case AR_CMD_INSERT: + rc = arCreateOrUpdateCommand(&cmd, 1, 0); + break; + + default: + assert( cmd.eCmd==AR_CMD_UPDATE ); + rc = arCreateOrUpdateCommand(&cmd, 1, 1); + break; + } + } +end_ar_command: + if( cmd.db!=pState->db ){ + close_db(cmd.db); + } + sqlite3_free(cmd.zSrcTable); + + return rc; +} +/* End of the ".archive" or ".ar" command logic +*******************************************************************************/ +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +/* +** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, the SQL statement or statements in zSql are executed using +** database connection db and the error code written to *pRc before +** this function returns. +*/ +static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + char *zErr = 0; + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", zErr); + } + *pRc = rc; + } +} + +/* +** Like shellExec(), except that zFmt is a printf() style format string. +*/ +static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellExec(db, pRc, z); + } + sqlite3_free(z); + } +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, an attempt is made to allocate, zero and return a pointer +** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set +** to SQLITE_NOMEM and NULL returned. +*/ +static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, zFmt is treated as a printf() style string. The result of +** formatting it along with any trailing arguments is written into a +** buffer obtained from sqlite3_malloc(), and pointer to which is returned. +** It is the responsibility of the caller to eventually free this buffer +** using a call to sqlite3_free(). +** +** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL +** pointer returned. +*/ +static char *shellMPrintf(int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return z; +} + +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + char *zQuoted; /* Quoted version of table name */ + int nCol; /* Number of columns in table */ + char **azlCol; /* Array of column lists */ + int iPk; /* Index of IPK column */ +}; + +/* +** Free a RecoverTable object allocated by recoverFindTable() or +** recoverOrphanTable(). +*/ +static void recoverFreeTable(RecoverTable *pTab){ + if( pTab ){ + sqlite3_free(pTab->zQuoted); + if( pTab->azlCol ){ + int i; + for(i=0; i<=pTab->nCol; i++){ + sqlite3_free(pTab->azlCol[i]); + } + sqlite3_free(pTab->azlCol); + } + sqlite3_free(pTab); + } +} + +/* +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. +** Otherwise, it allocates and returns a RecoverTable object based on the +** final four arguments passed to this function. It is the responsibility +** of the caller to eventually free the returned object using +** recoverFreeTable(). +*/ +static RecoverTable *recoverNewTable( + int *pRc, /* IN/OUT: Error code */ + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + RecoverTable *pTab = 0; + + pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, + shellIdQuote, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk to the index + ** of the column, where columns are 0-numbered from left to right. + ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, + ** leave zPk as "_rowid_" and pTab->iPk at -2. */ + pTab->iPk = -2; + if( bIntkey ){ + shellPreparePrintf(dbtmp, &rc, &pPkFinder, + "SELECT cid, name FROM pragma_table_info(%Q) " + " WHERE pk=1 AND type='integer' COLLATE nocase" + " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" + , zName, zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ + pTab->iPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + } + + pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); + pTab->nCol = nSqlCol; + + if( bIntkey ){ + pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); + }else{ + pTab->azlCol[0] = shellMPrintf(&rc, ""); + } + i = 1; + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT %Q || group_concat(shell_idquote(name), ', ') " + " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " + "FROM pragma_table_info(%Q)", + bIntkey ? ", " : "", pTab->iPk, + bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", + zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zText = (const char*)sqlite3_column_text(pStmt, 0); + pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); + i++; + } + shellFinalize(&rc, pStmt); + + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; + if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ + recoverFreeTable(pTab); + pTab = 0; + } + return pTab; +} + +/* +** This function is called to search the schema recovered from the +** sqlite_schema table of the (possibly) corrupt database as part +** of a ".recover" command. Specifically, for a table with root page +** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the +** table must be a WITHOUT ROWID table, or if non-zero, not one of +** those. +** +** If a table is found, a (RecoverTable*) object is returned. Or, if +** no such table is found, but bIntkey is false and iRoot is the +** root page of an index in the recovered schema, then (*pbNoop) is +** set to true and NULL returned. Or, if there is no such table or +** index, NULL is returned and (*pbNoop) set to 0, indicating that +** the caller should write data to the orphans table. +*/ +static RecoverTable *recoverFindTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + int iRoot, /* Root page of table */ + int bIntkey, /* True for an intkey table */ + int nCol, /* Number of columns in table */ + int *pbNoop /* OUT: True if iRoot is root of index */ +){ + sqlite3_stmt *pStmt = 0; + RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); + break; + } + } + + shellFinalize(pRc, pStmt); + *pbNoop = bNoop; + return pRet; +} + +/* +** Return a RecoverTable object representing the orphans table. +*/ +static RecoverTable *recoverOrphanTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + const char *zLostAndFound, /* Base name for orphans table */ + int nCol /* Number of user data columns */ +){ + RecoverTable *pTab = 0; + if( nCol>=0 && *pRc==SQLITE_OK ){ + int i; + + /* This block determines the name of the orphan table. The prefered + ** name is zLostAndFound. But if that clashes with another name + ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 + ** and so on until a non-clashing name is found. */ + int iTab = 0; + char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); + sqlite3_stmt *pTest = 0; + shellPrepare(pState->db, pRc, + "SELECT 1 FROM recovery.schema WHERE name=?", &pTest + ); + if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ + shellReset(pRc, pTest); + sqlite3_free(zTab); + zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); + sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + } + shellFinalize(pRc, pTest); + + pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pTab ){ + pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); + pTab->nCol = nCol; + pTab->iPk = -2; + if( nCol>0 ){ + pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); + if( pTab->azlCol ){ + pTab->azlCol[nCol] = shellMPrintf(pRc, ""); + for(i=nCol-1; i>=0; i--){ + pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); + } + } + } + + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + }else{ + raw_printf(pState->out, + "CREATE TABLE %s(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted + ); + for(i=0; iout, ", c%d", i); + } + raw_printf(pState->out, ");\n"); + } + } + sqlite3_free(zTab); + } + return pTab; +} + +/* +** This function is called to recover data from the database. A script +** to construct a new database containing all recovered data is output +** on stream pState->out. +*/ +static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ + int rc = SQLITE_OK; + sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ + sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ + const char *zLostAndFound = "lost_and_found"; + int i; + int nOrphan = -1; + RecoverTable *pOrphan = 0; + + int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + int bRowids = 1; /* 0 if --no-rowids */ + for(i=1; iout, azArg[0]); + return 1; + } + } + + shellExecPrintf(pState->db, &rc, + /* Attach an in-memory database named 'recovery'. Create an indexed + ** cache of the sqlite_dbptr virtual table. */ + "PRAGMA writable_schema = on;" + "ATTACH %Q AS recovery;" + "DROP TABLE IF EXISTS recovery.dbptr;" + "DROP TABLE IF EXISTS recovery.freelist;" + "DROP TABLE IF EXISTS recovery.map;" + "DROP TABLE IF EXISTS recovery.schema;" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + ); + + if( bFreelist ){ + shellExec(pState->db, &rc, + "WITH trunk(pgno) AS (" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " + " WHERE x>0" + " UNION" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " + " FROM trunk WHERE x>0" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " + " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" + " UNION ALL" + " SELECT data, n-1, shell_int32(data, 2+n) " + " FROM freelist WHERE n>=0" + ")" + "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" + ); + } + + /* If this is an auto-vacuum database, add all pointer-map pages to + ** the freelist table. Do this regardless of whether or not + ** --freelist-corrupt was specified. */ + shellExec(pState->db, &rc, + "WITH ptrmap(pgno) AS (" + " SELECT 2 WHERE shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" + " )" + " UNION ALL " + " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " + " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" + ")" + "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" + ); + + shellExec(pState->db, &rc, + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr" + " WHERE pgno NOT IN freelist AND child NOT IN freelist;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + + /* Create the "map" table that will (eventually) contain instructions + ** for dealing with each page in the db that contains one or more + ** records. */ + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" + + /* Populate table [map]. If there are circular loops of pages in the + ** database, the following adds all pages in such a loop to the map + ** as individual root pages. This could be handled better. */ + "WITH pages(i, maxlen) AS (" + " SELECT page_count, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" + " ) FROM pragma_page_count WHERE page_count>0" + " UNION ALL" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" + ")" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" + " WITH p(orig, pgno, parent) AS (" + " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" + " UNION " + " SELECT i, p.parent, " + " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" + " )" + " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" + ") " + "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" + + /* Extract data from page 1 and any linked pages into table + ** recovery.schema. With the same schema as an sqlite_schema table. */ + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + "INSERT INTO recovery.schema SELECT " + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=1" + ")" + "GROUP BY pgno, cell;" + "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" + ); + + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + /* ".recover" might output content in an order which causes immediate + ** foreign key constraints to be violated. So disable foreign-key + ** constraint enforcement to prevent problems when running the output + ** script. */ + raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); + raw_printf(pState->out, "BEGIN;\n"); + raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' AND sql LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", + &zCreateTable[12] + ); + } + shellFinalize(&rc, pStmt); + } + + /* Figure out if an orphan table will be required. And if so, how many + ** user columns it should contain */ + shellPrepare(pState->db, &rc, + "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" + , &pLoop + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + nOrphan = sqlite3_column_int(pLoop, 0); + } + shellFinalize(&rc, pLoop); + pLoop = 0; + + shellPrepare(pState->db, &rc, + "SELECT pgno FROM recovery.map WHERE root=?", &pPages + ); + + shellPrepare(pState->db, &rc, + "SELECT max(field), group_concat(shell_escape_crnl(quote" + "(case when (? AND field<0) then NULL else value end)" + "), ', ')" + ", min(field) " + "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" + "GROUP BY cell", &pCells + ); + + /* Loop through each root page. */ + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" + " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" + ")", &pLoop + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + int iRoot = sqlite3_column_int(pLoop, 0); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); + int bNoop = 0; + RecoverTable *pTab; + + assert( bIntkey==0 || bIntkey==1 ); + pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + if( bNoop || rc ) continue; + if( pTab==0 ){ + if( pOrphan==0 ){ + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); + } + pTab = pOrphan; + if( pTab==0 ) break; + } + + if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } + sqlite3_bind_int(pPages, 1, iRoot); + if( bRowids==0 && pTab->iPk<0 ){ + sqlite3_bind_int(pCells, 1, 1); + }else{ + sqlite3_bind_int(pCells, 1, 0); + } + sqlite3_bind_int(pCells, 3, pTab->iPk); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + int iPgno = sqlite3_column_int(pPages, 0); + sqlite3_bind_int(pCells, 2, iPgno); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int nField = sqlite3_column_int(pCells, 0); + int iMin = sqlite3_column_int(pCells, 2); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + + RecoverTable *pTab2 = pTab; + if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ + if( pOrphan==0 ){ + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); + } + pTab2 = pOrphan; + if( pTab2==0 ) break; + } + + nField = nField+1; + if( pTab2==pOrphan ){ + raw_printf(pState->out, + "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", + pTab2->zQuoted, iRoot, iPgno, nField, + iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] + ); + }else{ + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab2->zQuoted, pTab2->azlCol[nField], zVal + ); + } + } + shellReset(&rc, pCells); + } + shellReset(&rc, pPages); + if( pTab!=pOrphan ) recoverFreeTable(pTab); + } + shellFinalize(&rc, pLoop); + shellFinalize(&rc, pPages); + shellFinalize(&rc, pCells); + recoverFreeTable(pOrphan); + + /* The rest of the schema */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + shellPrepare(pState->db, &rc, + "SELECT sql, name FROM recovery.schema " + "WHERE sql NOT LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + char *zPrint = shellMPrintf(&rc, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + ); + raw_printf(pState->out, "%s;\n", zPrint); + sqlite3_free(zPrint); + }else{ + raw_printf(pState->out, "%s;\n", zSql); + } + } + shellFinalize(&rc, pStmt); + } + + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); + raw_printf(pState->out, "COMMIT;\n"); + } + sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; +} +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + + +/* +** If an input line begins with "." then invoke this routine to +** process that line. +** +** Return 1 on error, 2 to exit, and 0 otherwise. +*/ +static int do_meta_command(char *zLine, ShellState *p){ + int h = 1; + int nArg = 0; + int n, c; + int rc = 0; + char *azArg[52]; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( p->expert.pExpert ){ + expertFinish(p, 1, 0); + } +#endif + + /* Parse the input line into tokens. + */ + while( zLine[h] && nArgdb, shellAuth, p); + }else{ + sqlite3_set_authorizer(p->db, 0, 0); + } + }else +#endif + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) + if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ + open_db(p, 0); + rc = arDotCommand(p, 0, azArg, nArg); + }else +#endif + + if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) + || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) + ){ + const char *zDestFile = 0; + const char *zDb = 0; + sqlite3 *pDest; + sqlite3_backup *pBackup; + int j; + int bAsync = 0; + const char *zVfs = 0; + for(j=1; jdb, zDb); + if( pBackup==0 ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + close_db(pDest); + return 1; + } + while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK ){} + sqlite3_backup_finish(pBackup); + if( rc==SQLITE_DONE ){ + rc = 0; + }else{ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + rc = 1; + } + close_db(pDest); + }else + + if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ + if( nArg==2 ){ + bail_on_error = booleanValue(azArg[1]); + }else{ + raw_printf(stderr, "Usage: .bail on|off\n"); + rc = 1; + } + }else + + if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + if( nArg==2 ){ + if( booleanValue(azArg[1]) ){ + setBinaryMode(p->out, 1); + }else{ + setTextMode(p->out, 1); + } + }else{ + raw_printf(stderr, "Usage: .binary on|off\n"); + rc = 1; + } + }else + + if( c=='c' && strcmp(azArg[0],"cd")==0 ){ + if( nArg==2 ){ +#if defined(_WIN32) || defined(WIN32) + wchar_t *z = sqlite3_win32_utf8_to_unicode(azArg[1]); + rc = !SetCurrentDirectoryW(z); + sqlite3_free(z); +#else + rc = chdir(azArg[1]); +#endif + if( rc ){ + utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]); + rc = 1; + } + }else{ + raw_printf(stderr, "Usage: .cd DIRECTORY\n"); + rc = 1; + } + }else + + /* The undocumented ".breakpoint" command causes a call to the no-op + ** routine named test_breakpoint(). + */ + if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + test_breakpoint(); + }else + + if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ + if( nArg==2 ){ + setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .changes on|off\n"); + rc = 1; + } + }else + + /* Cancel output redirection, if it is currently set (by .testcase) + ** Then read the content of the testcase-out.txt file and compare against + ** azArg[1]. If there are differences, report an error and exit. + */ + if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + char *zRes = 0; + output_reset(p); + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + rc = 2; + }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ + raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); + rc = 2; + }else if( testcase_glob(azArg[1],zRes)==0 ){ + utf8_printf(stderr, + "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); + rc = 1; + }else{ + utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + p->nCheck++; + } + sqlite3_free(zRes); + }else + + if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ + if( nArg==2 ){ + tryToClone(p, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .clone FILENAME\n"); + rc = 1; + } + }else + + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + char **azName = 0; + int nName = 0; + sqlite3_stmt *pStmt; + int i; + open_db(p, 0); + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + rc = 1; + }else{ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zSchema = (const char *)sqlite3_column_text(pStmt,1); + const char *zFile = (const char*)sqlite3_column_text(pStmt,2); + azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*)); + if( azName==0 ){ shell_out_of_memory(); /* Does not return */ } + azName[nName*2] = strdup(zSchema); + azName[nName*2+1] = strdup(zFile); + nName++; + } + } + sqlite3_finalize(pStmt); + for(i=0; idb, azName[i*2]); + int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); + const char *z = azName[i*2+1]; + utf8_printf(p->out, "%s: %s %s%s\n", + azName[i*2], + z && z[0] ? z : "\"\"", + bRdonly ? "r/o" : "r/w", + eTxn==SQLITE_TXN_NONE ? "" : + eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); + free(azName[i*2]); + free(azName[i*2+1]); + } + sqlite3_free(azName); + }else + + if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + static const struct DbConfigChoices { + const char *zName; + int op; + } aDbConfig[] = { + { "defensive", SQLITE_DBCONFIG_DEFENSIVE }, + { "dqs_ddl", SQLITE_DBCONFIG_DQS_DDL }, + { "dqs_dml", SQLITE_DBCONFIG_DQS_DML }, + { "enable_fkey", SQLITE_DBCONFIG_ENABLE_FKEY }, + { "enable_qpsg", SQLITE_DBCONFIG_ENABLE_QPSG }, + { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, + { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, + { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, + { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, + { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, + { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, + { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, + { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, + }; + int ii, v; + open_db(p, 0); + for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>=3 ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } + sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); + utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + if( nArg>1 ) break; + } + if( nArg>1 && ii==ArraySize(aDbConfig) ){ + utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); + utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); + } + }else + + if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ + rc = shell_dbinfo_command(p, nArg, azArg); + }else + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + open_db(p, 0); + rc = recoverDatabaseCmd(p, nArg, azArg); + }else +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + char *zLike = 0; + char *zSql; + int i; + int savedShowHeader = p->showHeader; + int savedShellFlags = p->shellFlgs; + ShellClearFlag(p, + SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo + |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + for(i=1; ishellFlgs & SHFLG_DumpDataOnly)==0 ){ + /* When playing back a "dump", the content might appear in an order + ** which causes immediate foreign key constraints to be violated. + ** So disable foreign-key constraint enforcement to prevent problems. */ + raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); + raw_printf(p->out, "BEGIN TRANSACTION;\n"); + } + p->writableSchema = 0; + p->showHeader = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_schema table is + ** corrupt. */ + sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); + p->nErr = 0; + if( zLike==0 ) zLike = sqlite3_mprintf("true"); + zSql = sqlite3_mprintf( + "SELECT name, type, sql FROM sqlite_schema " + "WHERE (%s) AND type=='table'" + " AND sql NOT NULL" + " ORDER BY tbl_name='sqlite_sequence', rowid", + zLike + ); + run_schema_dump_query(p,zSql); + sqlite3_free(zSql); + if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ + zSql = sqlite3_mprintf( + "SELECT sql FROM sqlite_schema " + "WHERE (%s) AND sql NOT NULL" + " AND type IN ('index','trigger','view')", + zLike + ); + run_table_dump_query(p, zSql); + sqlite3_free(zSql); + } + sqlite3_free(zLike); + if( p->writableSchema ){ + raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); + p->writableSchema = 0; + } + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); + if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ + raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); + } + p->showHeader = savedShowHeader; + p->shellFlgs = savedShellFlags; + }else + + if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ + if( nArg==2 ){ + setOrClearFlag(p, SHFLG_Echo, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .echo on|off\n"); + rc = 1; + } + }else + + if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ + if( nArg==2 ){ + p->autoEQPtest = 0; + if( p->autoEQPtrace ){ + if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); + p->autoEQPtrace = 0; + } + if( strcmp(azArg[1],"full")==0 ){ + p->autoEQP = AUTOEQP_full; + }else if( strcmp(azArg[1],"trigger")==0 ){ + p->autoEQP = AUTOEQP_trigger; +#ifdef SQLITE_DEBUG + }else if( strcmp(azArg[1],"test")==0 ){ + p->autoEQP = AUTOEQP_on; + p->autoEQPtest = 1; + }else if( strcmp(azArg[1],"trace")==0 ){ + p->autoEQP = AUTOEQP_full; + p->autoEQPtrace = 1; + open_db(p, 0); + sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); +#endif + }else{ + p->autoEQP = (u8)booleanValue(azArg[1]); + } + }else{ + raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n"); + rc = 1; + } + }else + + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); + rc = 2; + }else + + /* The ".explain" command is automatic now. It is largely pointless. It + ** retained purely for backwards compatibility */ + if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + int val = 1; + if( nArg>=2 ){ + if( strcmp(azArg[1],"auto")==0 ){ + val = 99; + }else{ + val = booleanValue(azArg[1]); + } + } + if( val==1 && p->mode!=MODE_Explain ){ + p->normalMode = p->mode; + p->mode = MODE_Explain; + p->autoExplain = 0; + }else if( val==0 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 0; + }else if( val==99 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 1; + } + }else + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){ + open_db(p, 0); + expertDotCommand(p, azArg, nArg); + }else +#endif + + if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ + static const struct { + const char *zCtrlName; /* Name of a test-control option */ + int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ + } aCtrl[] = { + { "chunk_size", SQLITE_FCNTL_CHUNK_SIZE, "SIZE" }, + { "data_version", SQLITE_FCNTL_DATA_VERSION, "" }, + { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, + { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, + { "persist_wal", SQLITE_FCNTL_PERSIST_WAL, "[BOOLEAN]" }, + /* { "pragma", SQLITE_FCNTL_PRAGMA, "NAME ARG" },*/ + { "psow", SQLITE_FCNTL_POWERSAFE_OVERWRITE, "[BOOLEAN]" }, + { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, + { "size_limit", SQLITE_FCNTL_SIZE_LIMIT, "[LIMIT]" }, + { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, + /* { "win32_av_retry", SQLITE_FCNTL_WIN32_AV_RETRY, "COUNT DELAY" },*/ + }; + int filectrl = -1; + int iCtrl = -1; + sqlite3_int64 iRes = 0; /* Integer result to display if rc2==1 */ + int isOk = 0; /* 0: usage 1: %lld 2: no-result */ + int n2, i; + const char *zCmd = 0; + const char *zSchema = 0; + + open_db(p, 0); + zCmd = nArg>=2 ? azArg[1] : "help"; + + if( zCmd[0]=='-' + && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && nArg>=4 + ){ + zSchema = azArg[2]; + for(i=3; iout, "Available file-controls:\n"); + for(i=0; iout, " .filectrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); + } + rc = 1; + goto meta_command_exit; + } + + /* convert filectrl text option to value. allow any unique prefix + ** of the option name, or a numerical value. */ + n2 = strlen30(zCmd); + for(i=0; idb, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes); + isOk = 1; + break; + } + case SQLITE_FCNTL_LOCK_TIMEOUT: + case SQLITE_FCNTL_CHUNK_SIZE: { + int x; + if( nArg!=3 ) break; + x = (int)integerValue(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + isOk = 2; + break; + } + case SQLITE_FCNTL_PERSIST_WAL: + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + int x; + if( nArg!=2 && nArg!=3 ) break; + x = nArg==3 ? booleanValue(azArg[2]) : -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + iRes = x; + isOk = 1; + break; + } + case SQLITE_FCNTL_DATA_VERSION: + case SQLITE_FCNTL_HAS_MOVED: { + int x; + if( nArg!=2 ) break; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + iRes = x; + isOk = 1; + break; + } + case SQLITE_FCNTL_TEMPFILENAME: { + char *z = 0; + if( nArg!=2 ) break; + sqlite3_file_control(p->db, zSchema, filectrl, &z); + if( z ){ + utf8_printf(p->out, "%s\n", z); + sqlite3_free(z); + } + isOk = 2; + break; + } + case SQLITE_FCNTL_RESERVE_BYTES: { + int x; + if( nArg>=3 ){ + x = atoi(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + } + x = -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + utf8_printf(p->out,"%d\n", x); + isOk = 2; + break; + } + } + } + if( isOk==0 && iCtrl>=0 ){ + utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + char zBuf[100]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); + raw_printf(p->out, "%s\n", zBuf); + } + }else + + if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + ShellState data; + char *zErrMsg = 0; + int doStats = 0; + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + if( nArg==2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg = 1; + } + if( nArg!=1 ){ + raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = sqlite3_exec(p->db, + "SELECT sql FROM" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " FROM sqlite_schema UNION ALL" + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " + "ORDER BY rowid", + callback, &data, &zErrMsg + ); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare_v2(p->db, + "SELECT rowid FROM sqlite_schema" + " WHERE name GLOB 'sqlite_stat[134]'", + -1, &pStmt, 0); + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } + if( doStats==0 ){ + raw_printf(p->out, "/* No STAT tables available */\n"); + }else{ + raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_schema'", + callback, &data, &zErrMsg); + data.cMode = data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", &zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", &zErrMsg); + raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + } + }else + + if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){ + if( nArg==2 ){ + p->showHeader = booleanValue(azArg[1]); + p->shellFlgs |= SHFLG_HeaderSet; + }else{ + raw_printf(stderr, "Usage: .headers on|off\n"); + rc = 1; + } + }else + + if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + if( nArg>=2 ){ + n = showHelp(p->out, azArg[1]); + if( n==0 ){ + utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + } + }else{ + showHelp(p->out, 0); + } + }else + + if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ + char *zTable = 0; /* Insert data into this table */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + int nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + int nSep; /* Number of bytes in p->colSeparator[] */ + char *zSql; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + int nSkip = 0; /* Initial lines to skip */ + int useOutputMode = 1; /* Use output mode to determine separators */ + + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + for(i=1; iout, "ERROR: extra argument: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + }else if( strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( strcmp(z,"-skip")==0 && iout, "ERROR: unknown option: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + } + if( zTable==0 ){ + utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + seenInterrupt = 0; + open_db(p, 0); + if( useOutputMode ){ + /* If neither the --csv or --ascii options are specified, then set + ** the column and row separator characters from the output mode. */ + nSep = strlen30(p->colSeparator); + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null column separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep>1 ){ + raw_printf(stderr, + "Error: multi-character column separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } + nSep = strlen30(p->rowSeparator); + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null row separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + /* When importing CSV (only), if the row separator is set to the + ** default output row separator, change it to the default input + ** row separator. This avoids having to maintain different input + ** and output row separators. */ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + nSep = strlen30(p->rowSeparator); + } + if( nSep>1 ){ + raw_printf(stderr, "Error: multi-character row separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } + sCtx.cColSep = p->colSeparator[0]; + sCtx.cRowSep = p->rowSeparator[0]; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + goto meta_command_exit; +#else + sCtx.in = popen(sCtx.zFile+1, "r"); + sCtx.zFile = ""; + sCtx.xCloser = pclose; +#endif + }else{ + sCtx.in = fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + rc = 1; + goto meta_command_exit; + } + if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + utf8_printf(p->out, "Column separator "); + output_c_string(p->out, zSep); + utf8_printf(p->out, ", row separator "); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + utf8_printf(p->out, "\n"); + } + while( (nSkip--)>0 ){ + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + zSql = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + nByte = strlen30(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + char *zCreate = sqlite3_mprintf("CREATE TABLE \"%w\"", zTable); + char cSep = '('; + while( xRead(&sCtx) ){ + zCreate = sqlite3_mprintf("%z%c\n \"%w\" TEXT", zCreate, cSep, sCtx.z); + cSep = ','; + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + if( cSep=='(' ){ + sqlite3_free(zCreate); + import_cleanup(&sCtx); + utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); + rc = 1; + goto meta_command_exit; + } + zCreate = sqlite3_mprintf("%z\n)", zCreate); + if( eVerbose>=1 ){ + utf8_printf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc ){ + utf8_printf(stderr, "CREATE TABLE \"%s\"(...) failed: %s\n", zTable, + sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + nCol = sqlite3_column_count(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + j = strlen30(zSql); + for(i=1; i=2 ){ + utf8_printf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; imode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, + startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); + + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + utf8_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + } + }else + +#ifndef SQLITE_UNTESTABLE + if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + char *zSql; + char *zCollist = 0; + sqlite3_stmt *pStmt; + int tnum = 0; + int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ + int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ + int i; + if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ + utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); + /* Also allowed, but not documented: + ** + ** .imposter TABLE IMPOSTER + ** + ** where TABLE is a WITHOUT ROWID table. In that case, the + ** imposter is another WITHOUT ROWID table with the columns in + ** storage order. */ + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( nArg==2 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + goto meta_command_exit; + } + zSql = sqlite3_mprintf( + "SELECT rootpage, 0 FROM sqlite_schema" + " WHERE name='%q' AND type='index'" + "UNION ALL " + "SELECT rootpage, 1 FROM sqlite_schema" + " WHERE name='%q' AND type='table'" + " AND sql LIKE '%%without%%rowid%%'", + azArg[1], azArg[1] + ); + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + tnum = sqlite3_column_int(pStmt, 0); + isWO = sqlite3_column_int(pStmt, 1); + } + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char zLabel[20]; + const char *zCol = (const char*)sqlite3_column_text(pStmt,2); + i++; + if( zCol==0 ){ + if( sqlite3_column_int(pStmt,1)==-1 ){ + zCol = "_ROWID_"; + }else{ + sqlite3_snprintf(sizeof(zLabel),zLabel,"expr%d",i); + zCol = zLabel; + } + } + if( isWO && lenPK==0 && sqlite3_column_int(pStmt,5)==0 && zCollist ){ + lenPK = (int)strlen(zCollist); + } + if( zCollist==0 ){ + zCollist = sqlite3_mprintf("\"%w\"", zCol); + }else{ + zCollist = sqlite3_mprintf("%z,\"%w\"", zCollist, zCol); + } + } + sqlite3_finalize(pStmt); + if( i==0 || tnum==0 ){ + utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); + rc = 1; + sqlite3_free(zCollist); + goto meta_command_exit; + } + if( lenPK==0 ) lenPK = 100000; + zSql = sqlite3_mprintf( + "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID", + azArg[2], zCollist, lenPK, zCollist); + sqlite3_free(zCollist); + rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + if( rc ){ + utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); + }else{ + utf8_printf(stdout, "%s;\n", zSql); + raw_printf(stdout, + "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n", + azArg[1], isWO ? "table" : "index" + ); + } + }else{ + raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + rc = 1; + } + sqlite3_free(zSql); + }else +#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + +#ifdef SQLITE_ENABLE_IOTRACE + if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); + if( iotrace && iotrace!=stdout ) fclose(iotrace); + iotrace = 0; + if( nArg<2 ){ + sqlite3IoTrace = 0; + }else if( strcmp(azArg[1], "-")==0 ){ + sqlite3IoTrace = iotracePrintf; + iotrace = stdout; + }else{ + iotrace = fopen(azArg[1], "w"); + if( iotrace==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + sqlite3IoTrace = 0; + rc = 1; + }else{ + sqlite3IoTrace = iotracePrintf; + } + } + }else +#endif + + if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + static const struct { + const char *zLimitName; /* Name of a limit */ + int limitCode; /* Integer code for that limit */ + } aLimit[] = { + { "length", SQLITE_LIMIT_LENGTH }, + { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, + { "column", SQLITE_LIMIT_COLUMN }, + { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, + { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, + { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, + { "attached", SQLITE_LIMIT_ATTACHED }, + { "like_pattern_length", SQLITE_LIMIT_LIKE_PATTERN_LENGTH }, + { "variable_number", SQLITE_LIMIT_VARIABLE_NUMBER }, + { "trigger_depth", SQLITE_LIMIT_TRIGGER_DEPTH }, + { "worker_threads", SQLITE_LIMIT_WORKER_THREADS }, + }; + int i, n2; + open_db(p, 0); + if( nArg==1 ){ + for(i=0; idb, aLimit[i].limitCode, -1)); + } + }else if( nArg>3 ){ + raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n"); + rc = 1; + goto meta_command_exit; + }else{ + int iLimit = -1; + n2 = strlen30(azArg[1]); + for(i=0; idb, aLimit[iLimit].limitCode, + (int)integerValue(azArg[2])); + } + printf("%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); + } + }else + + if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + open_db(p, 0); + lintDotCommand(p, azArg, nArg); + }else + +#ifndef SQLITE_OMIT_LOAD_EXTENSION + if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ + const char *zFile, *zProc; + char *zErrMsg = 0; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); + rc = 1; + goto meta_command_exit; + } + zFile = azArg[1]; + zProc = nArg>=3 ? azArg[2] : 0; + open_db(p, 0); + rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + rc = 1; + } + }else +#endif + + if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .log FILENAME\n"); + rc = 1; + }else{ + const char *zFile = azArg[1]; + output_file_close(p->pLog); + p->pLog = output_file_open(zFile, 0); + } + }else + + if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){ + const char *zMode = nArg>=2 ? azArg[1] : ""; + int n2 = strlen30(zMode); + int c2 = zMode[0]; + if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ + p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ + p->mode = MODE_Column; + if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ + p->showHeader = 1; + } + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ + p->mode = MODE_Html; + }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ + p->mode = MODE_Tcl; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); + }else if( c2=='i' && strncmp(azArg[1],"insert",n2)==0 ){ + p->mode = MODE_Insert; + set_table_name(p, nArg>=3 ? azArg[2] : "table"); + }else if( c2=='q' && strncmp(azArg[1],"quote",n2)==0 ){ + p->mode = MODE_Quote; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( c2=='a' && strncmp(azArg[1],"ascii",n2)==0 ){ + p->mode = MODE_Ascii; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); + }else if( c2=='m' && strncmp(azArg[1],"markdown",n2)==0 ){ + p->mode = MODE_Markdown; + }else if( c2=='t' && strncmp(azArg[1],"table",n2)==0 ){ + p->mode = MODE_Table; + }else if( c2=='b' && strncmp(azArg[1],"box",n2)==0 ){ + p->mode = MODE_Box; + }else if( c2=='j' && strncmp(azArg[1],"json",n2)==0 ){ + p->mode = MODE_Json; + }else if( nArg==1 ){ + raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]); + }else{ + raw_printf(stderr, "Error: mode should be one of: " + "ascii box column csv html insert json line list markdown " + "quote table tabs tcl\n"); + rc = 1; + } + p->cMode = p->mode; + }else + + if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ + if( nArg==2 ){ + sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, + "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); + }else{ + raw_printf(stderr, "Usage: .nullvalue STRING\n"); + rc = 1; + } + }else + +#ifdef SQLITE_DEBUG + if( c=='o' && strcmp(azArg[0],"oom")==0 ){ + int i; + for(i=1; iout, "missing argument on \"%s\"\n", azArg[i]); + rc = 1; + }else{ + oomRepeat = (int)integerValue(azArg[++i]); + } + }else if( IsDigit(z[0]) ){ + oomCounter = (int)integerValue(azArg[i]); + }else{ + raw_printf(p->out, "unknown argument: \"%s\"\n", azArg[i]); + raw_printf(p->out, "Usage: .oom [--repeat N] [M]\n"); + rc = 1; + } + } + if( rc==0 ){ + raw_printf(p->out, "oomCounter = %d\n", oomCounter); + raw_printf(p->out, "oomRepeat = %d\n", oomRepeat); + } + }else +#endif /* SQLITE_DEBUG */ + + if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + char *zNewFilename = 0; /* Name of the database file to open */ + int iName = 1; /* Index in azArg[] of the filename */ + int newFlag = 0; /* True to delete file before opening */ + /* Close the existing database */ + session_close_all(p); + close_db(p->db); + p->db = 0; + p->zDbFilename = 0; + sqlite3_free(p->zFreeOnClose); + p->zFreeOnClose = 0; + p->openMode = SHELL_OPEN_UNSPEC; + p->openFlags = 0; + p->szMax = 0; + /* Check for command-line arguments */ + for(iName=1; iNameopenMode = SHELL_OPEN_ZIPFILE; +#endif + }else if( optionMatch(z, "append") ){ + p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; + }else if( optionMatch(z, "nofollow") ){ + p->openFlags |= SQLITE_OPEN_NOFOLLOW; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( optionMatch(z, "deserialize") ){ + p->openMode = SHELL_OPEN_DESERIALIZE; + }else if( optionMatch(z, "hexdb") ){ + p->openMode = SHELL_OPEN_HEXDB; + }else if( optionMatch(z, "maxsize") && iName+1szMax = integerValue(azArg[++iName]); +#endif /* SQLITE_ENABLE_DESERIALIZE */ + }else if( z[0]=='-' ){ + utf8_printf(stderr, "unknown option: %s\n", z); + rc = 1; + goto meta_command_exit; + }else if( zNewFilename ){ + utf8_printf(stderr, "extra argument: \"%s\"\n", z); + rc = 1; + goto meta_command_exit; + }else{ + zNewFilename = sqlite3_mprintf("%s", z); + } + } + /* If a filename is specified, try to open it first */ + if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){ + if( newFlag ) shellDeleteFile(zNewFilename); + p->zDbFilename = zNewFilename; + open_db(p, OPEN_DB_KEEPALIVE); + if( p->db==0 ){ + utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + sqlite3_free(zNewFilename); + }else{ + p->zFreeOnClose = zNewFilename; + } + } + if( p->db==0 ){ + /* As a fall-back open a TEMP database */ + p->zDbFilename = 0; + open_db(p, 0); + } + }else + + if( (c=='o' + && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + ){ + char *zFile = 0; + int bTxtMode = 0; + int i; + int eMode = 0; + int bBOM = 0; + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ + + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zFile==0 && eMode!='e' && eMode!='x' ){ + zFile = sqlite3_mprintf("%s", z); + if( zFile[0]=='|' ){ + while( i+1out,"ERROR: extra parameter: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + sqlite3_free(zFile); + goto meta_command_exit; + } + } + if( zFile==0 ) zFile = sqlite3_mprintf("stdout"); + if( bOnce ){ + p->outCount = 2; + }else{ + p->outCount = 0; + } + output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' ){ + p->doXdgOpen = 1; + outputModePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + ShellClearFlag(p, SHFLG_Echo); + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + bTxtMode = 1; + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + p->out = stdout; +#else + p->out = popen(zFile + 1, "w"); + if( p->out==0 ){ + utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + p->out = stdout; + rc = 1; + }else{ + if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + p->out = output_file_open(zFile, bTxtMode); + if( p->out==0 ){ + if( strcmp(zFile,"off")!=0 ){ + utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + p->out = stdout; + rc = 1; + } else { + if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); + }else + + if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + open_db(p,0); + if( nArg<=1 ) goto parameter_syntax_error; + + /* .parameter clear + ** Clear all bind parameters by dropping the TEMP table that holds them. + */ + if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", + 0, 0, 0); + }else + + /* .parameter list + ** List all bind parameters. + */ + if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + sqlite3_stmt *pStmt = 0; + int rx; + int len = 0; + rx = sqlite3_prepare_v2(p->db, + "SELECT max(length(key)) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + if( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + len = sqlite3_column_int(pStmt, 0); + if( len>40 ) len = 40; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( len ){ + rx = sqlite3_prepare_v2(p->db, + "SELECT key, quote(value) " + "FROM temp.sqlite_parameters;", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), + sqlite3_column_text(pStmt,1)); + } + sqlite3_finalize(pStmt); + } + }else + + /* .parameter init + ** Make sure the TEMP table used to hold bind parameters exists. + ** Create it if necessary. + */ + if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + bind_table_init(p); + }else + + /* .parameter set NAME VALUE + ** Set or reset a bind parameter. NAME should be the full parameter + ** name exactly as it appears in the query. (ex: $abc, @def). The + ** VALUE can be in either SQL literal notation, or if not it will be + ** understood to be a text string. + */ + if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + int rx; + char *zSql; + sqlite3_stmt *pStmt; + const char *zKey = azArg[2]; + const char *zValue = azArg[3]; + bind_table_init(p); + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%s);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + pStmt = 0; + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + zSql = sqlite3_mprintf( + "REPLACE INTO temp.sqlite_parameters(key,value)" + "VALUES(%Q,%Q);", zKey, zValue); + if( zSql==0 ) shell_out_of_memory(); + rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rx!=SQLITE_OK ){ + utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + pStmt = 0; + rc = 1; + } + } + sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + }else + + /* .parameter unset NAME + ** Remove the NAME binding from the parameter binding table, if it + ** exists. + */ + if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + char *zSql = sqlite3_mprintf( + "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); + if( zSql==0 ) shell_out_of_memory(); + sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + }else + /* If no command name matches, show a syntax error */ + parameter_syntax_error: + showHelp(p->out, "parameter"); + }else + + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + int i; + for(i=1; i1 ) raw_printf(p->out, " "); + utf8_printf(p->out, "%s", azArg[i]); + } + raw_printf(p->out, "\n"); + }else + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + int i; + int nn = 0; + p->flgProgress = 0; + p->mxProgress = 0; + p->nProgress = 0; + for(i=1; iflgProgress |= SHELL_PROGRESS_QUIET; + continue; + } + if( strcmp(z,"reset")==0 ){ + p->flgProgress |= SHELL_PROGRESS_RESET; + continue; + } + if( strcmp(z,"once")==0 ){ + p->flgProgress |= SHELL_PROGRESS_ONCE; + continue; + } + if( strcmp(z,"limit")==0 ){ + if( i+1>=nArg ){ + utf8_printf(stderr, "Error: missing argument on --limit\n"); + rc = 1; + goto meta_command_exit; + }else{ + p->mxProgress = (int)integerValue(azArg[++i]); + } + continue; + } + utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]); + rc = 1; + goto meta_command_exit; + }else{ + nn = (int)integerValue(z); + } + } + open_db(p, 0); + sqlite3_progress_handler(p->db, nn, progress_handler, p); + }else +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ + if( nArg >= 2) { + strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); + } + if( nArg >= 3) { + strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1); + } + }else + + if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ + rc = 2; + }else + + if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ + FILE *inSaved = p->in; + int savedLineno = p->lineno; + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .read FILE\n"); + rc = 1; + goto meta_command_exit; + } + if( azArg[1][0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + rc = 1; + p->out = stdout; +#else + p->in = popen(azArg[1]+1, "r"); + if( p->in==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + rc = 1; + }else{ + rc = process_input(p); + pclose(p->in); + } +#endif + }else if( notNormalFile(azArg[1]) || (p->in = fopen(azArg[1], "rb"))==0 ){ + utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + rc = 1; + }else{ + rc = process_input(p); + fclose(p->in); + } + p->in = inSaved; + p->lineno = savedLineno; + }else + + if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ + const char *zSrcFile; + const char *zDb; + sqlite3 *pSrc; + sqlite3_backup *pBackup; + int nTimeout = 0; + + if( nArg==2 ){ + zSrcFile = azArg[1]; + zDb = "main"; + }else if( nArg==3 ){ + zSrcFile = azArg[2]; + zDb = azArg[1]; + }else{ + raw_printf(stderr, "Usage: .restore ?DB? FILE\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_open(zSrcFile, &pSrc); + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile); + close_db(pSrc); + return 1; + } + open_db(p, 0); + pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); + if( pBackup==0 ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + close_db(pSrc); + return 1; + } + while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK + || rc==SQLITE_BUSY ){ + if( rc==SQLITE_BUSY ){ + if( nTimeout++ >= 3 ) break; + sqlite3_sleep(100); + } + } + sqlite3_backup_finish(pBackup); + if( rc==SQLITE_DONE ){ + rc = 0; + }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){ + raw_printf(stderr, "Error: source database is busy\n"); + rc = 1; + }else{ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + rc = 1; + } + close_db(pSrc); + }else + + if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( nArg==2 ){ + p->scanstatsOn = (u8)booleanValue(azArg[1]); +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); +#endif + }else{ + raw_printf(stderr, "Usage: .scanstats on|off\n"); + rc = 1; + } + }else + + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + ShellText sSelect; + ShellState data; + char *zErrMsg = 0; + const char *zDiv = "("; + const char *zName = 0; + int iSchema = 0; + int bDebug = 0; + int bNoSystemTabs = 0; + int ii; + + open_db(p, 0); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + initText(&sSelect); + for(ii=1; iidb, "SELECT name FROM pragma_database_list", + -1, &pStmt, 0); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + rc = 1; + goto meta_command_exit; + } + appendText(&sSelect, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + appendText(&sSelect, zDiv, 0); + zDiv = " UNION ALL "; + appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); + if( sqlite3_stricmp(zDb, "main")!=0 ){ + appendText(&sSelect, zDb, '\''); + }else{ + appendText(&sSelect, "NULL", 0); + } + appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); + appendText(&sSelect, zScNum, 0); + appendText(&sSelect, " AS snum, ", 0); + appendText(&sSelect, zDb, '\''); + appendText(&sSelect, " AS sname FROM ", 0); + appendText(&sSelect, zDb, quoteChar(zDb)); + appendText(&sSelect, ".sqlite_schema", 0); + } + sqlite3_finalize(pStmt); +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + if( zName ){ + appendText(&sSelect, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", + 0); + } +#endif + appendText(&sSelect, ") WHERE ", 0); + if( zName ){ + char *zQarg = sqlite3_mprintf("%Q", zName); + int bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); + }else{ + appendText(&sSelect, "lower(tbl_name)", 0); + } + appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); + appendText(&sSelect, zQarg, 0); + if( !bGlob ){ + appendText(&sSelect, " ESCAPE '\\' ", 0); + } + appendText(&sSelect, " AND ", 0); + sqlite3_free(zQarg); + } + if( bNoSystemTabs ){ + appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0); + } + appendText(&sSelect, "sql IS NOT NULL" + " ORDER BY snum, rowid", 0); + if( bDebug ){ + utf8_printf(p->out, "SQL: %s;\n", sSelect.z); + }else{ + rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg); + } + freeText(&sSelect); + } + if( zErrMsg ){ + utf8_printf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + rc = 1; + }else if( rc != SQLITE_OK ){ + raw_printf(stderr,"Error: querying schema information\n"); + rc = 1; + }else{ + rc = 0; + } + }else + + if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ + unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); + }else + +#if defined(SQLITE_ENABLE_SESSION) + if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + OpenSession *pSession = &p->aSession[0]; + char **azCmd = &azArg[1]; + int iSes = 0; + int nCmd = nArg - 1; + int i; + if( nArg<=1 ) goto session_syntax_error; + open_db(p, 0); + if( nArg>=3 ){ + for(iSes=0; iSesnSession; iSes++){ + if( strcmp(p->aSession[iSes].zName, azArg[1])==0 ) break; + } + if( iSesnSession ){ + pSession = &p->aSession[iSes]; + azCmd++; + nCmd--; + }else{ + pSession = &p->aSession[0]; + iSes = 0; + } + } + + /* .session attach TABLE + ** Invoke the sqlite3session_attach() interface to attach a particular + ** table so that it is never filtered. + */ + if( strcmp(azCmd[0],"attach")==0 ){ + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ){ + session_not_open: + raw_printf(stderr, "ERROR: No sessions are open\n"); + }else{ + rc = sqlite3session_attach(pSession->p, azCmd[1]); + if( rc ){ + raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + rc = 0; + } + } + }else + + /* .session changeset FILE + ** .session patchset FILE + ** Write a changeset or patchset into a file. The file is overwritten. + */ + if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + FILE *out = 0; + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ) goto session_not_open; + out = fopen(azCmd[1], "wb"); + if( out==0 ){ + utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", + azCmd[1]); + }else{ + int szChng; + void *pChng; + if( azCmd[0][0]=='c' ){ + rc = sqlite3session_changeset(pSession->p, &szChng, &pChng); + }else{ + rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); + } + if( rc ){ + printf("Error: error code %d\n", rc); + rc = 0; + } + if( pChng + && fwrite(pChng, szChng, 1, out)!=1 ){ + raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", + szChng); + } + sqlite3_free(pChng); + fclose(out); + } + }else + + /* .session close + ** Close the identified session + */ + if( strcmp(azCmd[0], "close")==0 ){ + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + session_close(pSession); + p->aSession[iSes] = p->aSession[--p->nSession]; + } + }else + + /* .session enable ?BOOLEAN? + ** Query or set the enable flag + */ + if( strcmp(azCmd[0], "enable")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_enable(pSession->p, ii); + utf8_printf(p->out, "session %s enable flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session filter GLOB .... + ** Set a list of GLOB patterns of table names to be excluded. + */ + if( strcmp(azCmd[0], "filter")==0 ){ + int ii, nByte; + if( nCmd<2 ) goto session_syntax_error; + if( p->nSession ){ + for(ii=0; iinFilter; ii++){ + sqlite3_free(pSession->azFilter[ii]); + } + sqlite3_free(pSession->azFilter); + nByte = sizeof(pSession->azFilter[0])*(nCmd-1); + pSession->azFilter = sqlite3_malloc( nByte ); + if( pSession->azFilter==0 ){ + raw_printf(stderr, "Error: out or memory\n"); + exit(1); + } + for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + } + pSession->nFilter = ii-1; + } + }else + + /* .session indirect ?BOOLEAN? + ** Query or set the indirect flag + */ + if( strcmp(azCmd[0], "indirect")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_indirect(pSession->p, ii); + utf8_printf(p->out, "session %s indirect flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session isempty + ** Determine if the session is empty + */ + if( strcmp(azCmd[0], "isempty")==0 ){ + int ii; + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + ii = sqlite3session_isempty(pSession->p); + utf8_printf(p->out, "session %s isempty flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session list + ** List all currently open sessions + */ + if( strcmp(azCmd[0],"list")==0 ){ + for(i=0; inSession; i++){ + utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName); + } + }else + + /* .session open DB NAME + ** Open a new session called NAME on the attached database DB. + ** DB is normally "main". + */ + if( strcmp(azCmd[0],"open")==0 ){ + char *zName; + if( nCmd!=3 ) goto session_syntax_error; + zName = azCmd[2]; + if( zName[0]==0 ) goto session_syntax_error; + for(i=0; inSession; i++){ + if( strcmp(p->aSession[i].zName,zName)==0 ){ + utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + goto meta_command_exit; + } + } + if( p->nSession>=ArraySize(p->aSession) ){ + raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession)); + goto meta_command_exit; + } + pSession = &p->aSession[p->nSession]; + rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); + if( rc ){ + raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + rc = 0; + goto meta_command_exit; + } + pSession->nFilter = 0; + sqlite3session_table_filter(pSession->p, session_filter, pSession); + p->nSession++; + pSession->zName = sqlite3_mprintf("%s", zName); + }else + /* If no command name matches, show a syntax error */ + session_syntax_error: + showHelp(p->out, "session"); + }else +#endif + +#ifdef SQLITE_DEBUG + /* Undocumented commands for internal testing. Subject to change + ** without notice. */ + if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ + if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + int i, v; + for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); + } + } + if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + int i; sqlite3_int64 v; + for(i=1; iout, "%s", zBuf); + } + } + }else +#endif + + if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + int bIsInit = 0; /* True to initialize the SELFTEST table */ + int bVerbose = 0; /* Verbose output */ + int bSelftestExists; /* True if SELFTEST already exists */ + int i, k; /* Loop counters */ + int nTest = 0; /* Number of tests runs */ + int nErr = 0; /* Number of errors seen */ + ShellText str; /* Answer for a query */ + sqlite3_stmt *pStmt = 0; /* Query against the SELFTEST table */ + + open_db(p,0); + for(i=1; idb,"main","selftest",0,0,0,0,0,0) + != SQLITE_OK ){ + bSelftestExists = 0; + }else{ + bSelftestExists = 1; + } + if( bIsInit ){ + createSelftestTable(p); + bSelftestExists = 1; + } + initText(&str); + appendText(&str, "x", 0); + for(k=bSelftestExists; k>=0; k--){ + if( k==1 ){ + rc = sqlite3_prepare_v2(p->db, + "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno", + -1, &pStmt, 0); + }else{ + rc = sqlite3_prepare_v2(p->db, + "VALUES(0,'memo','Missing SELFTEST table - default checks only','')," + " (1,'run','PRAGMA integrity_check','ok')", + -1, &pStmt, 0); + } + if( rc ){ + raw_printf(stderr, "Error querying the selftest table\n"); + rc = 1; + sqlite3_finalize(pStmt); + goto meta_command_exit; + } + for(i=1; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int tno = sqlite3_column_int(pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(pStmt, 1); + const char *zSql = (const char*)sqlite3_column_text(pStmt, 2); + const char *zAns = (const char*)sqlite3_column_text(pStmt, 3); + + k = 0; + if( bVerbose>0 ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + utf8_printf(p->out, "%s\n", zSql); + }else + if( strcmp(zOp,"run")==0 ){ + char *zErrMsg = 0; + str.n = 0; + str.z[0] = 0; + rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); + nTest++; + if( bVerbose ){ + utf8_printf(p->out, "Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + } + }else + { + utf8_printf(stderr, + "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + rc = 1; + break; + } + } /* End loop over rows of content from SELFTEST */ + sqlite3_finalize(pStmt); + } /* End loop over k */ + freeText(&str); + utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + }else + + if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ + if( nArg<2 || nArg>3 ){ + raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); + rc = 1; + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, + "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); + } + if( nArg>=3 ){ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, + "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); + } + }else + + if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + const char *zLike = 0; /* Which table to checksum. 0 means everything */ + int i; /* Loop counter */ + int bSchema = 0; /* Also hash the schema */ + int bSeparate = 0; /* Hash each table separately */ + int iSize = 224; /* Hash algorithm to use */ + int bDebug = 0; /* Only show the query that would have run */ + sqlite3_stmt *pStmt; /* For querying tables names */ + char *zSql; /* SQL to be run */ + char *zSep; /* Separator */ + ShellText sSql; /* Complete SQL for the query to run the hash */ + ShellText sQuery; /* Set of queries used to read all content */ + open_db(p, 0); + for(i=1; iout, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zLike ){ + raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); + rc = 1; + goto meta_command_exit; + }else{ + zLike = z; + bSeparate = 1; + if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1; + } + } + if( bSchema ){ + zSql = "SELECT lower(name) FROM sqlite_schema" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " UNION ALL SELECT 'sqlite_schema'" + " ORDER BY 1 collate nocase"; + }else{ + zSql = "SELECT lower(name) FROM sqlite_schema" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " AND name NOT LIKE 'sqlite_%'" + " ORDER BY 1 collate nocase"; + } + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + initText(&sQuery); + initText(&sSql); + appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); + zSep = "VALUES("; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; + if( strncmp(zTab, "sqlite_",7)!=0 ){ + appendText(&sQuery,"SELECT * FROM ", 0); + appendText(&sQuery,zTab,'"'); + appendText(&sQuery," NOT INDEXED;", 0); + }else if( strcmp(zTab, "sqlite_schema")==0 ){ + appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); + }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); + } + appendText(&sSql, zSep, 0); + appendText(&sSql, sQuery.z, '\''); + sQuery.n = 0; + appendText(&sSql, ",", 0); + appendText(&sSql, zTab, '\''); + zSep = "),("; + } + sqlite3_finalize(pStmt); + if( bSeparate ){ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" + " FROM [sha3sum$query]", + sSql.z, iSize); + }else{ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" + " FROM [sha3sum$query]", + sSql.z, iSize); + } + freeText(&sQuery); + freeText(&sSql); + if( bDebug ){ + utf8_printf(p->out, "%s\n", zSql); + }else{ + shell_exec(p, zSql, 0); + } + sqlite3_free(zSql); + }else + +#ifndef SQLITE_NOHAVE_SYSTEM + if( c=='s' + && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) + ){ + char *zCmd; + int i, x; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .system COMMAND\n"); + rc = 1; + goto meta_command_exit; + } + zCmd = sqlite3_mprintf(strchr(azArg[1],' ')==0?"%s":"\"%s\"", azArg[1]); + for(i=2; iout, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); + utf8_printf(p->out, "%12.12s: %s\n","explain", + p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); + utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); + utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); + utf8_printf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->nullValue); + raw_printf(p->out, "\n"); + utf8_printf(p->out,"%12.12s: %s\n","output", + strlen30(p->outfile) ? p->outfile : "stdout"); + utf8_printf(p->out,"%12.12s: ", "colseparator"); + output_c_string(p->out, p->colSeparator); + raw_printf(p->out, "\n"); + utf8_printf(p->out,"%12.12s: ", "rowseparator"); + output_c_string(p->out, p->rowSeparator); + raw_printf(p->out, "\n"); + switch( p->statsOn ){ + case 0: zOut = "off"; break; + default: zOut = "on"; break; + case 2: zOut = "stmt"; break; + case 3: zOut = "vmstep"; break; + } + utf8_printf(p->out, "%12.12s: %s\n","stats", zOut); + utf8_printf(p->out, "%12.12s: ", "width"); + for (i=0;inWidth;i++) { + raw_printf(p->out, "%d ", p->colWidth[i]); + } + raw_printf(p->out, "\n"); + utf8_printf(p->out, "%12.12s: %s\n", "filename", + p->zDbFilename ? p->zDbFilename : ""); + }else + + if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ + if( nArg==2 ){ + if( strcmp(azArg[1],"stmt")==0 ){ + p->statsOn = 2; + }else if( strcmp(azArg[1],"vmstep")==0 ){ + p->statsOn = 3; + }else{ + p->statsOn = (u8)booleanValue(azArg[1]); + } + }else if( nArg==1 ){ + display_stats(p->db, p, 0); + }else{ + raw_printf(stderr, "Usage: .stats ?on|off|stmt|vmstep?\n"); + rc = 1; + } + }else + + if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (strncmp(azArg[0], "indices", n)==0 + || strncmp(azArg[0], "indexes", n)==0) ) + ){ + sqlite3_stmt *pStmt; + char **azResult; + int nRow, nAlloc; + int ii; + ShellText s; + initText(&s); + open_db(p, 0); + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return shellDatabaseError(p->db); + } + + if( nArg>2 && c=='i' ){ + /* It is an historical accident that the .indexes command shows an error + ** when called with the wrong number of arguments whereas the .tables + ** command does not. */ + raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); + rc = 1; + sqlite3_finalize(pStmt); + goto meta_command_exit; + } + for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ + const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); + if( zDbName==0 ) continue; + if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0); + if( sqlite3_stricmp(zDbName, "main")==0 ){ + appendText(&s, "SELECT name FROM ", 0); + }else{ + appendText(&s, "SELECT ", 0); + appendText(&s, zDbName, '\''); + appendText(&s, "||'.'||name FROM ", 0); + } + appendText(&s, zDbName, '"'); + appendText(&s, ".sqlite_schema ", 0); + if( c=='t' ){ + appendText(&s," WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%'" + " AND name LIKE ?1", 0); + }else{ + appendText(&s," WHERE type='index'" + " AND tbl_name LIKE ?1", 0); + } + } + rc = sqlite3_finalize(pStmt); + appendText(&s, " ORDER BY 1", 0); + rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0); + freeText(&s); + if( rc ) return shellDatabaseError(p->db); + + /* Run the SQL statement prepared by the above block. Store the results + ** as an array of nul-terminated strings in azResult[]. */ + nRow = nAlloc = 0; + azResult = 0; + if( nArg>1 ){ + sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nRow>=nAlloc ){ + char **azNew; + int n2 = nAlloc*2 + 10; + azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); + if( azNew==0 ) shell_out_of_memory(); + nAlloc = n2; + azResult = azNew; + } + azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + if( 0==azResult[nRow] ) shell_out_of_memory(); + nRow++; + } + if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ + rc = shellDatabaseError(p->db); + } + + /* Pretty-print the contents of array azResult[] to the output */ + if( rc==0 && nRow>0 ){ + int len, maxlen = 0; + int i, j; + int nPrintCol, nPrintRow; + for(i=0; imaxlen ) maxlen = len; + } + nPrintCol = 80/(maxlen+2); + if( nPrintCol<1 ) nPrintCol = 1; + nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; + for(i=0; iout, "%s%-*s", zSp, maxlen, + azResult[j] ? azResult[j]:""); + } + raw_printf(p->out, "\n"); + } + } + + for(ii=0; iiout = output_file_open("testcase-out.txt", 0); + if( p->out==0 ){ + raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); + } + }else + +#ifndef SQLITE_UNTESTABLE + if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ + static const struct { + const char *zCtrlName; /* Name of a test-control option */ + int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ + } aCtrl[] = { + { "always", SQLITE_TESTCTRL_ALWAYS, "BOOLEAN" }, + { "assert", SQLITE_TESTCTRL_ASSERT, "BOOLEAN" }, + /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, "" },*/ + /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, "" },*/ + { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, + { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,"BOOLEAN" }, + /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" },*/ + { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, + { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "" }, + { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, + { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, + { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, +#ifdef YYCOVERAGE + { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE, "" }, +#endif + { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, + { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, + { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, + { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" }, + { "seek_count", SQLITE_TESTCTRL_SEEK_COUNT, "" }, + }; + int testctrl = -1; + int iCtrl = -1; + int rc2 = 0; /* 0: usage. 1: %d 2: %x 3: no-output */ + int isOk = 0; + int i, n2; + const char *zCmd = 0; + + open_db(p, 0); + zCmd = nArg>=2 ? azArg[1] : "help"; + + /* The argument can optionally begin with "-" or "--" */ + if( zCmd[0]=='-' && zCmd[1] ){ + zCmd++; + if( zCmd[0]=='-' && zCmd[1] ) zCmd++; + } + + /* --help lists all test-controls */ + if( strcmp(zCmd,"help")==0 ){ + utf8_printf(p->out, "Available test-controls:\n"); + for(i=0; iout, " .testctrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); + } + rc = 1; + goto meta_command_exit; + } + + /* convert testctrl text option to value. allow any unique prefix + ** of the option name, or a numerical value. */ + n2 = strlen30(zCmd); + for(i=0; idb, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(int) */ + case SQLITE_TESTCTRL_PRNG_SAVE: + case SQLITE_TESTCTRL_PRNG_RESTORE: + case SQLITE_TESTCTRL_BYTEORDER: + if( nArg==2 ){ + rc2 = sqlite3_test_control(testctrl); + isOk = testctrl==SQLITE_TESTCTRL_BYTEORDER ? 1 : 3; + } + break; + + /* sqlite3_test_control(int, uint) */ + case SQLITE_TESTCTRL_PENDING_BYTE: + if( nArg==3 ){ + unsigned int opt = (unsigned int)integerValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(int, int, sqlite3*) */ + case SQLITE_TESTCTRL_PRNG_SEED: + if( nArg==3 || nArg==4 ){ + int ii = (int)integerValue(azArg[2]); + sqlite3 *db; + if( ii==0 && strcmp(azArg[2],"random")==0 ){ + sqlite3_randomness(sizeof(ii),&ii); + printf("-- random seed: %d\n", ii); + } + if( nArg==3 ){ + db = 0; + }else{ + db = p->db; + /* Make sure the schema has been loaded */ + sqlite3_table_column_metadata(db, 0, "x", 0, 0, 0, 0, 0, 0); + } + rc2 = sqlite3_test_control(testctrl, ii, db); + isOk = 3; + } + break; + + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_ASSERT: + case SQLITE_TESTCTRL_ALWAYS: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 1; + } + break; + + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: + case SQLITE_TESTCTRL_NEVER_CORRUPT: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 3; + } + break; + + /* sqlite3_test_control(sqlite3*) */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: + rc2 = sqlite3_test_control(testctrl, p->db); + isOk = 3; + break; + + case SQLITE_TESTCTRL_IMPOSTER: + if( nArg==5 ){ + rc2 = sqlite3_test_control(testctrl, p->db, + azArg[2], + integerValue(azArg[3]), + integerValue(azArg[4])); + isOk = 3; + } + break; + + case SQLITE_TESTCTRL_SEEK_COUNT: { + u64 x = 0; + rc2 = sqlite3_test_control(testctrl, p->db, &x); + utf8_printf(p->out, "%llu\n", x); + isOk = 3; + break; + } + +#ifdef YYCOVERAGE + case SQLITE_TESTCTRL_PARSER_COVERAGE: + if( nArg==2 ){ + sqlite3_test_control(testctrl, p->out); + isOk = 3; + } +#endif + } + } + if( isOk==0 && iCtrl>=0 ){ + utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + raw_printf(p->out, "%d\n", rc2); + }else if( isOk==2 ){ + raw_printf(p->out, "0x%08x\n", rc2); + } + }else +#endif /* !defined(SQLITE_UNTESTABLE) */ + + if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ + open_db(p, 0); + sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); + }else + + if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ + if( nArg==2 ){ + enableTimer = booleanValue(azArg[1]); + if( enableTimer && !HAS_TIMER ){ + raw_printf(stderr, "Error: timer not available on this system.\n"); + enableTimer = 0; + } + }else{ + raw_printf(stderr, "Usage: .timer on|off\n"); + rc = 1; + } + }else + +#ifndef SQLITE_OMIT_TRACE + if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; + open_db(p, 0); + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } + } + if( p->traceOut==0 ){ + sqlite3_trace_v2(p->db, 0, 0, 0); + }else{ + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); + } + }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) + if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ + int ii; + int lenOpt; + char *zOpt; + if( nArg<2 ){ + raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + zOpt = azArg[1]; + if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; + lenOpt = (int)strlen(zOpt); + if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ + assert( azArg[nArg]==0 ); + sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); + }else{ + for(ii=1; iidb, azArg[ii], 0, 0); + } + } + }else +#endif + +#if SQLITE_USER_AUTHENTICATION + if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( nArg<2 ){ + raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( strcmp(azArg[1],"login")==0 ){ + if( nArg!=4 ){ + raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], + strlen30(azArg[3])); + if( rc ){ + utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); + rc = 1; + } + }else if( strcmp(azArg[1],"add")==0 ){ + if( nArg!=5 ){ + raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + raw_printf(stderr, "User-Add failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"edit")==0 ){ + if( nArg!=5 ){ + raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + raw_printf(stderr, "User-Edit failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"delete")==0 ){ + if( nArg!=3 ){ + raw_printf(stderr, "Usage: .user delete USER\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_delete(p->db, azArg[2]); + if( rc ){ + raw_printf(stderr, "User-Delete failed: %d\n", rc); + rc = 1; + } + }else{ + raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); + rc = 1; + goto meta_command_exit; + } + }else +#endif /* SQLITE_USER_AUTHENTICATION */ + + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + sqlite3_libversion(), sqlite3_sourceid()); +#if SQLITE_HAVE_ZLIB + utf8_printf(p->out, "zlib version %s\n", zlibVersion()); +#endif +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) +#if defined(__clang__) && defined(__clang_major__) + utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__) "\n"); +#elif defined(_MSC_VER) + utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n"); +#elif defined(__GNUC__) && defined(__VERSION__) + utf8_printf(p->out, "gcc-" __VERSION__ "\n"); +#endif + }else + + if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ + const char *zDbName = nArg==2 ? azArg[1] : "main"; + sqlite3_vfs *pVfs = 0; + if( p->db ){ + sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); + if( pVfs ){ + utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + } + } + }else + + if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + sqlite3_vfs *pVfs; + sqlite3_vfs *pCurrent = 0; + if( p->db ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); + } + for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ + utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + if( pVfs->pNext ){ + raw_printf(p->out, "-----------------------------------\n"); + } + } + }else + + if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + const char *zDbName = nArg==2 ? azArg[1] : "main"; + char *zVfsName = 0; + if( p->db ){ + sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); + if( zVfsName ){ + utf8_printf(p->out, "%s\n", zVfsName); + sqlite3_free(zVfsName); + } + } + }else + + if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); + }else + + if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + int j; + assert( nArg<=ArraySize(azArg) ); + p->nWidth = nArg-1; + p->colWidth = realloc(p->colWidth, p->nWidth*sizeof(int)*2); + if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); + if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; + for(j=1; jcolWidth[j-1] = (int)integerValue(azArg[j]); + } + }else + + { + utf8_printf(stderr, "Error: unknown command or invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[0]); + rc = 1; + } + +meta_command_exit: + if( p->outCount ){ + p->outCount--; + if( p->outCount==0 ) output_reset(p); + } + return rc; +} + +/* +** Return TRUE if a semicolon occurs anywhere in the first N characters +** of string z[]. +*/ +static int line_contains_semicolon(const char *z, int N){ + int i; + for(i=0; iflgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; + BEGIN_TIMER; + rc = shell_exec(p, zSql, &zErrMsg); + END_TIMER; + if( rc || zErrMsg ){ + char zPrefix[100]; + if( in!=0 || !stdin_is_interactive ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "Error: near line %d:", startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); + } + if( zErrMsg!=0 ){ + utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg); + sqlite3_free(zErrMsg); + zErrMsg = 0; + }else{ + utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); + } + return 1; + }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ + raw_printf(p->out, "changes: %3d total_changes: %d\n", + sqlite3_changes(p->db), sqlite3_total_changes(p->db)); + } + return 0; +} + + +/* +** Read input from *in and process it. If *in==0 then input +** is interactive - the user is typing it it. Otherwise, input +** is coming from a file or device. A prompt is issued and history +** is saved only if input is interactive. An interrupt signal will +** cause this routine to exit immediately, unless input is interactive. +** +** Return the number of errors. +*/ +static int process_input(ShellState *p){ + char *zLine = 0; /* A single input line */ + char *zSql = 0; /* Accumulated SQL text */ + int nLine; /* Length of current line */ + int nSql = 0; /* Bytes of zSql[] used */ + int nAlloc = 0; /* Allocated zSql[] space */ + int nSqlPrior = 0; /* Bytes of zSql[] used by prior line */ + int rc; /* Error code */ + int errCnt = 0; /* Number of errors seen */ + int startline = 0; /* Line number for start of current input */ + + p->lineno = 0; + while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ + fflush(p->out); + zLine = one_input_line(p->in, zLine, nSql>0); + if( zLine==0 ){ + /* End of input */ + if( p->in==0 && stdin_is_interactive ) printf("\n"); + break; + } + if( seenInterrupt ){ + if( p->in!=0 ) break; + seenInterrupt = 0; + } + p->lineno++; + if( nSql==0 && _all_whitespace(zLine) ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); + continue; + } + if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); + if( zLine[0]=='.' ){ + rc = do_meta_command(zLine, p); + if( rc==2 ){ /* exit requested */ + break; + }else if( rc ){ + errCnt++; + } + } + continue; + } + if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){ + memcpy(zLine,";",2); + } + nLine = strlen30(zLine); + if( nSql+nLine+2>=nAlloc ){ + nAlloc = nSql+nLine+100; + zSql = realloc(zSql, nAlloc); + if( zSql==0 ) shell_out_of_memory(); + } + nSqlPrior = nSql; + if( nSql==0 ){ + int i; + for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} + assert( nAlloc>0 && zSql!=0 ); + memcpy(zSql, zLine+i, nLine+1-i); + startline = p->lineno; + nSql = nLine-i; + }else{ + zSql[nSql++] = '\n'; + memcpy(zSql+nSql, zLine, nLine+1); + nSql += nLine; + } + if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior) + && sqlite3_complete(zSql) ){ + errCnt += runOneSqlLine(p, zSql, p->in, startline); + nSql = 0; + if( p->outCount ){ + output_reset(p); + p->outCount = 0; + }else{ + clearTempFile(p); + } + }else if( nSql && _all_whitespace(zSql) ){ + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); + nSql = 0; + } + } + if( nSql && !_all_whitespace(zSql) ){ + errCnt += runOneSqlLine(p, zSql, p->in, startline); + } + free(zSql); + free(zLine); + return errCnt>0; +} + +/* +** Return a pathname which is the user's home directory. A +** 0 return indicates an error of some kind. +*/ +static char *find_home_dir(int clearFlag){ + static char *home_dir = NULL; + if( clearFlag ){ + free(home_dir); + home_dir = 0; + return 0; + } + if( home_dir ) return home_dir; + +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) + { + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != NULL) { + home_dir = pwent->pw_dir; + } + } +#endif + +#if defined(_WIN32_WCE) + /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() + */ + home_dir = "/"; +#else + +#if defined(_WIN32) || defined(WIN32) + if (!home_dir) { + home_dir = getenv("USERPROFILE"); + } +#endif + + if (!home_dir) { + home_dir = getenv("HOME"); + } + +#if defined(_WIN32) || defined(WIN32) + if (!home_dir) { + char *zDrive, *zPath; + int n; + zDrive = getenv("HOMEDRIVE"); + zPath = getenv("HOMEPATH"); + if( zDrive && zPath ){ + n = strlen30(zDrive) + strlen30(zPath) + 1; + home_dir = malloc( n ); + if( home_dir==0 ) return 0; + sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath); + return home_dir; + } + home_dir = "c:\\"; + } +#endif + +#endif /* !_WIN32_WCE */ + + if( home_dir ){ + int n = strlen30(home_dir) + 1; + char *z = malloc( n ); + if( z ) memcpy(z, home_dir, n); + home_dir = z; + } + + return home_dir; +} + +/* +** Read input from the file given by sqliterc_override. Or if that +** parameter is NULL, take input from ~/.sqliterc +** +** Returns the number of errors. +*/ +static void process_sqliterc( + ShellState *p, /* Configuration data */ + const char *sqliterc_override /* Name of config file. NULL to use default */ +){ + char *home_dir = NULL; + const char *sqliterc = sqliterc_override; + char *zBuf = 0; + FILE *inSaved = p->in; + int savedLineno = p->lineno; + + if (sqliterc == NULL) { + home_dir = find_home_dir(0); + if( home_dir==0 ){ + raw_printf(stderr, "-- warning: cannot find home directory;" + " cannot read ~/.sqliterc\n"); + return; + } + zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); + sqliterc = zBuf; + } + p->in = fopen(sqliterc,"rb"); + if( p->in ){ + if( stdin_is_interactive ){ + utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc); + } + if( process_input(p) && bail_on_error ) exit(1); + fclose(p->in); + }else if( sqliterc_override!=0 ){ + utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) exit(1); + } + p->in = inSaved; + p->lineno = savedLineno; + sqlite3_free(zBuf); +} + +/* +** Show available command line options +*/ +static const char zOptions[] = +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + " -A ARGS... run \".archive ARGS\" and exit\n" +#endif + " -append append the database to the end of the file\n" + " -ascii set output mode to 'ascii'\n" + " -bail stop after hitting an error\n" + " -batch force batch I/O\n" + " -box set output mode to 'box'\n" + " -column set output mode to 'column'\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" + " -csv set output mode to 'csv'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -deserialize open the database using sqlite3_deserialize()\n" +#endif + " -echo print commands before execution\n" + " -init FILENAME read/process named file\n" + " -[no]header turn headers on or off\n" +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + " -heap SIZE Size of heap for memsys3 or memsys5\n" +#endif + " -help show this message\n" + " -html set output mode to HTML\n" + " -interactive force interactive I/O\n" + " -json set output mode to 'json'\n" + " -line set output mode to 'line'\n" + " -list set output mode to 'list'\n" + " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" + " -markdown set output mode to 'markdown'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -maxsize N maximum size for a --deserialize database\n" +#endif + " -memtrace trace all memory allocations and deallocations\n" + " -mmap N default mmap size set to N\n" +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif + " -newline SEP set output row separator. Default: '\\n'\n" + " -nofollow refuse to open symbolic links to database files\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -quote set output mode to 'quote'\n" + " -readonly open the database read-only\n" + " -separator SEP set output column separator. Default: '|'\n" +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + " -sorterref SIZE sorter references threshold size\n" +#endif + " -stats print memory stats before each finalize\n" + " -table set output mode to 'table'\n" + " -tabs set output mode to 'tabs'\n" + " -version show SQLite version\n" + " -vfs NAME use NAME as the default VFS\n" +#ifdef SQLITE_ENABLE_VFSTRACE + " -vfstrace enable tracing of all VFS calls\n" +#endif +#ifdef SQLITE_HAVE_ZLIB + " -zip open the file as a ZIP Archive\n" +#endif +; +static void usage(int showDetail){ + utf8_printf(stderr, + "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "FILENAME is the name of an SQLite database. A new database is created\n" + "if the file does not previously exist.\n", Argv0); + if( showDetail ){ + utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); + }else{ + raw_printf(stderr, "Use the -help option for additional information\n"); + } + exit(1); +} + +/* +** Internal check: Verify that the SQLite is uninitialized. Print a +** error message if it is initialized. +*/ +static void verify_uninitialized(void){ + if( sqlite3_config(-1)==SQLITE_MISUSE ){ + utf8_printf(stdout, "WARNING: attempt to configure SQLite after" + " initialization.\n"); + } +} + +/* +** Initialize the state information in data +*/ +static void main_init(ShellState *data) { + memset(data, 0, sizeof(*data)); + data->normalMode = data->cMode = data->mode = MODE_List; + data->autoExplain = 1; + memcpy(data->colSeparator,SEP_Column, 2); + memcpy(data->rowSeparator,SEP_Row, 2); + data->showHeader = 0; + data->shellFlgs = SHFLG_Lookaside; + verify_uninitialized(); + sqlite3_config(SQLITE_CONFIG_URI, 1); + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); + sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); +} + +/* +** Output text to the console in a font that attracts extra attention. +*/ +#ifdef _WIN32 +static void printBold(const char *zText){ +#if !SQLITE_OS_WINRT + HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; + GetConsoleScreenBufferInfo(out, &defaultScreenInfo); + SetConsoleTextAttribute(out, + FOREGROUND_RED|FOREGROUND_INTENSITY + ); +#endif + printf("%s", zText); +#if !SQLITE_OS_WINRT + SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); +#endif +} +#else +static void printBold(const char *zText){ + printf("\033[1m%s\033[0m", zText); +} +#endif + +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static char *cmdline_option_value(int argc, char **argv, int i){ + if( i==argc ){ + utf8_printf(stderr, "%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + +#ifndef SQLITE_SHELL_IS_UTF8 +# if (defined(_WIN32) || defined(WIN32)) \ + && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) +# define SQLITE_SHELL_IS_UTF8 (0) +# else +# define SQLITE_SHELL_IS_UTF8 (1) +# endif +#endif + +#if SQLITE_SHELL_IS_UTF8 +int SQLITE_CDECL main(int argc, char **argv){ +#else +int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +#endif + char *zErrMsg = 0; + ShellState data; + const char *zInitFile = 0; + int i; + int rc = 0; + int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; + const char *zVfs = 0; /* Value of -vfs command-line option */ +#if !SQLITE_SHELL_IS_UTF8 + char **argvToFree = 0; + int argcToFree = 0; +#endif + + setBinaryMode(stdin, 0); + setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); + +#ifdef SQLITE_DEBUG + registerOomSimulator(); +#endif + +#if !defined(_WIN32_WCE) + if( getenv("SQLITE_DEBUG_BREAK") ){ + if( isatty(0) && isatty(2) ){ + fprintf(stderr, + "attach debugger to process %d and press any key to continue.\n", + GETPID()); + fgetc(stdin); + }else{ +#if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT + __debugbreak(); +#else + DebugBreak(); +#endif +#elif defined(SIGTRAP) + raise(SIGTRAP); +#endif + } + } +#endif + +#if USE_SYSTEM_SQLITE+0!=1 + if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ + utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); + exit(1); + } +#endif + main_init(&data); + + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ +#if !SQLITE_SHELL_IS_UTF8 + sqlite3_initialize(); + argvToFree = malloc(sizeof(argv[0])*argc*2); + argcToFree = argc; + argv = argvToFree + argc; + if( argv==0 ) shell_out_of_memory(); + for(i=0; i=1 && argv && argv[0] ); + Argv0 = argv[0]; + + /* Make sure we have a valid signal handler early, before anything + ** else is done. + */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); +#endif + +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif + + /* Do an initial pass through the command-line argument to locate + ** the name of the database file, the name of the initialization file, + ** the size of the alternative malloc heap, + ** and the first command to execute. + */ + verify_uninitialized(); + for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; + sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); +#else + (void)cmdline_option_value(argc, argv, ++i); +#endif + }else if( strcmp(z,"-pagecache")==0 ){ + sqlite3_int64 n, sz; + sz = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>70000 ) sz = 70000; + if( sz<0 ) sz = 0; + n = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>0 && n>0 && 0xffffffffffffLL/sz0 && sz>0) ? malloc(n*sz) : 0, sz, n); + data.shellFlgs |= SHFLG_Pagecache; + }else if( strcmp(z,"-lookaside")==0 ){ + int n, sz; + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( sz<0 ) sz = 0; + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( n<0 ) n = 0; + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); + if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; +#ifdef SQLITE_ENABLE_VFSTRACE + }else if( strcmp(z,"-vfstrace")==0 ){ + extern int vfstrace_register( + const char *zTraceName, + const char *zOldVfsName, + int (*xOut)(const char*,void*), + void *pOutArg, + int makeDefault + ); + vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(z,"-multiplex")==0 ){ + extern int sqlite3_multiple_initialize(const char*,int); + sqlite3_multiplex_initialize(0, 1); +#endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + }else if( strcmp(z,"-sorterref")==0 ){ + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); +#endif + }else if( strcmp(z,"-vfs")==0 ){ + zVfs = cmdline_option_value(argc, argv, ++i); +#ifdef SQLITE_HAVE_ZLIB + }else if( strcmp(z,"-zip")==0 ){ + data.openMode = SHELL_OPEN_ZIPFILE; +#endif + }else if( strcmp(z,"-append")==0 ){ + data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; + }else if( strcmp(z,"-maxsize")==0 && i+10 ){ + utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" + " with \"%s\"\n", z); + return 1; + } + open_db(&data, OPEN_DB_ZIPFILE); + if( z[2] ){ + argv[i] = &z[2]; + arDotCommand(&data, 1, argv+(i-1), argc-(i-1)); + }else{ + arDotCommand(&data, 1, argv+i, argc-i); + } + readStdin = 0; + break; +#endif + }else{ + utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + raw_printf(stderr,"Use -help for a list of options.\n"); + return 1; + } + data.cMode = data.mode; + } + + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. + */ + for(i=0; iFossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evaluates to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. If the source code has +** been edited in any way since it was last checked in, then the last +** four hexadecimal digits of the hash may be modified. +** +** See also: [sqlite3_libversion()], +** [sqlite3_libversion_number()], [sqlite3_sourceid()], +** [sqlite_version()] and [sqlite_source_id()]. +*/ +#define SQLITE_VERSION "3.35.5" +#define SQLITE_VERSION_NUMBER 3035005 +#define SQLITE_SOURCE_ID "2021-04-19 18:32:05 1b256d97b553a9611efca188a3d995a2fff712759044ba480f9a0c9e98fae886" + +/* +** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version sqlite3_sourceid +** +** These interfaces provide the same information as the [SQLITE_VERSION], +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious +** programmers might include assert() statements in their application to +** verify that values returned by these interfaces match the macros in +** the header, and thus ensure that the application is +** compiled with matching library and header files. +** +** 
+** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
+**
)^ +** +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^(The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. Except if SQLite is built +** using an edited copy of [the amalgamation], then the last four characters +** of the hash might be different from [SQLITE_SOURCE_ID].)^ +** +** See also: [sqlite_version()] and [sqlite_source_id()]. +*/ +SQLITE_API SQLITE_EXTERN const char sqlite3_version[]; +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); + +/* +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows iterating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifying the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) +#endif + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled with mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. +** +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** ^The default behavior is for mutexes to be enabled. +** +** This interface can be used by an application to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_SERIALIZED]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ +** +** See the [threading mode] documentation for additional information. +*/ +SQLITE_API int sqlite3_threadsafe(void); + +/* +** CAPI3REF: Database Connection Handle +** KEYWORDS: {database connection} {database connections} +** +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. +*/ +typedef struct sqlite3 sqlite3; + +/* +** CAPI3REF: 64-Bit Integer Types +** KEYWORDS: sqlite_int64 sqlite_uint64 +*/ +#define sqlite_int64 int64_t +#define sqlite_uint64 uint64_t +#define sqlite3_int64 int64_t +#define sqlite3_uint64 uint64_t + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite3_int64 +#endif + +/* +** CAPI3REF: Closing A Database Connection +** DESTRUCTOR: sqlite3 +** +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. +** +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, +** the transaction is automatically rolled back. +** +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. +*/ +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); + +/* +** The type for a callback function. +** This is legacy and deprecated. It is included for historical +** compatibility and is not documented. +*/ +typedef int (*sqlite3_callback)(void*,int,char**, char**); + +/* +** CAPI3REF: One-Step Query Execution Interface +** METHOD: sqlite3 +** +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. +** +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. +** +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. +** +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. +** +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. +** +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. +** +** Restrictions: +** +**
    +**
  • The application must ensure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
  • The application must not close the [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
+*/ +SQLITE_API int sqlite3_exec( + sqlite3*, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ +); + +/* +** CAPI3REF: Result Codes +** KEYWORDS: {result code definitions} +** +** Many SQLite functions return an integer result code from the set shown +** here in order to indicate success or failure. +** +** New error codes may be added in future versions of SQLite. +** +** See also: [extended result code definitions] +*/ +#define SQLITE_OK 0 /* Successful result */ +/* beginning-of-error-codes */ +#define SQLITE_ERROR 1 /* Generic error */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ +#define SQLITE_PERM 3 /* Access permission denied */ +#define SQLITE_ABORT 4 /* Callback routine requested an abort */ +#define SQLITE_BUSY 5 /* The database file is locked */ +#define SQLITE_LOCKED 6 /* A table in the database is locked */ +#define SQLITE_NOMEM 7 /* A malloc() failed */ +#define SQLITE_READONLY 8 /* Attempt to write a readonly database */ +#define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ +#define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ +#define SQLITE_CORRUPT 11 /* The database disk image is malformed */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ +#define SQLITE_FULL 13 /* Insertion failed because database is full */ +#define SQLITE_CANTOPEN 14 /* Unable to open the database file */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Internal use only */ +#define SQLITE_SCHEMA 17 /* The database schema changed */ +#define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ +#define SQLITE_MISMATCH 20 /* Data type mismatch */ +#define SQLITE_MISUSE 21 /* Library used incorrectly */ +#define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ +#define SQLITE_AUTH 23 /* Authorization denied */ +#define SQLITE_FORMAT 24 /* Not used */ +#define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ +#define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ +#define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ +#define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ +/* end-of-error-codes */ + +/* +** CAPI3REF: Extended Result Codes +** KEYWORDS: {extended result code definitions} +** +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include +** support for additional result codes that provide more detailed information +** about errors. These [extended result codes] are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. +*/ +#define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) +#define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) +#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) +#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) +#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) +#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) +#define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) +#define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) +#define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) +#define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) +#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) +#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ +#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) +#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) +#define SQLITE_READONLY_DIRECTORY (SQLITE_READONLY | (6<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) + +/* +** CAPI3REF: Flags For File Open Operations +** +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ +#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ +#define SQLITE_OPEN_SUPER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ + +/* Reserved: 0x00F00000 */ +/* Legacy compatibility: */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ + + +/* +** CAPI3REF: Device Characteristics +** +** The xDeviceCharacteristics method of the [sqlite3_io_methods] +** object returns an integer which is a vector of these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicates that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. +** +** The SQLITE_IOCAP_BATCH_ATOMIC property means that the underlying +** filesystem supports doing multiple write operations atomically when those +** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and +** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 +#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 +#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 + +/* +** CAPI3REF: File Locking Levels +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. +*/ +#define SQLITE_LOCK_NONE 0 +#define SQLITE_LOCK_SHARED 1 +#define SQLITE_LOCK_RESERVED 2 +#define SQLITE_LOCK_PENDING 3 +#define SQLITE_LOCK_EXCLUSIVE 4 + +/* +** CAPI3REF: Synchronization Type Flags +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. If the lower four bits of the flag +** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. +** If the lower four bits equal SQLITE_SYNC_FULL, that means +** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle +** +** An [sqlite3_file] object represents an open file in the +** [sqlite3_vfs | OS interface layer]. Individual OS interface +** implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object +** +** Every file opened by the [sqlite3_vfs.xOpen] method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** to a non-NULL pointer, then the sqlite3_io_methods.xClose method +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
    +**
  • [SQLITE_LOCK_NONE], +**
  • [SQLITE_LOCK_SHARED], +**
  • [SQLITE_LOCK_RESERVED], +**
  • [SQLITE_LOCK_PENDING], or +**
  • [SQLITE_LOCK_EXCLUSIVE]. +**
+** xLock() increases the lock. xUnlock() decreases the lock. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns true +** if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [file control opcodes | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
    +**
  • [SQLITE_IOCAP_ATOMIC] +**
  • [SQLITE_IOCAP_ATOMIC512] +**
  • [SQLITE_IOCAP_ATOMIC1K] +**
  • [SQLITE_IOCAP_ATOMIC2K] +**
  • [SQLITE_IOCAP_ATOMIC4K] +**
  • [SQLITE_IOCAP_ATOMIC8K] +**
  • [SQLITE_IOCAP_ATOMIC16K] +**
  • [SQLITE_IOCAP_ATOMIC32K] +**
  • [SQLITE_IOCAP_ATOMIC64K] +**
  • [SQLITE_IOCAP_SAFE_APPEND] +**
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] +**
  • [SQLITE_IOCAP_BATCH_ATOMIC] +**
+** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); + int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file*); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +**
    +**
  • [[SQLITE_FCNTL_LOCKSTATE]] +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. This capability +** is used during testing and is only available when the SQLITE_TEST +** compile-time option is used. +** +**
  • [[SQLITE_FCNTL_SIZE_HINT]] +** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS +** layer a hint of how large the database file will grow to be during the +** current transaction. This hint is not guaranteed to be accurate but it +** is often close. The underlying VFS might choose to preallocate database +** file space based on this hint in order to help writes to the database +** file run faster. +** +**
  • [[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** +**
  • [[SQLITE_FCNTL_CHUNK_SIZE]] +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. +** +**
  • [[SQLITE_FCNTL_FILE_POINTER]] +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +**
  • [[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. +** +**
  • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
  • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions super-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer is the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
  • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log ([WAL file]) and shared memory +** files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
  • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
  • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
  • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
  • [[SQLITE_FCNTL_VFS_POINTER]] +** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level +** [VFSes] currently in use. ^(The argument X in +** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be +** of type "[sqlite3_vfs] **". This opcodes will set *X +** to a pointer to the top-level VFS.)^ +** ^When there are multiple VFS shims in the stack, this opcode finds the +** upper-most shim only. +** +**
  • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement if result string is NULL, or that returns a copy +** of the result string if the string is non-NULL. +** ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connection's busy-handler callback. The argument is of type (void**) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connection's +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
  • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
  • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** +**
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
  • [[SQLITE_FCNTL_WAL_BLOCK]] +** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might +** be advantageous to block on the next WAL lock if the lock is not immediately +** available. The WAL subsystem issues this signal during rare +** circumstances in order to fix a problem with priority inversion. +** Applications should not use this file-control. +** +**
  • [[SQLITE_FCNTL_ZIPVFS]] +** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other +** VFS should return SQLITE_NOTFOUND for this opcode. +** +**
  • [[SQLITE_FCNTL_RBU]] +** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by +** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for +** this opcode. +** +**
  • [[SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]] +** If the [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] opcode returns SQLITE_OK, then +** the file descriptor is placed in "batch write mode", which +** means all subsequent write operations will be deferred and done +** atomically at the next [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. Systems +** that do not support batch atomic writes will return SQLITE_NOTFOUND. +** ^Following a successful SQLITE_FCNTL_BEGIN_ATOMIC_WRITE and prior to +** the closing [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] or +** [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE], SQLite will make +** no VFS interface calls on the same [sqlite3_file] file descriptor +** except for calls to the xWrite method and the xFileControl method +** with [SQLITE_FCNTL_SIZE_HINT]. +** +**
  • [[SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be performed atomically. +** This file control returns [SQLITE_OK] if and only if the writes were +** all performed successfully and have been committed to persistent storage. +** ^Regardless of whether or not it is successful, this file control takes +** the file descriptor out of batch write mode so that all subsequent +** write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_COMMIT_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back. +** ^This file control takes the file descriptor out of batch write mode +** so that all subsequent write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_LOCK_TIMEOUT]] +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS +** to block for up to M milliseconds before failing when attempting to +** obtain a file lock using the xLock or xShmLock methods of the VFS. +** The parameter is a pointer to a 32-bit signed integer that contains +** the value that M is to be set to. Before returning, the 32-bit signed +** integer is overwritten with the previous value of M. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provides a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. +** +**
  • [[SQLITE_FCNTL_CKPT_START]] +** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint +** in wal mode before the client starts to copy pages from the wal +** file to the database file. +** +**
  • [[SQLITE_FCNTL_CKPT_DONE]] +** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint +** in wal mode after the client has finished copying pages from the wal +** file to the database file, but before the *-shm file is updated to +** record the fact that the pages have been checkpointed. +**
+*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_WAL_BLOCK 24 +#define SQLITE_FCNTL_ZIPVFS 25 +#define SQLITE_FCNTL_RBU 26 +#define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 +#define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31 +#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 +#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 +#define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 +#define SQLITE_FCNTL_CKPT_START 39 + +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO + + +/* +** CAPI3REF: Mutex Handle +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** CAPI3REF: OS Interface Object +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. +** +** The VFS interface is sometimes extended by adding new methods onto +** the end. Each time such an extension occurs, the iVersion field +** is incremented. The iVersion value started out as 1 in +** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2 +** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased +** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields +** may be appended to the sqlite3_vfs object and the iVersion value +** may increase again in future versions of SQLite. +** Note that due to an oversight, the structure +** of the sqlite3_vfs object changed in the transition from +** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +** and yet the iVersion field was not increased. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** [[sqlite3_vfs.xOpen]] +** ^SQLite guarantees that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 11 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentence, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** ^(SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
    +**
  • [SQLITE_OPEN_MAIN_DB] +**
  • [SQLITE_OPEN_MAIN_JOURNAL] +**
  • [SQLITE_OPEN_TEMP_DB] +**
  • [SQLITE_OPEN_TEMP_JOURNAL] +**
  • [SQLITE_OPEN_TRANSIENT_DB] +**
  • [SQLITE_OPEN_SUBJOURNAL] +**
  • [SQLITE_OPEN_SUPER_JOURNAL] +**
  • [SQLITE_OPEN_WAL] +**
)^ +** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
    +**
  • [SQLITE_OPEN_DELETEONCLOSE] +**
  • [SQLITE_OPEN_EXCLUSIVE] +**
+** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. +** +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** with the [SQLITE_OPEN_CREATE] flag, which are both directly +** analogous to the O_EXCL and O_CREAT flags of the POSIX open() +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** SQLITE_OPEN_CREATE, is used to indicate that file should always +** be created, and that it is an error if it already exists. +** It is not used to indicate the file should be opened +** for exclusive access. +** +** ^At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. Note that +** the xOpen method must set the sqlite3_file.pMethods to either +** a valid [sqlite3_io_methods] object or to NULL. xOpen must do +** this even if the open fails. SQLite expects that the sqlite3_file.pMethods +** element will be valid after xOpen returns regardless of the success +** or failure of the xOpen call. +** +** [[sqlite3_vfs.xAccess]] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. +** +** ^SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), xCurrentTime(), and xCurrentTimeInt64() +** interfaces are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. ^The xCurrentTime() +** method returns a Julian Day Number for the current date and time as +** a floating point value. +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). +** ^SQLite will use the xCurrentTimeInt64() method to get the current +** date and time if that method is available (if iVersion is 2 or +** greater and the function pointer is not NULL) and will fall back +** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in future versions. The iVersion + ** value will increment whenever this happens. + */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ + +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants define the various locking operations +** allowed by the xShmLock method of [sqlite3_io_methods]. The +** following are the only legal combinations of flags to the +** xShmLock method: +** +**
    +**
  • SQLITE_SHM_LOCK | SQLITE_SHM_SHARED +**
  • SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE +**
  • SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED +**
  • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE +**
+** +** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as +** was given on the corresponding lock. +** +** The xShmLock method can transition between unlocked and SHARED or +** between unlocked and EXCLUSIVE. It cannot transition between SHARED +** and EXCLUSIVE. +*/ +#define SQLITE_SHM_UNLOCK 1 +#define SQLITE_SHM_LOCK 2 +#define SQLITE_SHM_SHARED 4 +#define SQLITE_SHM_EXCLUSIVE 8 + +/* +** CAPI3REF: Maximum xShmLock index +** +** The xShmLock method on [sqlite3_io_methods] may use values +** between 0 and this upper bound as its "offset" argument. +** The SQLite core will never attempt to acquire or release a +** lock outside of this range +*/ +#define SQLITE_SHM_NLOCK 8 + + +/* +** CAPI3REF: Initialize The SQLite Library +** +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** These routines are designed to aid in process initialization and +** shutdown on embedded systems. Workstation applications using +** SQLite normally do not need to invoke either of these routines. +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). ^(Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops.)^ +** +** A call to sqlite3_shutdown() is an "effective" call if it is the first +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only +** an effective call to sqlite3_shutdown() does any deinitialization. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ +** +** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() +** is not. The sqlite3_shutdown() interface must only be called from a +** single thread. All open [database connections] must be closed and all +** other SQLite resources must be deallocated prior to invoking +** sqlite3_shutdown(). +** +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() +** will invoke sqlite3_os_end(). +** +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** ^The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for Unix, Windows, or OS/2. +** When [custom builds | built for other platforms] +** (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. +** +** The sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** The first argument to sqlite3_config() is an integer +** [configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [configuration option] +** in the first argument. +** +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +*/ +SQLITE_API int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections +** METHOD: sqlite3 +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). +** +** The second argument to sqlite3_db_config(D,V,...) is the +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. +** +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. +*/ +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** By creating an instance of this object +** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) +** during configuration, an application can specify an alternative +** memory allocation subsystem for SQLite to use for all of its +** dynamic memory needs. +** +** Note that SQLite comes with several [built-in memory allocators] +** that are perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to +** xRealloc is always a value returned by a prior call to xRoundup. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** Every memory allocation request coming in through [sqlite3_malloc()] +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** that causes the corresponding memory allocation to fail. +** +** The xInit method initializes the memory allocator. For example, +** it might allocate any required mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +** +** SQLite holds the [SQLITE_MUTEX_STATIC_MAIN] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. For all other methods, SQLite +** holds the [SQLITE_MUTEX_STATIC_MEM] mutex as long as the +** [SQLITE_CONFIG_MEMSTATUS] configuration option is turned on (which +** it is by default) and so the methods are automatically serialized. +** However, if [SQLITE_CONFIG_MEMSTATUS] is disabled, then the other +** methods must be threadsafe or else make their own arrangements for +** serialization. +** +** SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option.
+** +** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option.
+** +** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
+**
There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option.
+** +** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
+**
^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns.
+** +** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
+**
^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines.)^ +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
+** +** [[SQLITE_CONFIG_SMALL_MALLOC]]
SQLITE_CONFIG_SMALL_MALLOC
+**
^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** type int, interpreted as a boolean, which if true provides a hint to +** SQLite that it should avoid large memory allocations if possible. +** SQLite will run faster if it is free to make large memory allocations, +** but some application might prefer to run slower in exchange for +** guarantees about memory fragmentation that are possible if large +** allocations are avoided. This hint is normally off. +**
+** +** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
+**
^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: +**
    +**
  • [sqlite3_hard_heap_limit64()] +**
  • [sqlite3_memory_used()] +**
  • [sqlite3_memory_highwater()] +**
  • [sqlite3_soft_heap_limit64()] +**
  • [sqlite3_status64()] +**
)^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. +**
+** +** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
+**
The SQLITE_CONFIG_SCRATCH option is no longer used. +**
+** +** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
+**
^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool +** that SQLite can use for the database page cache with the default page +** cache implementation. +** This configuration option is a no-op if an application-defined page +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned memory (pMem), the size of each page cache line (sz), +** and the number of cache lines (N). +** The sz argument should be the size of the largest database page +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The pMem +** argument must be either a NULL pointer or a pointer to an 8-byte +** aligned block of memory of at least sz*N bytes, otherwise +** subsequent behavior is undefined. +** ^When pMem is not NULL, SQLite will strive to use the memory provided +** to satisfy page cache needs, falling back to [sqlite3_malloc()] if +** a page cache line is larger than sz bytes or if all of the pMem buffer +** is exhausted. +** ^If pMem is NULL and N is non-zero, then each database connection +** does an initial bulk allocation for page cache memory +** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or +** of -1024*N bytes if N is negative, . ^If additional +** page cache memory is needed beyond what is provided by the initial +** allocation, then SQLite goes to [sqlite3_malloc()] separately for each +** additional cache line.
+** +** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
+**
^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, +** the number of bytes in the memory buffer, and the minimum allocation size. +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the +** memory pointer is not NULL then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs. +** The first pointer (the memory pointer) must be aligned to an 8-byte +** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8.
+** +** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
+**
^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR].
+** +** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
+**
^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines.)^ +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR].
+** +** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
+**
^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside +** configuration on individual connections.)^
+** +** [[SQLITE_CONFIG_PCACHE2]]
SQLITE_CONFIG_PCACHE2
+**
^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
+** +** [[SQLITE_CONFIG_GETPCACHE2]]
SQLITE_CONFIG_GETPCACHE2
+**
^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
+** +** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
+**
The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,int,const char*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked by [sqlite3_log()] to process each logging event. ^If the +** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. +** ^The void pointer that is the second argument to SQLITE_CONFIG_LOG is +** passed through as the first parameter to the application-defined logger +** function whenever that function is invoked. ^The second parameter to +** the logger function is a copy of the first parameter to the corresponding +** [sqlite3_log()] call and is intended to be a [result code] or an +** [extended result code]. ^The third parameter passed to the logger is +** log message after formatting via [sqlite3_snprintf()]. +** The SQLite logging interface is not reentrant; the logger function +** supplied by the application must not invoke any SQLite interface. +** In a multi-threaded application, the application-defined logger +** function must be threadsafe.
+** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. ^If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. ^(By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
SQLITE_CONFIG_COVERING_INDEX_SCAN +**
^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
+** +** [[SQLITE_CONFIG_SQLLOG]] +**
SQLITE_CONFIG_SQLLOG +**
This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
+** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
SQLITE_CONFIG_MMAP_SIZE +**
^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
SQLITE_CONFIG_WIN32_HEAPSIZE +**
^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
SQLITE_CONFIG_PCACHE_HDRSZ +**
^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** +** [[SQLITE_CONFIG_PMASZ]] +**
SQLITE_CONFIG_PMASZ +**
^The SQLITE_CONFIG_PMASZ option takes a single parameter which +** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded +** sorter to that integer. The default minimum PMA Size is set by the +** [SQLITE_SORTER_PMASZ] compile-time option. New threads are launched +** to help with sort operations when multithreaded sorting +** is enabled (using the [PRAGMA threads] command) and the amount of content +** to be sorted exceeds the page size times the minimum of the +** [PRAGMA cache_size] setting and this value. +** +** [[SQLITE_CONFIG_STMTJRNL_SPILL]] +**
SQLITE_CONFIG_STMTJRNL_SPILL +**
^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which +** becomes the [statement journal] spill-to-disk threshold. +** [Statement journals] are held in memory until their size (in bytes) +** exceeds this threshold, at which point they are written to disk. +** Or if the threshold is -1, statement journals are always held +** exclusively in memory. +** Since many statement journals never become large, setting the spill +** threshold to a value such as 64KiB can greatly reduce the amount of +** I/O required to support statement rollback. +** The default value for this setting is controlled by the +** [SQLITE_STMTJRNL_SPILL] compile-time option. +** +** [[SQLITE_CONFIG_SORTERREF_SIZE]] +**
SQLITE_CONFIG_SORTERREF_SIZE +**
The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted record and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behaviour. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +**
SQLITE_CONFIG_MEMDB_MAXSIZE +**
The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. +**
+*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ +#define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ +#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ +#define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ + +/* +** CAPI3REF: Database Connection Configuration Options +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. ^The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+** [[SQLITE_DBCONFIG_LOOKASIDE]] +**
SQLITE_DBCONFIG_LOOKASIDE
+**
^This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** ^The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments. The buffer +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
+** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] +**
SQLITE_DBCONFIG_ENABLE_FKEY
+**
^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
+** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] +**
SQLITE_DBCONFIG_ENABLE_TRIGGER
+**
^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back. +** +**

Originally this option disabled all triggers. ^(However, since +** SQLite version 3.35.0, TEMP triggers are still allowed even if +** this option is off. So, in other words, this option now only disables +** triggers in the main database schema or in the schemas of ATTACH-ed +** databases.)^

+** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
SQLITE_DBCONFIG_ENABLE_VIEW
+**
^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back. +** +**

Originally this option disabled all views. ^(However, since +** SQLite version 3.35.0, TEMP views are still allowed even if +** this option is off. So, in other words, this option now only disables +** views in the main database schema or in the schemas of ATTACH-ed +** databases.)^

+** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] +**
SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
+**
^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the +** [FTS3] full-text search engine extension. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable fts3_tokenizer() or +** positive to enable fts3_tokenizer() or negative to leave the setting +** unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the new setting is not reported back.
+** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] +**
SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
+**
^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argument to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
+** +** [[SQLITE_DBCONFIG_MAINDBNAME]]
SQLITE_DBCONFIG_MAINDBNAME
+**
^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
+** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] +**
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
+**
Usually, when a database in wal mode is closed or detached from a +** database handle, SQLite checks if this will mean that there are now no +** connections at all to the database. If so, it performs a checkpoint +** operation before closing the connection. This option may be used to +** override this behaviour. The first parameter passed to this operation +** is an integer - positive to disable checkpoints-on-close, or zero (the +** default) to enable them, and negative to leave the setting unchanged. +** The second parameter is a pointer to an integer +** into which is written 0 or 1 to indicate whether checkpoints-on-close +** have been disabled - 0 if they are not disabled, 1 if they are. +**
+** +** [[SQLITE_DBCONFIG_ENABLE_QPSG]]
SQLITE_DBCONFIG_ENABLE_QPSG
+**
^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates +** the [query planner stability guarantee] (QPSG). When the QPSG is active, +** a single SQL query statement will always use the same algorithm regardless +** of values of [bound parameters].)^ The QPSG disables some query optimizations +** that look at the values of bound parameters, which can make some queries +** slower. But the QPSG has the advantage of more predictable behavior. With +** the QPSG active, SQLite will always use the same query plan in the field as +** was used during testing in the lab. +** The first argument to this setting is an integer which is 0 to disable +** the QPSG, positive to enable QPSG, or negative to leave the setting +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether the QPSG is disabled or enabled +** following this call. +**
+** +** [[SQLITE_DBCONFIG_TRIGGER_EQP]]
SQLITE_DBCONFIG_TRIGGER_EQP
+**
By default, the output of EXPLAIN QUERY PLAN commands does not +** include output for any operations performed by trigger programs. This +** option is used to set or clear (the default) a flag that governs this +** behavior. The first parameter passed to this operation is an integer - +** positive to enable output for trigger programs, or zero to disable it, +** or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which is written +** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if +** it is not disabled, 1 if it is. +**
+** +** [[SQLITE_DBCONFIG_RESET_DATABASE]]
SQLITE_DBCONFIG_RESET_DATABASE
+**
Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run +** [VACUUM] in order to reset a database back to an empty database +** with no schema and no content. The following process works even for +** a badly corrupted database file: +**
    +**
  1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. +**
  2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); +**
  3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); +**
  4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); +**
+** Because resetting a database is destructive and irreversible, the +** process requires the use of this obscure API and multiple steps to help +** ensure that it does not happen by accident. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]]
SQLITE_DBCONFIG_DEFENSIVE
+**
The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +**
    +**
  • The [PRAGMA writable_schema=ON] statement. +**
  • The [PRAGMA journal_mode=OFF] statement. +**
  • Writes to the [sqlite_dbpage] virtual table. +**
  • Direct writes to [shadow tables]. +**
+**
+** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
SQLITE_DBCONFIG_WRITABLE_SCHEMA
+**
The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
+** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
+**
The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
+** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
SQLITE_DBCONFIG_DQS_DML +**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statements +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
+** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
SQLITE_DBCONFIG_DQS_DDL +**
The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
+** +** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] +**
SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to +** assume that database schemas are untainted by malicious content. +** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite +** takes additional defensive steps to protect the application from harm +** including: +**
    +**
  • Prohibit the use of SQL functions inside triggers, views, +** CHECK constraints, DEFAULT clauses, expression indexes, +** partial indexes, or generated columns +** unless those functions are tagged with [SQLITE_INNOCUOUS]. +**
  • Prohibit the use of virtual tables inside of triggers or views +** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. +**
+** This setting defaults to "on" for legacy compatibility, however +** all applications are advised to turn it off if possible. This setting +** can also be controlled using the [PRAGMA trusted_schema] statement. +**
+** +** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] +**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates +** the legacy file format flag. When activated, this flag causes all newly +** created database file to have a schema format version number (the 4-byte +** integer found at offset 44 into the database header) of 1. This in turn +** means that the resulting database file will be readable and writable by +** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, +** newly created databases are generally not understandable by SQLite versions +** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there +** is now scarcely any need to generated database files that are compatible +** all the way back to version 3.0.0, and so this setting is of little +** practical use, but is provided so that SQLite can continue to claim the +** ability to generate new database files that are compatible with version +** 3.0.0. +**

Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, +** the [VACUUM] command will fail with an obscure error when attempting to +** process a table with generated columns and a descending index. This is +** not considered a bug since SQLite versions 3.3.0 and earlier do not support +** either generated columns or decending indexes. +**

+**
+*/ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ +#define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ +#define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ +#define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ +#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ + +/* +** CAPI3REF: Enable Or Disable Extended Result Codes +** METHOD: sqlite3 +** +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); + +/* +** CAPI3REF: Last Insert Rowid +** METHOD: sqlite3 +** +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. +** +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. +** +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ +** +** ^An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. ^(When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface.)^ +** +** ^For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** +** If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. +*/ +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); + +/* +** CAPI3REF: Count The Number Of Rows Modified +** METHOD: sqlite3 +** +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. +** +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. +** +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
    +**
  • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
  • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
+** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. +** +** See also: +**
    +**
  • the [sqlite3_total_changes()] interface +**
  • the [count_changes pragma] +**
  • the [changes() SQL function] +**
  • the [data_version pragma] +**
+*/ +SQLITE_API int sqlite3_changes(sqlite3*); + +/* +** CAPI3REF: Total Number Of Rows Modified +** METHOD: sqlite3 +** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** +** The [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. +** +** See also: +**
    +**
  • the [sqlite3_changes()] interface +**
  • the [count_changes pragma] +**
  • the [changes() SQL function] +**
  • the [data_version pragma] +**
  • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
+*/ +SQLITE_API int sqlite3_total_changes(sqlite3*); + +/* +** CAPI3REF: Interrupt A Long-Running Query +** METHOD: sqlite3 +** +** ^This function causes any pending database operation to abort and +** return at its earliest opportunity. This routine is typically +** called in response to a user action such as pressing "Cancel" +** or Ctrl-C where the user wants a long query operation to halt +** immediately. +** +** ^It is safe to call this routine from a thread different from the +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. +** +** ^If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements +** that are started after the sqlite3_interrupt() call and before the +** running statement count reaches zero are interrupted as if they had been +** running prior to the sqlite3_interrupt() call. ^New SQL statements +** that are started after the running statement count reaches zero are +** not effected by the sqlite3_interrupt(). +** ^A call to sqlite3_interrupt(D) that occurs when there are no running +** SQL statements is a no-op and has no effect on SQL statements +** that are started after the sqlite3_interrupt() call returns. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3*); + +/* +** CAPI3REF: Determine If An SQL Statement Is Complete +** +** These routines are useful during command-line input to determine if the +** currently entered text seems to form a complete SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be +** complete if it ends with a semicolon token and is not a prefix of a +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. ^Whitespace +** and comments that follow the final semicolon are ignored. +** +** ^These routines return 0 if the statement is incomplete. ^If a +** memory allocation fails, then SQLITE_NOMEM is returned. +** +** ^These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked +** automatically by sqlite3_complete16(). If that initialization fails, +** then the return value from sqlite3_complete16() will be non-zero +** regardless of whether or not the input SQL is complete.)^ +** +** The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. +*/ +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); + +/* +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} +** METHOD: sqlite3 +** +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. +** +** ^If the busy callback is NULL, then [SQLITE_BUSY] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. +** +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked previously for the same locking event. ^If the +** busy callback returns 0, then no additional attempts are made to +** access the database and [SQLITE_BUSY] is returned +** to the application. +** ^If the callback returns non-zero, then another attempt +** is made to access the database and the cycle repeats. +** +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. ^If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** to the application instead of invoking the +** busy handler. +** Consider a scenario where one process is holding a read lock that +** it is trying to promote to a reserved lock and +** a second process is holding a reserved lock that it is trying +** to promote to an exclusive lock. The first process cannot proceed +** because it is blocked by the second and the second process cannot +** proceed because it is blocked by the first. If both processes +** invoke the busy handlers, neither will make any progress. Therefore, +** SQLite returns [SQLITE_BUSY] for the first process, hoping that this +** will induce the first process to release its read lock and allow +** the second process to proceed. +** +** ^The default busy callback is NULL. +** +** ^(There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. +** +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions +** result in undefined behavior. +** +** A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. +*/ +SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); + +/* +** CAPI3REF: Set A Busy Timeout +** METHOD: sqlite3 +** +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. ^After at least "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY]. +** +** ^Calling this routine with an argument less than or equal to zero +** turns off all busy handlers. +** +** ^(There can only be a single busy handler for a particular +** [database connection] at any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling +** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); + +/* +** CAPI3REF: Convenience Routines For Running Queries +** METHOD: sqlite3 +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. +** +** The table conceptually has a number of rows and columns. But +** these numbers are not part of the result table itself. These +** numbers are obtained separately. Let N be the number of rows +** and M be the number of columns. +** +** A result table is an array of pointers to zero-terminated UTF-8 strings. +** There are (N+1)*M elements in the array. The first M pointers point +** to zero-terminated strings that contain the names of the columns. +** The remaining entries all point to query results. NULL values result +** in NULL pointers. All other values are in their UTF-8 zero-terminated +** string representation as returned by [sqlite3_column_text()]. +** +** A result table might consist of one or more memory allocations. +** It is not safe to pass a result table directly to [sqlite3_free()]. +** A result table should be deallocated using [sqlite3_free_table()]. +** +** ^(As an example of the result table format, suppose a query result +** is as follows: +** +** 
+**        Name        | Age
+**        -----------------------
+**        Alice       | 43
+**        Bob         | 28
+**        Cindy       | 21
+**
+** +** There are two columns (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array named azResult. Then azResult holds this content: +** +** 
+**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
)^ +** +** ^The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter and returns a result table to the +** pointer given in its 3rd parameter. +** +** After the application has finished with the result from sqlite3_get_table(), +** it must pass the result table pointer to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. +** +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()]. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ +); +SQLITE_API void sqlite3_free_table(char **result); + +/* +** CAPI3REF: Formatted String Printing Functions +** +** These routines are work-alikes of the "printf()" family of functions +** from the standard C library. +** These routines understand most of the common formatting options from +** the standard library printf() +** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]). +** See the [built-in printf()] documentation for details. +** +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** results into memory obtained from [sqlite3_malloc64()]. +** The strings returned by these two routines should be +** released by [sqlite3_free()]. ^Both routines return a +** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough +** memory to hold the resulting string. +** +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from +** the standard C library. The result is written into the +** buffer supplied as the second parameter whose size is given by +** the first parameter. Note that the order of the +** first two parameters is reversed from snprintf().)^ This is an +** historical accident that cannot be fixed without breaking +** backwards compatibility. ^(Note also that sqlite3_snprintf() +** returns a pointer to its buffer instead of the number of +** characters actually written into the buffer.)^ We admit that +** the number of characters written would be a more useful return +** value but we cannot change the implementation of sqlite3_snprintf() +** now without breaking compatibility. +** +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first +** parameter "n" is the total size of the buffer, including space for +** the zero terminator. So the longest string that can be completely +** written will be n-1 characters. +** +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). +** +** See also: [built-in printf()], [printf() SQL function] +*/ +SQLITE_API char *sqlite3_mprintf(const char*,...); +SQLITE_API char *sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); + +/* +** CAPI3REF: Memory Allocation Subsystem +** +** The SQLite core uses these three routines for all of its own +** internal memory allocation needs. "Core" in the previous sentence +** does not include operating-system specific [VFS] implementation. The +** Windows VFS uses native malloc() and free() for some operations. +** +** ^The sqlite3_malloc() routine returns a pointer to a block +** of memory at least N bytes in length, where N is the parameter. +** ^If sqlite3_malloc() is unable to obtain sufficient free +** memory, it returns a NULL pointer. ^If the parameter N to +** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns +** a NULL pointer. +** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** +** ^Calling sqlite3_free() with a pointer previously returned +** by sqlite3_malloc() or sqlite3_realloc() releases that memory so +** that it might be reused. ^The sqlite3_free() routine is +** a no-op if is called with a NULL pointer. Passing a NULL pointer +** to sqlite3_free() is harmless. After being freed, memory +** should neither be read nor written. Even reading previously freed +** memory might result in a segmentation fault or other severe error. +** Memory corruption, a segmentation fault, or other severe error +** might result if sqlite3_free() is called with a non-NULL pointer that +** was not obtained from sqlite3_malloc() or sqlite3_realloc(). +** +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) +** is a NULL pointer then its behavior is identical to calling +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or +** negative then the behavior is exactly the same as calling +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. +** ^If M is the size of the prior allocation, then min(N,M) bytes +** of the prior allocation are copied into the beginning of buffer returned +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. +** +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. +** +** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] +** must be either NULL or else pointers obtained from a prior +** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +** not yet been released. +** +** The application must not read or write any part of +** a block of memory after it has been released using +** [sqlite3_free()] or [sqlite3_realloc()]. +*/ +SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void sqlite3_free(void*); +SQLITE_API sqlite3_uint64 sqlite3_msize(void*); + +/* +** CAPI3REF: Memory Allocator Statistics +** +** SQLite provides these two interfaces for reporting on the status +** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()] +** routines, which form the built-in memory allocation subsystem. +** +** ^The [sqlite3_memory_used()] routine returns the number of bytes +** of memory currently outstanding (malloced but not freed). +** ^The [sqlite3_memory_highwater()] routine returns the maximum +** value of [sqlite3_memory_used()] since the high-water mark +** was last reset. ^The values returned by [sqlite3_memory_used()] and +** [sqlite3_memory_highwater()] include any overhead +** added by SQLite in its implementation of [sqlite3_malloc()], +** but not overhead added by the any underlying system library +** routines that [sqlite3_malloc()] may call. +** +** ^The memory high-water mark is reset to the current value of +** [sqlite3_memory_used()] if and only if the parameter to +** [sqlite3_memory_highwater()] is true. ^The value returned +** by [sqlite3_memory_highwater(1)] is the high-water mark +** prior to the reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); + +/* +** CAPI3REF: Pseudo-Random Number Generator +** +** SQLite contains a high-quality pseudo-random number generator (PRNG) used to +** select random [ROWID | ROWIDs] when inserting new records into a table that +** already uses the largest possible [ROWID]. The PRNG is also used for +** the built-in random() and randomblob() SQL functions. This interface allows +** applications to access the same PRNG for other purposes. +** +** ^A call to this routine stores N bytes of randomness into buffer P. +** ^The P parameter can be a NULL pointer. +** +** ^If this routine has not been previously called or if the previous +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated +** internally and without recourse to the [sqlite3_vfs] xRandomness +** method. +*/ +SQLITE_API void sqlite3_randomness(int N, void *P); + +/* +** CAPI3REF: Compile-Time Authorization Callbacks +** METHOD: sqlite3 +** KEYWORDS: {authorizer callback} +** +** ^This routine registers an authorizer callback with a particular +** [database connection], supplied in the first argument. +** ^The authorizer callback is invoked as SQL statements are being compiled +** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], +** [sqlite3_prepare_v3()], [sqlite3_prepare16()], [sqlite3_prepare16_v2()], +** and [sqlite3_prepare16_v3()]. ^At various +** points during the compilation process, as logic is being created +** to perform various actions, the authorizer callback is invoked to +** see if those actions are allowed. ^The authorizer callback should +** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the +** specific action but allow the SQL statement to continue to be +** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be +** rejected with an error. ^If the authorizer callback returns +** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY] +** then the [sqlite3_prepare_v2()] or equivalent call that triggered +** the authorizer will fail with an error message. +** +** When the callback returns [SQLITE_OK], that means the operation +** requested is ok. ^When the callback returns [SQLITE_DENY], the +** [sqlite3_prepare_v2()] or equivalent call that triggered the +** authorizer will fail with an error message explaining that +** access is denied. +** +** ^The first parameter to the authorizer callback is a copy of the third +** parameter to the sqlite3_set_authorizer() interface. ^The second parameter +** to the callback is an integer [SQLITE_COPY | action code] that specifies +** the particular action to be authorized. ^The third through sixth parameters +** to the callback are either NULL pointers or zero-terminated strings +** that contain additional details about the action to be authorized. +** Applications must always be prepared to encounter a NULL pointer in any +** of the third through the sixth parameters of the authorization callback. +** +** ^If the action code is [SQLITE_READ] +** and the callback returns [SQLITE_IGNORE] then the +** [prepared statement] statement is constructed to substitute +** a NULL value in place of the table column that would have +** been read if [SQLITE_OK] had been returned. The [SQLITE_IGNORE] +** return can be used to deny an untrusted user access to individual +** columns of a table. +** ^When a table is referenced by a [SELECT] but no column values are +** extracted from that table (for example in a query like +** "SELECT count(*) FROM tab") then the [SQLITE_READ] authorizer callback +** is invoked once for that table with a column name that is an empty string. +** ^If the action code is [SQLITE_DELETE] and the callback returns +** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the +** [truncate optimization] is disabled and all rows are deleted individually. +** +** An authorizer is used when [sqlite3_prepare | preparing] +** SQL statements from an untrusted source, to ensure that the SQL statements +** do not try to access data they are not allowed to see, or that they do not +** try to execute malicious statements that damage the database. For +** example, an application may allow a user to enter arbitrary +** SQL queries for evaluation by a database. But the application does +** not want the user to be able to make arbitrary changes to the +** database. An authorizer could then be put in place while the +** user-entered SQL is being [sqlite3_prepare | prepared] that +** disallows everything except [SELECT] statements. +** +** Applications that need to process SQL from untrusted sources +** might also consider lowering resource limits using [sqlite3_limit()] +** and limiting database size using the [max_page_count] [PRAGMA] +** in addition to using an authorizer. +** +** ^(Only a single authorizer can be in place on a database connection +** at a time. Each call to sqlite3_set_authorizer overrides the +** previous call.)^ ^Disable the authorizer by installing a NULL callback. +** The authorizer is disabled by default. +** +** The authorizer callback must not do anything that will modify +** the database connection that invoked the authorizer callback. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the +** statement might be re-prepared during [sqlite3_step()] due to a +** schema change. Hence, the application should ensure that the +** correct authorizer callback remains in place during the [sqlite3_step()]. +** +** ^Note that the authorizer callback is invoked only during +** [sqlite3_prepare()] or its variants. Authorization is not +** performed during statement evaluation in [sqlite3_step()], unless +** as stated in the previous paragraph, sqlite3_step() invokes +** sqlite3_prepare_v2() to reprepare a statement after a schema change. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3*, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pUserData +); + +/* +** CAPI3REF: Authorizer Return Codes +** +** The [sqlite3_set_authorizer | authorizer callback function] must +** return either [SQLITE_OK] or one of these two constants in order +** to signal SQLite whether or not the action is permitted. See the +** [sqlite3_set_authorizer | authorizer documentation] for additional +** information. +** +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. +*/ +#define SQLITE_DENY 1 /* Abort the SQL statement with an error */ +#define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ + +/* +** CAPI3REF: Authorizer Action Codes +** +** The [sqlite3_set_authorizer()] interface registers a callback function +** that is invoked to authorize certain SQL statement actions. The +** second parameter to the callback is an integer code that specifies +** what action is being authorized. These are the integer action codes that +** the authorizer callback may be passed. +** +** These action code values signify what kind of operation is to be +** authorized. The 3rd and 4th parameters to the authorization +** callback function will be parameters or NULL depending on which of these +** codes is used as the second parameter. ^(The 5th parameter to the +** authorizer callback is the name of the database ("main", "temp", +** etc.) if applicable.)^ ^The 6th parameter to the authorizer callback +** is the name of the inner-most trigger or view that is responsible for +** the access attempt or NULL if this access attempt is directly from +** top-level SQL code. +*/ +/******************************************* 3rd ************ 4th ***********/ +#define SQLITE_CREATE_INDEX 1 /* Index Name Table Name */ +#define SQLITE_CREATE_TABLE 2 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_INDEX 3 /* Index Name Table Name */ +#define SQLITE_CREATE_TEMP_TABLE 4 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_TRIGGER 5 /* Trigger Name Table Name */ +#define SQLITE_CREATE_TEMP_VIEW 6 /* View Name NULL */ +#define SQLITE_CREATE_TRIGGER 7 /* Trigger Name Table Name */ +#define SQLITE_CREATE_VIEW 8 /* View Name NULL */ +#define SQLITE_DELETE 9 /* Table Name NULL */ +#define SQLITE_DROP_INDEX 10 /* Index Name Table Name */ +#define SQLITE_DROP_TABLE 11 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_INDEX 12 /* Index Name Table Name */ +#define SQLITE_DROP_TEMP_TABLE 13 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_TRIGGER 14 /* Trigger Name Table Name */ +#define SQLITE_DROP_TEMP_VIEW 15 /* View Name NULL */ +#define SQLITE_DROP_TRIGGER 16 /* Trigger Name Table Name */ +#define SQLITE_DROP_VIEW 17 /* View Name NULL */ +#define SQLITE_INSERT 18 /* Table Name NULL */ +#define SQLITE_PRAGMA 19 /* Pragma Name 1st arg or NULL */ +#define SQLITE_READ 20 /* Table Name Column Name */ +#define SQLITE_SELECT 21 /* NULL NULL */ +#define SQLITE_TRANSACTION 22 /* Operation NULL */ +#define SQLITE_UPDATE 23 /* Table Name Column Name */ +#define SQLITE_ATTACH 24 /* Filename NULL */ +#define SQLITE_DETACH 25 /* Database Name NULL */ +#define SQLITE_ALTER_TABLE 26 /* Database Name Table Name */ +#define SQLITE_REINDEX 27 /* Index Name NULL */ +#define SQLITE_ANALYZE 28 /* Table Name NULL */ +#define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ +#define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ +#define SQLITE_FUNCTION 31 /* NULL Function Name */ +#define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ +#define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ + +/* +** CAPI3REF: Tracing And Profiling Functions +** METHOD: sqlite3 +** +** These routines are deprecated. Use the [sqlite3_trace_v2()] interface +** instead of the routines described here. +** +** These routines register callback functions that can be used for +** tracing and profiling the execution of SQL statements. +** +** ^The callback function registered by sqlite3_trace() is invoked at +** various times when an SQL statement is being run by [sqlite3_step()]. +** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the +** SQL statement text as the statement first begins executing. +** ^(Additional sqlite3_trace() callbacks might occur +** as each triggered subprogram is entered. The callbacks for triggers +** contain a UTF-8 SQL comment that identifies the trigger.)^ +** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** +** ^The callback function registered by sqlite3_profile() is invoked +** as each SQL statement finishes. ^The profile callback contains +** the original statement text and an estimate of wall-clock time +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. +*/ +SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, + void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, + void(*xProfile)(void*,const char*,sqlite3_uint64), void*); + +/* +** CAPI3REF: SQL Trace Event Codes +** KEYWORDS: SQLITE_TRACE +** +** These constants identify classes of events that can be monitored +** using the [sqlite3_trace_v2()] tracing logic. The M argument +** to [sqlite3_trace_v2(D,M,X,P)] is an OR-ed combination of one or more of +** the following constants. ^The first argument to the trace callback +** is one of the following constants. +** +** New tracing constants may be added in future releases. +** +** ^A trace callback has four arguments: xCallback(T,C,P,X). +** ^The T argument is one of the integer type codes above. +** ^The C argument is a copy of the context pointer passed in as the +** fourth argument to [sqlite3_trace_v2()]. +** The P and X arguments are pointers whose meanings depend on T. +** +**
+** [[SQLITE_TRACE_STMT]]
SQLITE_TRACE_STMT
+**
^An SQLITE_TRACE_STMT callback is invoked when a prepared statement +** first begins running and possibly at other times during the +** execution of the prepared statement, such as at the start of each +** trigger subprogram. ^The P argument is a pointer to the +** [prepared statement]. ^The X argument is a pointer to a string which +** is the unexpanded SQL text of the prepared statement or an SQL comment +** that indicates the invocation of a trigger. ^The callback can compute +** the same text that would have been returned by the legacy [sqlite3_trace()] +** interface by using the X argument when X begins with "--" and invoking +** [sqlite3_expanded_sql(P)] otherwise. +** +** [[SQLITE_TRACE_PROFILE]]
SQLITE_TRACE_PROFILE
+**
^An SQLITE_TRACE_PROFILE callback provides approximately the same +** information as is provided by the [sqlite3_profile()] callback. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument points to a 64-bit integer which is the estimated of +** the number of nanosecond that the prepared statement took to run. +** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. +** +** [[SQLITE_TRACE_ROW]]
SQLITE_TRACE_ROW
+**
^An SQLITE_TRACE_ROW callback is invoked whenever a prepared +** statement generates a single row of result. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument is unused. +** +** [[SQLITE_TRACE_CLOSE]]
SQLITE_TRACE_CLOSE
+**
^An SQLITE_TRACE_CLOSE callback is invoked when a database +** connection closes. +** ^The P argument is a pointer to the [database connection] object +** and the X argument is unused. +**
+*/ +#define SQLITE_TRACE_STMT 0x01 +#define SQLITE_TRACE_PROFILE 0x02 +#define SQLITE_TRACE_ROW 0x04 +#define SQLITE_TRACE_CLOSE 0x08 + +/* +** CAPI3REF: SQL Trace Hook +** METHOD: sqlite3 +** +** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback +** function X against [database connection] D, using property mask M +** and context pointer P. ^If the X callback is +** NULL or if the M mask is zero, then tracing is disabled. The +** M argument should be the bitwise OR-ed combination of +** zero or more [SQLITE_TRACE] constants. +** +** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides +** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** +** ^The X callback is invoked whenever any of the events identified by +** mask M occur. ^The integer return value from the callback is currently +** ignored, though this may change in future releases. Callback +** implementations should return zero to ensure future compatibility. +** +** ^A trace callback is invoked with four arguments: callback(T,C,P,X). +** ^The T argument is one of the [SQLITE_TRACE] +** constants to indicate why the callback was invoked. +** ^The C argument is a copy of the context pointer. +** The P and X arguments are pointers whose meanings depend on T. +** +** The sqlite3_trace_v2() interface is intended to replace the legacy +** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which +** are deprecated. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3*, + unsigned uMask, + int(*xCallback)(unsigned,void*,void*,void*), + void *pCtx +); + +/* +** CAPI3REF: Query Progress Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this +** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the approximate number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** +** ^If the progress callback returns non-zero, the operation is +** interrupted. This feature can be used to implement a +** "Cancel" button on a GUI progress dialog box. +** +** The progress handler callback must not do anything that will modify +** the database connection that invoked the progress handler. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +*/ +SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); + +/* +** CAPI3REF: Opening A New Database Connection +** CONSTRUCTOR: sqlite3 +** +** ^These routines open an SQLite database file as specified by the +** filename argument. ^The filename argument is interpreted as UTF-8 for +** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte +** order for sqlite3_open16(). ^(A [database connection] handle is usually +** returned in *ppDb, even if an error occurs. The only exception is that +** if SQLite is unable to allocate memory to hold the [sqlite3] object, +** a NULL will be written into *ppDb instead of a pointer to the [sqlite3] +** object.)^ ^(If the database is opened (and/or created) successfully, then +** [SQLITE_OK] is returned. Otherwise an [error code] is returned.)^ ^The +** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain +** an English language description of the error following a failure of any +** of the sqlite3_open() routines. +** +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. +** +** Whether or not an error occurs when it is opened, resources +** associated with the [database connection] handle should be released by +** passing it to [sqlite3_close()] when it is no longer required. +** +** The sqlite3_open_v2() interface works like sqlite3_open() +** except that it accepts two additional parameters for additional control +** over the new database connection. ^(The flags parameter to +** sqlite3_open_v2() must include, at a minimum, one of the following +** three flag combinations:)^ +** +**
+** ^(
[SQLITE_OPEN_READONLY]
+**
The database is opened in read-only mode. If the database does not +** already exist, an error is returned.
)^ +** +** ^(
[SQLITE_OPEN_READWRITE]
+**
The database is opened for reading and writing if possible, or reading +** only if the file is write protected by the operating system. In either +** case the database must already exist, otherwise an error is returned.
)^ +** +** ^(
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
+**
The database is opened for reading and writing, and is created if +** it does not already exist. This is the behavior that is always used for +** sqlite3_open() and sqlite3_open16().
)^ +**
+** +** In addition to the required flags, the following optional flags are +** also supported: +** +**
+** ^(
[SQLITE_OPEN_URI]
+**
The filename can be interpreted as a URI if this flag is set.
)^ +** +** ^(
[SQLITE_OPEN_MEMORY]
+**
The database will be opened as an in-memory database. The database +** is named by the "filename" argument for the purposes of cache-sharing, +** if shared cache mode is enabled, but the "filename" is otherwise ignored. +**
)^ +** +** ^(
[SQLITE_OPEN_NOMUTEX]
+**
The new database connection will use the "multi-thread" +** [threading mode].)^ This means that separate threads are allowed +** to use SQLite at the same time, as long as each thread is using +** a different [database connection]. +** +** ^(
[SQLITE_OPEN_FULLMUTEX]
+**
The new database connection will use the "serialized" +** [threading mode].)^ This means the multiple threads can safely +** attempt to use the same database connection at the same time. +** (Mutexes will block any actual concurrency, but in this mode +** there is no harm in trying.) +** +** ^(
[SQLITE_OPEN_SHAREDCACHE]
+**
The database is opened [shared cache] enabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** ^(
[SQLITE_OPEN_PRIVATECACHE]
+**
The database is opened [shared cache] disabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** [[OPEN_NOFOLLOW]] ^(
[SQLITE_OPEN_NOFOLLOW]
+**
The database filename is not allowed to be a symbolic link
+**
)^ +** +** If the 3rd parameter to sqlite3_open_v2() is not one of the +** required combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] +** then the behavior is undefined. +** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** +** ^If the filename is ":memory:", then a private, temporary in-memory database +** is created for the connection. ^This in-memory database will vanish when +** the database connection is closed. Future versions of SQLite might +** make use of additional special filenames that begin with the ":" character. +** It is recommended that when a database filename actually does begin with +** a ":" character you should prefix the filename with a pathname such as +** "./" to avoid ambiguity. +** +** ^If the filename is an empty string, then a private, temporary +** on-disk database will be created. ^This private database will be +** automatically deleted as soon as the database connection is closed. +** +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the third argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite and its built-in [VFSes] interpret the +** following query parameters: +** +**
    +**
  • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
  • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_open_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). +** +**
  • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behavior requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter indicates whether or not the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** +**
+** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +**
+**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +** Use "ro" instead: "file:data.db?mode=ro". +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. +** +** Note to Windows users: The encoding used for the filename argument +** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever +** codepage is currently defined. Filenames containing international +** characters must be converted to UTF-8 prior to passing them into +** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] +*/ +SQLITE_API int sqlite3_open( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open16( + const void *filename, /* Database filename (UTF-16) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** These are utility routines, useful to [VFS|custom VFS implementations], +** that check if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of that query parameter. +** +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
    +**
  • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implemention, or +**
  • A filename obtained from [sqlite3_db_filename()], or +**
  • A new filename constructed using [sqlite3_create_filename()]. +**
+** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) +** and if P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F and it +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** The sqlite3_uri_key(F,N) returns a pointer to the name (not +** the value) of the N-th query parameter for filename F, or a NULL +** pointer if N is less than zero or greater than the number of query +** parameters minus 1. The N value is zero-based so N should be 0 to obtain +** the name of the first query parameter, 1 for the second parameter, and +** so forth. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that the SQLite core passed +** into the xOpen VFS method, then the behavior of this routine is undefined +** and probably undesirable. +** +** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F +** parameter can also be the name of a rollback journal file or WAL file +** in addition to the main database file. Prior to version 3.31.0, these +** routines would only work if F was the name of the main database file. +** When the F parameter is the name of the rollback journal or WAL file, +** it has access to all the same query parameters as were found on the +** main database file. +** +** See the [URI filename] documentation for additional information. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); + +/* +** CAPI3REF: Translate filenames +** +** These routines are available to [VFS|custom VFS implementations] for +** translating filenames between the main database file, the journal file, +** and the WAL file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, then sqlite3_filename_database(F) +** returns the name of the corresponding database file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, or if F is a database filename +** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F) +** returns the name of the corresponding rollback journal file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** that was passed by the SQLite core into the VFS, or if F is a database +** filename obtained from [sqlite3_db_filename()], then +** sqlite3_filename_wal(F) returns the name of the corresponding +** WAL file. +** +** In all of the above, if F is not the name of a database, journal or WAL +** filename passed into the VFS from the SQLite core and F is not the +** return value from [sqlite3_db_filename()], then the result is +** undefined and is likely a memory access violation. +*/ +SQLITE_API const char *sqlite3_filename_database(const char*); +SQLITE_API const char *sqlite3_filename_journal(const char*); +SQLITE_API const char *sqlite3_filename_wal(const char*); + +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); + +/* +** CAPI3REF: Create and Destroy VFS Filenames +** +** These interfces are provided for use by [VFS shim] implementations and +** are not useful outside of that context. +** +** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of +** database filename D with corresponding journal file J and WAL file W and +** with N URI parameters key/values pairs in the array P. The result from +** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that +** is safe to pass to routines like: +**
    +**
  • [sqlite3_uri_parameter()], +**
  • [sqlite3_uri_boolean()], +**
  • [sqlite3_uri_int64()], +**
  • [sqlite3_uri_key()], +**
  • [sqlite3_filename_database()], +**
  • [sqlite3_filename_journal()], or +**
  • [sqlite3_filename_wal()]. +**
+** If a memory allocation error occurs, sqlite3_create_filename() might +** return a NULL pointer. The memory obtained from sqlite3_create_filename(X) +** must be released by a corresponding call to sqlite3_free_filename(Y). +** +** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array +** of 2*N pointers to strings. Each pair of pointers in this array corresponds +** to a key and value for a query parameter. The P parameter may be a NULL +** pointer if N is zero. None of the 2*N pointers in the P array may be +** NULL pointers and key pointers should not be empty strings. +** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may +** be NULL pointers, though they can be empty strings. +** +** The sqlite3_free_filename(Y) routine releases a memory allocation +** previously obtained from sqlite3_create_filename(). Invoking +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. +** +** If the Y parameter to sqlite3_free_filename(Y) is anything other +** than a NULL pointer or a pointer previously acquired from +** sqlite3_create_filename(), then bad things such as heap +** corruption or segfaults may occur. The value Y should not be +** used again after sqlite3_free_filename(Y) has been called. This means +** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y, +** then the corresponding [sqlite3_module.xClose() method should also be +** invoked prior to calling sqlite3_free_filename(Y). +*/ +SQLITE_API char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +); +SQLITE_API void sqlite3_free_filename(char*); + +/* +** CAPI3REF: Error Codes And Messages +** METHOD: sqlite3 +** +** ^If the most recent sqlite3_* API call associated with +** [database connection] D failed, then the sqlite3_errcode(D) interface +** returns the numeric [result code] or [extended result code] for that +** API call. +** ^The sqlite3_extended_errcode() +** interface is the same except that it always returns the +** [extended result code] even when extended result codes are +** disabled. +** +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces are: +** +**
    +**
  • sqlite3_errcode() +**
  • sqlite3_extended_errcode() +**
  • sqlite3_errmsg() +**
  • sqlite3_errmsg16() +**
+** +** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language +** text that describes the error, as either UTF-8 or UTF-16 respectively. +** ^(Memory to hold the error message string is managed internally. +** The application does not need to worry about freeing the result. +** However, the error string might be overwritten or deallocated by +** subsequent calls to other SQLite interface functions.)^ +** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** +** When the serialized [threading mode] is in use, it might be the +** case that a second error occurs on a separate thread in between +** the time of the first error and the call to these interfaces. +** When that happens, the second error will be reported since these +** interfaces always report the most recent result. To avoid +** this, each thread can obtain exclusive use of the [database connection] D +** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning +** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after +** all calls to the interfaces listed here are completed. +** +** If an interface fails with SQLITE_MISUSE, that means the interface +** was invoked incorrectly by the application. In that case, the +** error code and message may or may not be set. +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db); +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *sqlite3_errmsg(sqlite3*); +SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); + +/* +** CAPI3REF: Prepared Statement Object +** KEYWORDS: {prepared statement} {prepared statements} +** +** An instance of this object represents a single SQL statement that +** has been compiled into binary form and is ready to be evaluated. +** +** Think of each SQL statement as a separate computer program. The +** original SQL text is source code. A prepared statement object +** is the compiled object code. All SQL must be converted into a +** prepared statement before it can be run. +** +** The life-cycle of a prepared statement object usually goes like this: +** +**
    +**
  1. Create the prepared statement object using [sqlite3_prepare_v2()]. +**
  2. Bind values to [parameters] using the sqlite3_bind_*() +** interfaces. +**
  3. Run the SQL by calling [sqlite3_step()] one or more times. +**
  4. Reset the prepared statement using [sqlite3_reset()] then go back +** to step 2. Do this zero or more times. +**
  5. Destroy the object using [sqlite3_finalize()]. +**
+*/ +typedef struct sqlite3_stmt sqlite3_stmt; + +/* +** CAPI3REF: Run-time Limits +** METHOD: sqlite3 +** +** ^(This interface allows the size of various constructs to be limited +** on a connection by connection basis. The first parameter is the +** [database connection] whose limit is to be set or queried. The +** second parameter is one of the [limit categories] that define a +** class of constructs to be size limited. The third parameter is the +** new limit for that construct.)^ +** +** ^If the new limit is a negative number, the limit is unchanged. +** ^(For each limit category SQLITE_LIMIT_NAME there is a +** [limits | hard upper bound] +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. +** (The "_LIMIT_" in the name is changed to "_MAX_".))^ +** ^Attempts to increase a limit above its hard upper bound are +** silently truncated to the hard upper bound. +** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** +** Run-time limits are intended for use in applications that manage +** both their own internal database and also databases that are controlled +** by untrusted external sources. An example application might be a +** web browser that has its own databases for storing history and +** separate databases controlled by JavaScript applications downloaded +** off the Internet. The internal databases can be given the +** large, default limits. Databases managed by external sources can +** be given much smaller limits designed to prevent a denial of service +** attack. Developers might also want to use the [sqlite3_set_authorizer()] +** interface to further control untrusted SQL. The size of the database +** created by an untrusted script can be contained using the +** [max_page_count] [PRAGMA]. +** +** New run-time limit categories may be added in future releases. +*/ +SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); + +/* +** CAPI3REF: Run-Time Limit Categories +** KEYWORDS: {limit category} {*limit categories} +** +** These constants define various performance limits +** that can be lowered at run-time using [sqlite3_limit()]. +** The synopsis of the meanings of the various limits is shown below. +** Additional information is available at [limits | Limits in SQLite]. +** +**
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
+**
The maximum size of any string or BLOB or table row, in bytes.
)^ +** +** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
+**
The maximum length of an SQL statement, in bytes.
)^ +** +** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
+**
The maximum number of columns in a table definition or in the +** result set of a [SELECT] or the maximum number of columns in an index +** or in an ORDER BY or GROUP BY clause.
)^ +** +** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
+**
The maximum depth of the parse tree on any expression.
)^ +** +** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
+**
The maximum number of terms in a compound SELECT statement.
)^ +** +** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
+**
The maximum number of instructions in a virtual machine program +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned.
)^ +** +** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
+**
The maximum number of arguments on a function.
)^ +** +** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
+**
The maximum number of [ATTACH | attached databases].)^
+** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] +** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
+**
The maximum length of the pattern argument to the [LIKE] or +** [GLOB] operators.
)^ +** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] +** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
+**
The maximum index number of any [parameter] in an SQL statement.)^ +** +** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+**
The maximum depth of recursion for triggers.
)^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
SQLITE_LIMIT_WORKER_THREADS
+**
The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
)^ +**
+*/ +#define SQLITE_LIMIT_LENGTH 0 +#define SQLITE_LIMIT_SQL_LENGTH 1 +#define SQLITE_LIMIT_COLUMN 2 +#define SQLITE_LIMIT_EXPR_DEPTH 3 +#define SQLITE_LIMIT_COMPOUND_SELECT 4 +#define SQLITE_LIMIT_VDBE_OP 5 +#define SQLITE_LIMIT_FUNCTION_ARG 6 +#define SQLITE_LIMIT_ATTACHED 7 +#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 +#define SQLITE_LIMIT_VARIABLE_NUMBER 9 +#define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 + +/* +** CAPI3REF: Prepare Flags +** +** These constants define various flags that can be passed into +** "prepFlags" parameter of the [sqlite3_prepare_v3()] and +** [sqlite3_prepare16_v3()] interfaces. +** +** New flags may be added in future releases of SQLite. +** +**
+** [[SQLITE_PREPARE_PERSISTENT]] ^(
SQLITE_PREPARE_PERSISTENT
+**
The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner +** that the prepared statement will be retained for a long time and +** probably reused many times.)^ ^Without this flag, [sqlite3_prepare_v3()] +** and [sqlite3_prepare16_v3()] assume that the prepared statement will +** be used just once or at most a few times and then destroyed using +** [sqlite3_finalize()] relatively soon. The current implementation acts +** on this hint by avoiding the use of [lookaside memory] so as not to +** deplete the limited store of lookaside memory. Future versions of +** SQLite may act on this hint differently. +** +** [[SQLITE_PREPARE_NORMALIZE]]
SQLITE_PREPARE_NORMALIZE
+**
The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. +** +** [[SQLITE_PREPARE_NO_VTAB]]
SQLITE_PREPARE_NO_VTAB
+**
The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler +** to return an error (error code SQLITE_ERROR) if the statement uses +** any virtual tables. +**
+*/ +#define SQLITE_PREPARE_PERSISTENT 0x01 +#define SQLITE_PREPARE_NORMALIZE 0x02 +#define SQLITE_PREPARE_NO_VTAB 0x04 + +/* +** CAPI3REF: Compiling An SQL Statement +** KEYWORDS: {SQL statement compiler} +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_stmt +** +** To execute an SQL statement, it must first be compiled into a byte-code +** program using one of these routines. Or, in other words, these routines +** are constructors for the [prepared statement] object. +** +** The preferred routine to use is [sqlite3_prepare_v2()]. The +** [sqlite3_prepare()] interface is legacy and should be avoided. +** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used +** for special purposes. +** +** The use of the UTF-8 interfaces is preferred, as SQLite currently +** does all parsing using UTF-8. The UTF-16 interfaces are provided +** as a convenience. The UTF-16 interfaces work by converting the +** input text into UTF-8, then invoking the corresponding UTF-8 interface. +** +** The first argument, "db", is a [database connection] obtained from a +** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or +** [sqlite3_open16()]. The database connection must not have been closed. +** +** The second argument, "zSql", is the statement to be compiled, encoded +** as either UTF-8 or UTF-16. The sqlite3_prepare(), sqlite3_prepare_v2(), +** and sqlite3_prepare_v3() +** interfaces use UTF-8, and sqlite3_prepare16(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() use UTF-16. +** +** ^If the nByte argument is negative, then zSql is read up to the +** first zero terminator. ^If nByte is positive, then it is the +** number of bytes read from zSql. ^If nByte is zero, then no prepared +** statement is generated. +** If the caller knows that the supplied string is nul-terminated, then +** there is a small performance advantage to passing an nByte parameter that +** is the number of bytes in the input string including +** the nul-terminator. +** +** ^If pzTail is not NULL then *pzTail is made to point to the first byte +** past the end of the first SQL statement in zSql. These routines only +** compile the first statement in zSql, so *pzTail is left pointing to +** what remains uncompiled. +** +** ^*ppStmt is left pointing to a compiled [prepared statement] that can be +** executed using [sqlite3_step()]. ^If there is an error, *ppStmt is set +** to NULL. ^If the input text contains no SQL (if the input is an empty +** string or a comment) then *ppStmt is set to NULL. +** The calling procedure is responsible for deleting the compiled +** SQL statement using [sqlite3_finalize()] after it has finished with it. +** ppStmt may not be NULL. +** +** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK]; +** otherwise an [error code] is returned. +** +** The sqlite3_prepare_v2(), sqlite3_prepare_v3(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() interfaces are recommended for all new programs. +** The older interfaces (sqlite3_prepare() and sqlite3_prepare16()) +** are retained for backwards compatibility, but their use is discouraged. +** ^In the "vX" interfaces, the prepared statement +** that is returned (the [sqlite3_stmt] object) contains a copy of the +** original SQL text. This causes the [sqlite3_step()] interface to +** behave differently in three ways: +** +**
    +**
  1. +** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it +** always used to do, [sqlite3_step()] will automatically recompile the SQL +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. +**
    2. +** +**
  2. +** ^When an error occurs, [sqlite3_step()] will return one of the detailed +** [error codes] or [extended error codes]. ^The legacy behavior was that +** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code +** and the application would have to make a second call to [sqlite3_reset()] +** in order to find the underlying cause of the problem. With the "v2" prepare +** interfaces, the underlying reason for the error is returned immediately. +**
    3. +** +**
  3. +** ^If the specific value bound to a [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of a WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. +**
    4. +**
+** +**

^sqlite3_prepare_v3() differs from sqlite3_prepare_v2() only in having +** the extra prepFlags parameter, which is a bit array consisting of zero or +** more of the [SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_*] flags. ^The +** sqlite3_prepare_v2() interface works exactly the same as +** sqlite3_prepare_v3() with a zero prepFlags parameter. +*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); + +/* +** CAPI3REF: Retrieving Statement SQL +** METHOD: sqlite3_stmt +** +** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 +** SQL text used to create [prepared statement] P if P was +** created by [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 +** string containing the SQL text of prepared statement P with +** [bound parameters] expanded. +** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8 +** string containing the normalized SQL text of prepared statement P. The +** semantics used to normalize a SQL statement are unspecified and subject +** to change. At a minimum, literal values will be replaced with suitable +** placeholders. +** +** ^(For example, if a prepared statement is created using the SQL +** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 +** and parameter :xyz is unbound, then sqlite3_sql() will return +** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() +** will return "SELECT 2345,NULL".)^ +** +** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory +** is available to hold the result, or if the result would exceed the +** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** +** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of +** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time +** option causes sqlite3_expanded_sql() to always return NULL. +** +** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P) +** are managed by SQLite and are automatically freed when the prepared +** statement is finalized. +** ^The string returned by sqlite3_expanded_sql(P), on the other hand, +** is obtained from [sqlite3_malloc()] and must be free by the application +** by passing it to [sqlite3_free()]. +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 

+**    SELECT eval('DELETE FROM t1') FROM t2;
+**
+** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. +** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since +** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and +** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so +** sqlite3_stmt_readonly() returns false for those commands. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + +/* +** CAPI3REF: Dynamically Typed Value Object +** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} +** +** SQLite uses the sqlite3_value object to represent all values +** that can be stored in a database table. SQLite uses dynamic typing +** for the values it stores. ^Values stored in sqlite3_value objects +** can be integers, floating point values, strings, BLOBs, or NULL. +** +** An sqlite3_value object may be either "protected" or "unprotected". +** Some interfaces require a protected sqlite3_value. Other interfaces +** will accept either a protected or an unprotected sqlite3_value. +** Every interface that accepts sqlite3_value arguments specifies +** whether or not it requires a protected sqlite3_value. The +** [sqlite3_value_dup()] interface can be used to construct a new +** protected sqlite3_value from an unprotected sqlite3_value. +** +** The terms "protected" and "unprotected" refer to whether or not +** a mutex is held. An internal mutex is held for a protected +** sqlite3_value object but no mutex is held for an unprotected +** sqlite3_value object. If SQLite is compiled to be single-threaded +** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +** or if SQLite is run in one of reduced mutex modes +** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +** then there is no distinction between protected and unprotected +** sqlite3_value objects and they can be used interchangeably. However, +** for maximum code portability it is recommended that applications +** still make the distinction between protected and unprotected +** sqlite3_value objects even when not strictly required. +** +** ^The sqlite3_value objects that are passed as parameters into the +** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value object returned by +** [sqlite3_column_value()] is unprotected. +** Unprotected sqlite3_value objects may only be used as arguments +** to [sqlite3_result_value()], [sqlite3_bind_value()], and +** [sqlite3_value_dup()]. +** The [sqlite3_value_blob | sqlite3_value_type()] family of +** interfaces require protected sqlite3_value objects. +*/ +typedef struct sqlite3_value sqlite3_value; + +/* +** CAPI3REF: SQL Function Context Object +** +** The context in which an SQL function executes is stored in an +** sqlite3_context object. ^A pointer to an sqlite3_context object +** is always first parameter to [application-defined SQL functions]. +** The application-defined SQL function implementation will pass this +** pointer through into calls to [sqlite3_result_int | sqlite3_result()], +** [sqlite3_aggregate_context()], [sqlite3_user_data()], +** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()], +** and/or [sqlite3_set_auxdata()]. +*/ +typedef struct sqlite3_context sqlite3_context; + +/* +** CAPI3REF: Binding Values To Prepared Statements +** KEYWORDS: {host parameter} {host parameters} {host parameter name} +** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** METHOD: sqlite3_stmt +** +** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, +** literals may be replaced by a [parameter] that matches one of following +** templates: +** +**
    +**
  • ? +**
  • ?NNN +**
  • :VVV +**
  • @VVV +**
  • $VVV +**
+** +** In the templates above, NNN represents an integer literal, +** and VVV represents an alphanumeric identifier.)^ ^The values of these +** parameters (also called "host parameter names" or "SQL parameters") +** can be set using the sqlite3_bind_*() routines defined here. +** +** ^The first argument to the sqlite3_bind_*() routines is always +** a pointer to the [sqlite3_stmt] object returned from +** [sqlite3_prepare_v2()] or its variants. +** +** ^The second argument is the index of the SQL parameter to be set. +** ^The leftmost SQL parameter has an index of 1. ^When the same named +** SQL parameter is used more than once, second and subsequent +** occurrences have the same index as the first occurrence. +** ^The index for named parameters can be looked up using the +** [sqlite3_bind_parameter_index()] API if desired. ^The index +** for "?NNN" parameters is the value of NNN. +** ^The NNN value must be between 1 and the [sqlite3_limit()] +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766). +** +** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. +** +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. +** +** ^(In those routines that have a fourth argument, its value is the +** number of bytes in the parameter. To be clear: the value is the +** number of bytes in the value, not the number of characters.)^ +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is +** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occurs at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. +** +** ^The fifth argument to the BLOB and string binding interfaces +** is a destructor used to dispose of the BLOB or +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to the bind API fails, +** except the destructor is not called if the third parameter is a NULL +** pointer or the fourth parameter is negative. +** ^If the fifth argument is +** the special value [SQLITE_STATIC], then SQLite assumes that the +** information is in static, unmanaged space and does not need to be freed. +** ^If the fifth argument has the value [SQLITE_TRANSIENT], then +** SQLite makes its own private copy of the data immediately, before +** the sqlite3_bind_*() routine returns. +** +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** +** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that +** is filled with zeroes. ^A zeroblob uses a fixed amount of memory +** (just an integer to hold its size) while it is being processed. +** Zeroblobs are intended to serve as placeholders for BLOBs whose +** content is later written using +** [sqlite3_blob_open | incremental BLOB I/O] routines. +** ^A negative value for the zeroblob results in a zero-length BLOB. +** +** ^The sqlite3_bind_pointer(S,I,P,T,D) routine causes the I-th parameter in +** [prepared statement] S to have an SQL value of NULL, but to also be +** associated with the pointer P of type T. ^D is either a NULL pointer or +** a pointer to a destructor function for P. ^SQLite will invoke the +** destructor D with a single argument of P when it is finished using +** P. The T parameter should be a static string, preferably a string +** literal. The sqlite3_bind_pointer() routine is part of the +** [pointer passing interface] added for SQLite 3.20.0. +** +** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer +** for the [prepared statement] or with a prepared statement for which +** [sqlite3_step()] has been called more recently than [sqlite3_reset()], +** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() +** routine is passed a [prepared statement] that has been finalized, the +** result is undefined and probably harmful. +** +** ^Bindings are not cleared by the [sqlite3_reset()] routine. +** ^Unbound parameters are interpreted as NULL. +** +** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an +** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. +** ^[SQLITE_RANGE] is returned if the parameter +** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. +** +** See also: [sqlite3_bind_parameter_count()], +** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); +SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*,void(*)(void*)); +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); + +/* +** CAPI3REF: Number Of SQL Parameters +** METHOD: sqlite3_stmt +** +** ^This routine can be used to find the number of [SQL parameters] +** in a [prepared statement]. SQL parameters are tokens of the +** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as +** placeholders for values that are [sqlite3_bind_blob | bound] +** to the parameters at a later time. +** +** ^(This routine actually returns the index of the largest (rightmost) +** parameter. For all forms except ?NNN, this will correspond to the +** number of unique parameters. If parameters of the ?NNN form are used, +** there may be gaps in the list.)^ +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_name()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); + +/* +** CAPI3REF: Name Of A Host Parameter +** METHOD: sqlite3_stmt +** +** ^The sqlite3_bind_parameter_name(P,N) interface returns +** the name of the N-th [SQL parameter] in the [prepared statement] P. +** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA" +** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA" +** respectively. +** In other words, the initial ":" or "$" or "@" or "?" +** is included as part of the name.)^ +** ^Parameters of the form "?" without a following integer have no name +** and are referred to as "nameless" or "anonymous parameters". +** +** ^The first host parameter has an index of 1, not 0. +** +** ^If the value N is out of range or if the N-th parameter is +** nameless, then NULL is returned. ^The returned string is +** always in UTF-8 encoding even if the named parameter was +** originally specified as UTF-16 in [sqlite3_prepare16()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); + +/* +** CAPI3REF: Index Of A Parameter With A Given Name +** METHOD: sqlite3_stmt +** +** ^Return the index of an SQL parameter given its name. ^The +** index value returned is suitable for use as the second +** parameter to [sqlite3_bind_blob|sqlite3_bind()]. ^A zero +** is returned if no matching parameter is found. ^The parameter +** name must be given in UTF-8 even if the original statement +** was prepared from UTF-16 text using [sqlite3_prepare16_v2()] or +** [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_name()]. +*/ +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); + +/* +** CAPI3REF: Reset All Bindings On A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset +** the [sqlite3_bind_blob | bindings] on a [prepared statement]. +** ^Use this routine to reset all host parameters to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); + +/* +** CAPI3REF: Number Of Columns In A Result Set +** METHOD: sqlite3_stmt +** +** ^Return the number of columns in the result set returned by the +** [prepared statement]. ^If this routine returns 0, that means the +** [prepared statement] returns no data (for example an [UPDATE]). +** ^However, just because this routine returns a positive number does not +** mean that one or more rows of data will be returned. ^A SELECT statement +** will always have a positive sqlite3_column_count() but depending on the +** WHERE clause constraints and the table content, it might return no rows. +** +** See also: [sqlite3_data_count()] +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Column Names In A Result Set +** METHOD: sqlite3_stmt +** +** ^These routines return the name assigned to a particular column +** in the result set of a [SELECT] statement. ^The sqlite3_column_name() +** interface returns a pointer to a zero-terminated UTF-8 string +** and sqlite3_column_name16() returns a pointer to a zero-terminated +** UTF-16 string. ^The first parameter is the [prepared statement] +** that implements the [SELECT] statement. ^The second parameter is the +** column number. ^The leftmost column is number 0. +** +** ^The returned string pointer is valid until either the [prepared statement] +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to +** sqlite3_column_name() or sqlite3_column_name16() on the same column. +** +** ^If sqlite3_malloc() fails during the processing of either routine +** (for example during a conversion from UTF-8 to UTF-16) then a +** NULL pointer is returned. +** +** ^The name of a result column is the value of the "AS" clause for +** that column, if there is an AS clause. If there is no AS clause +** then the name of the column is unspecified and may change from +** one release of SQLite to the next. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); + +/* +** CAPI3REF: Source Of Data In A Query Result +** METHOD: sqlite3_stmt +** +** ^These routines provide a means to determine the database, table, and +** table column that is the origin of a particular result column in +** [SELECT] statement. +** ^The name of the database or table or column can be returned as +** either a UTF-8 or UTF-16 string. ^The _database_ routines return +** the database name, the _table_ routines return the table name, and +** the origin_ routines return the column name. +** ^The returned string is valid until the [prepared statement] is destroyed +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested +** again in a different encoding. +** +** ^The names returned are the original un-aliased names of the +** database, table, and column. +** +** ^The first argument to these interfaces is a [prepared statement]. +** ^These functions return information about the Nth result column returned by +** the statement, where N is the second function argument. +** ^The left-most column is column 0 for these routines. +** +** ^If the Nth column returned by the statement is an expression or +** subquery and is not a column value, then all of these functions return +** NULL. ^These routines might also return NULL if a memory allocation error +** occurs. ^Otherwise, they return the name of the attached database, table, +** or column that query result column was extracted from. +** +** ^As with all other SQLite APIs, those whose names end with "16" return +** UTF-16 encoded strings and the other functions return UTF-8. +** +** ^These APIs are only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. +** +** If two or more threads call one or more +** [sqlite3_column_database_name | column metadata interfaces] +** for the same [prepared statement] and result column +** at the same time then the results are undefined. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Declared Datatype Of A Query Result +** METHOD: sqlite3_stmt +** +** ^(The first parameter is a [prepared statement]. +** If this statement is a [SELECT] statement and the Nth column of the +** returned result set of that [SELECT] is a table column (not an +** expression or subquery) then the declared type of the table +** column is returned.)^ ^If the Nth column of the result set is an +** expression or subquery, then a NULL pointer is returned. +** ^The returned string is always UTF-8 encoded. +** +** ^(For example, given the database schema: +** +** CREATE TABLE t1(c1 VARIANT); +** +** and the following statement to be compiled: +** +** SELECT c1 + 1, c1 FROM t1; +** +** this routine would return the string "VARIANT" for the second result +** column (i==1), and a NULL pointer for the first result column (i==0).)^ +** +** ^SQLite uses dynamic run-time typing. ^So just because a column +** is declared to contain a particular type does not mean that the +** data stored in that column is of the declared type. SQLite is +** strongly typed, but the typing is dynamic not static. ^Type +** is associated with individual values, not with the containers +** used to hold those values. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Evaluate An SQL Statement +** METHOD: sqlite3_stmt +** +** After a [prepared statement] has been prepared using any of +** [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], [sqlite3_prepare16_v2()], +** or [sqlite3_prepare16_v3()] or one of the legacy +** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function +** must be called one or more times to evaluate the statement. +** +** The details of the behavior of the sqlite3_step() interface depend +** on whether the statement was prepared using the newer "vX" interfaces +** [sqlite3_prepare_v3()], [sqlite3_prepare_v2()], [sqlite3_prepare16_v3()], +** [sqlite3_prepare16_v2()] or the older legacy +** interfaces [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the +** new "vX" interface is recommended for new applications but the legacy +** interface will continue to be supported. +** +** ^In the legacy interface, the return value will be either [SQLITE_BUSY], +** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. +** ^With the "v2" interface, any of the other [result codes] or +** [extended result codes] might be returned as well. +** +** ^[SQLITE_BUSY] means that the database engine was unable to acquire the +** database locks it needs to do its job. ^If the statement is a [COMMIT] +** or occurs outside of an explicit transaction, then you can retry the +** statement. If the statement is not a [COMMIT] and occurs within an +** explicit transaction then you should rollback the transaction before +** continuing. +** +** ^[SQLITE_DONE] means that the statement has finished executing +** successfully. sqlite3_step() should not be called again on this virtual +** machine without first calling [sqlite3_reset()] to reset the virtual +** machine back to its initial state. +** +** ^If the SQL statement being executed returns any data, then [SQLITE_ROW] +** is returned each time a new row of data is ready for processing by the +** caller. The values may be accessed using the [column access functions]. +** sqlite3_step() is called again to retrieve the next row of data. +** +** ^[SQLITE_ERROR] means that a run-time error (such as a constraint +** violation) has occurred. sqlite3_step() should not be called again on +** the VM. More information may be found by calling [sqlite3_errmsg()]. +** ^With the legacy interface, a more specific error code (for example, +** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) +** can be obtained by calling [sqlite3_reset()] on the +** [prepared statement]. ^In the "v2" interface, +** the more specific error code is returned directly by sqlite3_step(). +** +** [SQLITE_MISUSE] means that the this routine was called inappropriately. +** Perhaps it was called on a [prepared statement] that has +** already been [sqlite3_finalize | finalized] or on one that had +** previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could +** be the case that the same database connection is being used by two or +** more threads at the same moment in time. +** +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. +** +** Goofy Interface Alert: In the legacy interface, the sqlite3_step() +** API always returns a generic error code, [SQLITE_ERROR], following any +** error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call +** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the +** specific [error codes] that better describes the error. +** We admit that this is a goofy design. The problem has been fixed +** with the "v2" interface. If you prepare all of your SQL statements +** using [sqlite3_prepare_v3()] or [sqlite3_prepare_v2()] +** or [sqlite3_prepare16_v2()] or [sqlite3_prepare16_v3()] instead +** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, +** then the more specific [error codes] are returned directly +** by sqlite3_step(). The use of the "vX" interfaces is recommended. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt*); + +/* +** CAPI3REF: Number of columns in a result set +** METHOD: sqlite3_stmt +** +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column()] family of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. +** +** See also: [sqlite3_column_count()] +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Fundamental Datatypes +** KEYWORDS: SQLITE_TEXT +** +** ^(Every value in SQLite has one of five fundamental datatypes: +** +**
    +**
  • 64-bit signed integer +**
  • 64-bit IEEE floating point number +**
  • string +**
  • BLOB +**
  • NULL +**
)^ +** +** These constants are codes for each of those types. +** +** Note that the SQLITE_TEXT constant was also used in SQLite version 2 +** for a completely different meaning. Software that links against both +** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not +** SQLITE_TEXT. +*/ +#define SQLITE_INTEGER 1 +#define SQLITE_FLOAT 2 +#define SQLITE_BLOB 4 +#define SQLITE_NULL 5 +#ifdef SQLITE_TEXT +# undef SQLITE_TEXT +#else +# define SQLITE_TEXT 3 +#endif +#define SQLITE3_TEXT 3 + +/* +** CAPI3REF: Result Values From A Query +** KEYWORDS: {column access functions} +** METHOD: sqlite3_stmt +** +** Summary: +**
+**
sqlite3_column_blob→BLOB result +**
sqlite3_column_double→REAL result +**
sqlite3_column_int→32-bit INTEGER result +**
sqlite3_column_int64→64-bit INTEGER result +**
sqlite3_column_text→UTF-8 TEXT result +**
sqlite3_column_text16→UTF-16 TEXT result +**
sqlite3_column_value→The result as an +** [sqlite3_value|unprotected sqlite3_value] object. +**
    +**
sqlite3_column_bytes→Size of a BLOB +** or a UTF-8 TEXT result in bytes +**
sqlite3_column_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
sqlite3_column_type→Default +** datatype of the result +**
+** +** Details: +** +** ^These routines return information about a single column of the current +** result row of a query. ^In every case the first argument is a pointer +** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] +** that was returned from [sqlite3_prepare_v2()] or one of its variants) +** and the second argument is the index of the column for which information +** should be returned. ^The leftmost column of the result set has the index 0. +** ^The number of columns in the result can be determined using +** [sqlite3_column_count()]. +** +** If the SQL statement does not currently point to a valid row, or if the +** column index is out of range, the result is undefined. +** These routines may only be called when the most recent call to +** [sqlite3_step()] has returned [SQLITE_ROW] and neither +** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently. +** If any of these routines are called after [sqlite3_reset()] or +** [sqlite3_finalize()] or after [sqlite3_step()] has returned +** something other than [SQLITE_ROW], the results are undefined. +** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()] +** are called from a different thread while any of these routines +** are pending, then the results are undefined. +** +** The first six interfaces (_blob, _double, _int, _int64, _text, and _text16) +** each return the value of a result column in a specific data format. If +** the result column is not initially in the requested format (for example, +** if the query returns an integer but the sqlite3_column_text() interface +** is used to extract the value) then an automatic type conversion is performed. +** +** ^The sqlite3_column_type() routine returns the +** [SQLITE_INTEGER | datatype code] for the initial data type +** of the result column. ^The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. +** The return value of sqlite3_column_type() can be used to decide which +** of the first six interface should be used to extract the column value. +** The value returned by sqlite3_column_type() is only meaningful if no +** automatic type conversions have occurred for the value in question. +** After a type conversion, the result of calling sqlite3_column_type() +** is undefined, though harmless. Future +** versions of SQLite may change the behavior of sqlite3_column_type() +** following a type conversion. +** +** If the result is a BLOB or a TEXT string, then the sqlite3_column_bytes() +** or sqlite3_column_bytes16() interfaces can be used to determine the size +** of that BLOB or string. +** +** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts +** the string to UTF-8 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes() uses +** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of +** bytes in the string, not the number of characters. +** +** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), +** even empty strings, are always zero-terminated. ^The return +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. +** +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. +** If the [unprotected sqlite3_value] object returned by +** [sqlite3_column_value()] is used in any other way, including calls +** to routines like [sqlite3_value_int()], [sqlite3_value_text()], +** or [sqlite3_value_bytes()], the behavior is not threadsafe. +** Hence, the sqlite3_column_value() interface +** is normally only useful within the implementation of +** [application-defined SQL functions] or [virtual tables], not within +** top-level application code. +** +** The these routines may attempt to convert the datatype of the result. +** ^For example, if the internal representation is FLOAT and a text result +** is requested, [sqlite3_snprintf()] is used internally to perform the +** conversion automatically. ^(The following table details the conversions +** that are applied: +** +**
+** +**
Internal
Type 		Requested
Type 		Conversion +** +**
NULL INTEGER Result is 0 +**
NULL FLOAT Result is 0.0 +**
NULL TEXT Result is a NULL pointer +**
NULL BLOB Result is a NULL pointer +**
INTEGER FLOAT Convert from integer to float +**
INTEGER TEXT ASCII rendering of the integer +**
INTEGER BLOB Same as INTEGER->TEXT +**
FLOAT INTEGER [CAST] to INTEGER +**
FLOAT TEXT ASCII rendering of the float +**
FLOAT BLOB [CAST] to BLOB +**
TEXT INTEGER [CAST] to INTEGER +**
TEXT FLOAT [CAST] to REAL +**
TEXT BLOB No change +**
BLOB INTEGER [CAST] to INTEGER +**
BLOB FLOAT [CAST] to REAL +**
BLOB TEXT Add a zero terminator if needed +**
+**
)^ +** +** Note that when type conversions occur, pointers returned by prior +** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or +** sqlite3_column_text16() may be invalidated. +** Type conversions and pointer invalidations might occur +** in the following cases: +** +**
    +**
  • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
    • +**
  • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
    • +**
  • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
    • +**
+** +** ^Conversions between UTF-16be and UTF-16le are always done in place and do +** not invalidate a prior pointer, though of course the content of the buffer +** that the prior pointer references will have been modified. Other kinds +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. +** +** The safest policy is to invoke these routines +** in one of the following ways: +** +**
    +**
  • sqlite3_column_text() followed by sqlite3_column_bytes()
    • +**
  • sqlite3_column_blob() followed by sqlite3_column_bytes()
    • +**
  • sqlite3_column_text16() followed by sqlite3_column_bytes16()
    • +**
+** +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** ^The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. ^The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
    +**
  • sqlite3_column_blob() +**
  • sqlite3_column_text() +**
  • sqlite3_column_text16() +**
  • sqlite3_column_bytes() +**
  • sqlite3_column_bytes16() +**
+** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); + +/* +** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt +** +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the most recent evaluation of the statement encountered no errors +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. +** +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt +** +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. +** ^Any SQL statement variables that had values bound to them using +** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. +** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Create Or Redefine SQL Functions +** KEYWORDS: {function creation routines} +** METHOD: sqlite3 +** +** ^These functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. +** +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. +** +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. +** +** ^The third parameter (nArg) +** is the number of arguments that the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or +** aggregate may take any number of arguments between 0 and the limit +** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third +** parameter is less than -1 or greater than 127 then the behavior is +** undefined. +** +** ^The fourth parameter, eTextRep, specifies what +** [SQLITE_UTF8 | text encoding] this SQL function prefers for +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. +** ^When multiple implementations of the same function are available, SQLite +** will pick the one that involves the least amount of data conversion. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. +** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, +** index expressions, or the WHERE clause of partial indexes. +** +** +** For best security, the [SQLITE_DIRECTONLY] flag is recommended for +** all application-defined SQL functions that do not need to be +** used inside of triggers, view, CHECK constraints, or other elements of +** the database schema. This flags is especially recommended for SQL +** functions that have side effects or reveal internal application state. +** Without this flag, an attacker might be able to modify the schema of +** a database file to include invocations of the function with parameters +** chosen by the attacker, which the application will then execute when +** the database file is opened and read. +** +** +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ +** +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are +** pointers to C-language functions that implement the SQL function or +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers must be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL pointers for all three function +** callbacks. +** +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). +** +** ^It is permitted to register multiple implementations of the same +** functions with the same name but with either differing numbers of +** arguments or differing preferred text encodings. ^SQLite will use +** the implementation that most closely matches the way in which the +** SQL function is used. ^A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. ^A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** ^A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** ^Built-in functions may be overloaded by new application-defined functions. +** +** ^An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); + +/* +** CAPI3REF: Text Encodings +** +** These constant define integer codes that represent the various +** text encodings supported by SQLite. +*/ +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ +#define SQLITE_UTF16 4 /* Use native byte order */ +#define SQLITE_ANY 5 /* Deprecated */ +#define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ + +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +** +**
+** [[SQLITE_DETERMINISTIC]]
SQLITE_DETERMINISTIC
+** The SQLITE_DETERMINISTIC flag means that the new function always gives +** the same output when the input parameters are the same. +** The [abs|abs() function] is deterministic, for example, but +** [randomblob|randomblob()] is not. Functions must +** be deterministic in order to be used in certain contexts such as +** with the WHERE clause of [partial indexes] or in [generated columns]. +** SQLite might also optimize deterministic functions by factoring them +** out of inner loops. +**
+** +** [[SQLITE_DIRECTONLY]]
SQLITE_DIRECTONLY
+** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], or [generated columns]. +** The SQLITE_DIRECTONLY flags is a security feature which is recommended +** for all [application-defined SQL functions], and especially for functions +** that have side-effects or that could potentially leak sensitive +** information. +**
+** +** [[SQLITE_INNOCUOUS]]
SQLITE_INNOCUOUS
+** The SQLITE_INNOCUOUS flag means that the function is unlikely +** to cause problems even if misused. An innocuous function should have +** no side effects and should not depend on any values other than its +** input parameters. The [abs|abs() function] is an example of an +** innocuous function. +** The [load_extension() SQL function] is not innocuous because of its +** side effects. +**

SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not +** exactly the same. The [random|random() function] is an example of a +** function that is innocuous but not deterministic. +**

Some heightened security settings +** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) +** disable the use of SQL functions inside views and triggers and in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], and [generated columns] unless +** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions +** are innocuous. Developers are advised to avoid using the +** SQLITE_INNOCUOUS flag for application-defined functions unless the +** function has been carefully audited and found to be free of potentially +** security-adverse side-effects and information-leaks. +**

+** +** [[SQLITE_SUBTYPE]]
SQLITE_SUBTYPE
+** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** Specifying this flag makes no difference for scalar or aggregate user +** functions. However, if it is not specified for a user-defined window +** function, then any sub-types belonging to arguments passed to the window +** function may be discarded before the window function is called (i.e. +** sqlite3_value_subtype() will always return 0). +**
+**
+*/ +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 +#define SQLITE_INNOCUOUS 0x000200000 + +/* +** CAPI3REF: Deprecated Functions +** DEPRECATED +** +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); +#endif + +/* +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value +** +** Summary: +**
+**
sqlite3_value_blob→BLOB value +**
sqlite3_value_double→REAL value +**
sqlite3_value_int→32-bit INTEGER value +**
sqlite3_value_int64→64-bit INTEGER value +**
sqlite3_value_pointer→Pointer value +**
sqlite3_value_text→UTF-8 TEXT value +**
sqlite3_value_text16→UTF-16 TEXT value in +** the native byteorder +**
sqlite3_value_text16be→UTF-16be TEXT value +**
sqlite3_value_text16le→UTF-16le TEXT value +**
    +**
sqlite3_value_bytes→Size of a BLOB +** or a UTF-8 TEXT in bytes +**
sqlite3_value_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
sqlite3_value_type→Default +** datatype of the value +**
sqlite3_value_numeric_type   +** →  Best numeric datatype of the value +**
sqlite3_value_nochange   +** →  True if the column is unchanged in an UPDATE +** against a virtual table. +**
sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] +**
+** +** Details: +** +** These routines extract type, size, and content information from +** [protected sqlite3_value] objects. Protected sqlite3_value objects +** are used to pass parameter information into the functions that +** implement [application-defined SQL functions] and [virtual tables]. +** +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** is not threadsafe. +** +** ^These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The +** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces +** extract UTF-16 strings as big-endian and little-endian respectively. +** +** ^If [sqlite3_value] object V was initialized +** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)] +** and if X and Y are strings that compare equal according to strcmp(X,Y), +** then sqlite3_value_pointer(V,Y) will return the pointer P. ^Otherwise, +** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** ^(The sqlite3_value_type(V) interface returns the +** [SQLITE_INTEGER | datatype code] for the initial datatype of the +** [sqlite3_value] object V. The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].)^ +** Other interfaces might change the datatype for an sqlite3_value object. +** For example, if the datatype is initially SQLITE_INTEGER and +** sqlite3_value_text(V) is called to extract a text value for that +** integer, then subsequent calls to sqlite3_value_type(V) might return +** SQLITE_TEXT. Whether or not a persistent internal datatype conversion +** occurs is undefined and may change from one release of SQLite to the next. +** +** ^(The sqlite3_value_numeric_type() interface attempts to apply +** numeric affinity to the value. This means that an attempt is +** made to convert the value to an integer or floating point. If +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ +** +** ^Within the [xUpdate] method of a [virtual table], the +** sqlite3_value_nochange(X) interface returns true if and only if +** the column corresponding to X is unchanged by the UPDATE operation +** that the xUpdate method call was invoked to implement and if +** and the prior [xColumn] method call that was invoked to extracted +** the value for that column returned without setting a result (probably +** because it queried [sqlite3_vtab_nochange()] and found that the column +** was unchanging). ^Within an [xUpdate] method, any value for which +** sqlite3_value_nochange(X) is true will in all other respects appear +** to be a NULL value. If sqlite3_value_nochange(X) is invoked anywhere other +** than within an [xUpdate] method call for an UPDATE statement, then +** the return value is arbitrary and meaningless. +** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** or an expression, then sqlite3_value_frombind(X) returns zero. +** +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or +** [sqlite3_value_text16()] can be invalidated by a subsequent call to +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
    +**
  • sqlite3_value_blob() +**
  • sqlite3_value_text() +**
  • sqlite3_value_text16() +**
  • sqlite3_value_text16le() +**
  • sqlite3_value_text16be() +**
  • sqlite3_value_bytes() +**
  • sqlite3_value_bytes16() +**
+** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API void *sqlite3_value_pointer(sqlite3_value*, const char*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +*/ +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void sqlite3_value_free(sqlite3_value*); + +/* +** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context +** +** Implementations of aggregate SQL functions use this +** routine to allocate memory for storing their state. +** +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite allocates +** N bytes of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ +** +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. +** +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in any subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the xStep or xFinal callback routine that implements the aggregate +** function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); + +/* +** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context +** +** ^The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** This routine must be called from the same thread in which +** the application-defined function is running. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); + +/* +** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context +** +** These functions may be used by (non-aggregate) SQL functions to +** associate metadata with argument values. If the same value is passed to +** multiple invocations of the same SQL function during query execution, under +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. +** +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument +** value to the application-defined function. ^N is zero for the left-most +** function argument. ^If there is no metadata +** associated with the function argument, the sqlite3_get_auxdata(C,N) interface +** returns a NULL pointer. +** +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
    +**
  • ^(when the corresponding function parameter changes)^, or +**
  • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
  • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
  • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^
+** +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. +** +** ^(In practice, metadata is preserved between function calls for +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ +** +** The value of the N parameter to these interfaces should be non-negative. +** Future enhancements may make use of negative N values to define new +** kinds of function caching behavior. +** +** These routines must be called from the same thread in which +** the SQL function is running. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); + + +/* +** CAPI3REF: Constants Defining Special Destructor Behavior +** +** These are special values for the destructor that is passed in as the +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor +** argument is SQLITE_STATIC, it means that the content pointer is constant +** and will never change. It does not need to be destroyed. ^The +** SQLITE_TRANSIENT value means that the content will likely change in +** the near future and that SQLite should make its own private copy of +** the content before returning. +** +** The typedef is necessary to work around problems in certain +** C++ compilers. +*/ +typedef void (*sqlite3_destructor_type)(void*); +#define SQLITE_STATIC ((sqlite3_destructor_type)0) +#define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) + +/* +** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context +** +** These routines are used by the xFunc or xFinal callbacks that +** implement SQL functions and aggregates. See +** [sqlite3_create_function()] and [sqlite3_create_function16()] +** for additional information. +** +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. +** +** ^The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. +** +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. +** +** ^The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. +** +** ^The sqlite3_result_error() and sqlite3_result_error16() functions +** cause the implemented SQL function to throw an exception. +** ^SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** ^If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** ^The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. +** +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. +** +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. +** +** ^The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** ^The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** ^The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** ^The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** ^SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is negative, then SQLite takes result text from the 2nd parameter +** through the first zero character. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the content of the parameter nor call a destructor on the content +** when it has finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained +** from [sqlite3_malloc()] before it returns. +** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** +** ^The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy of the +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** ^A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an +** SQL NULL value, just like [sqlite3_result_null(C)], except that it +** also associates the host-language pointer P or type T with that +** NULL value such that the pointer can be retrieved within an +** [application-defined SQL function] using [sqlite3_value_pointer()]. +** ^If the D parameter is not NULL, then it is a pointer to a destructor +** for the P parameter. ^SQLite invokes D with P as its only argument +** when SQLite is finished with P. The T parameter should be a static +** string and preferably a string literal. The sqlite3_result_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. +*/ +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_pointer(sqlite3_context*, void*,const char*,void(*)(void*)); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); + +/* +** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 +** +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. +** +** ^The name of the collation is a UTF-8 string +** for sqlite3_create_collation() and sqlite3_create_collation_v2() +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +**
    +**
  • [SQLITE_UTF8], +**
  • [SQLITE_UTF16LE], +**
  • [SQLITE_UTF16BE], +**
  • [SQLITE_UTF16], or +**
  • [SQLITE_UTF16_ALIGNED]. +**
)^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCompare. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is an application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCompare, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCompare argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The two integer parameters to the collating +** function callback are the length of the two strings, in bytes. The collating +** function must return an integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must always return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
    +**
  1. If A==B then B==A. +**
  2. If A==B and B==C then A==C. +**
  3. If A<B THEN B>A. +**
  4. If A<B and B<C then A<C. +**
+** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. +** +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** +** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_collation16( + sqlite3*, + const void *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); + +/* +** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 +** +** ^To avoid having to register all collation sequences before a database +** can be used, a single callback function may be registered with the +** [database connection] to be invoked whenever an undefined collation +** sequence is required. +** +** ^If the function is registered using the sqlite3_collation_needed() API, +** then it is passed the names of undefined collation sequences as strings +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** ^A call to either function replaces the existing collation-needed callback. +** +** ^(When the callback is invoked, the first argument passed is a copy +** of the second argument to sqlite3_collation_needed() or +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the +** required collation sequence.)^ +** +** The callback function should register the desired collation using +** [sqlite3_create_collation()], [sqlite3_create_collation16()], or +** [sqlite3_create_collation_v2()]. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const char*) +); +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const void*) +); + +#ifdef SQLITE_ENABLE_CEROD +/* +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. +*/ +SQLITE_API void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +/* +** CAPI3REF: Suspend Execution For A Short Time +** +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. +** +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually +** requested from the operating system is returned. +** +** ^SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. +*/ +SQLITE_API int sqlite3_sleep(int); + +/* +** CAPI3REF: Name Of The Folder Holding Temporary Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all temporary files +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. +** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [temp_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
+** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
+*/ +SQLITE_API SQLITE_EXTERN char *sqlite3_temp_directory; + +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; + +/* +** CAPI3REF: Win32 Specific Interface +** +** These interfaces are available only on Windows. The +** [sqlite3_win32_set_directory] interface is used to set the value associated +** with the [sqlite3_temp_directory] or [sqlite3_data_directory] variable, to +** zValue, depending on the value of the type parameter. The zValue parameter +** should be NULL to cause the previous value to be freed via [sqlite3_free]; +** a non-NULL value will be copied into memory obtained from [sqlite3_malloc] +** prior to being used. The [sqlite3_win32_set_directory] interface returns +** [SQLITE_OK] to indicate success, [SQLITE_ERROR] if the type is unsupported, +** or [SQLITE_NOMEM] if memory could not be allocated. The value of the +** [sqlite3_data_directory] variable is intended to act as a replacement for +** the current directory on the sub-platforms of Win32 where that concept is +** not present, e.g. WinRT and UWP. The [sqlite3_win32_set_directory8] and +** [sqlite3_win32_set_directory16] interfaces behave exactly the same as the +** sqlite3_win32_set_directory interface except the string parameter must be +** UTF-8 or UTF-16, respectively. +*/ +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +); +SQLITE_API int sqlite3_win32_set_directory8(unsigned long type, const char *zValue); +SQLITE_API int sqlite3_win32_set_directory16(unsigned long type, const void *zValue); + +/* +** CAPI3REF: Win32 Directory Types +** +** These macros are only available on Windows. They define the allowed values +** for the type argument to the [sqlite3_win32_set_directory] interface. +*/ +#define SQLITE_WIN32_DATA_DIRECTORY_TYPE 1 +#define SQLITE_WIN32_TEMP_DIRECTORY_TYPE 2 + +/* +** CAPI3REF: Test For Auto-Commit Mode +** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 +** +** ^The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3*); + +/* +** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename +** associated with database N of connection D. +** ^If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** this function will return either a NULL pointer or an empty string. +** +** ^The string value returned by this routine is owned and managed by +** the database connection. ^The value will be valid until the database N +** is [DETACH]-ed or until the database connection closes. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +** +** If the filename pointer returned by this routine is not NULL, then it +** can be used as the filename input parameter to these routines: +**
    +**
  • [sqlite3_uri_parameter()] +**
  • [sqlite3_uri_boolean()] +**
  • [sqlite3_uri_int64()] +**
  • [sqlite3_filename_database()] +**
  • [sqlite3_filename_journal()] +**
  • [sqlite3_filename_wal()] +**
+*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine the transaction state of a database +** METHOD: sqlite3 +** +** ^The sqlite3_txn_state(D,S) interface returns the current +** [transaction state] of schema S in database connection D. ^If S is NULL, +** then the highest transaction state of any schema on database connection D +** is returned. Transaction states are (in order of lowest to highest): +**
    +**
  1. SQLITE_TXN_NONE +**
  2. SQLITE_TXN_READ +**
  3. SQLITE_TXN_WRITE +**
+** ^If the S argument to sqlite3_txn_state(D,S) is not the name of +** a valid schema, then -1 is returned. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); + +/* +** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** KEYWORDS: {transaction state} +** +** These constants define the current transaction state of a database file. +** ^The [sqlite3_txn_state(D,S)] interface returns one of these +** constants in order to describe the transaction state of schema S +** in [database connection] D. +** +**
+** [[SQLITE_TXN_NONE]]
SQLITE_TXN_NONE
+**
The SQLITE_TXN_NONE state means that no transaction is currently +** pending.
+** +** [[SQLITE_TXN_READ]]
SQLITE_TXN_READ
+**
The SQLITE_TXN_READ state means that the database is currently +** in a read transaction. Content has been read from the database file +** but nothing in the database file has changed. The transaction state +** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** no other conflicting concurrent write transactions. The transaction +** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or +** [COMMIT].
+** +** [[SQLITE_TXN_WRITE]]
SQLITE_TXN_WRITE
+**
The SQLITE_TXN_WRITE state means that the database is currently +** in a write transaction. Content has been written to the database file +** but has not yet committed. The transaction state will change to +** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
+*/ +#define SQLITE_TXN_NONE 0 +#define SQLITE_TXN_READ 1 +#define SQLITE_TXN_WRITE 2 + +/* +** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 +** +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. ^If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is [COMMIT | committed]. +** ^Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** ^The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. +** ^Any callback set by a previous call to sqlite3_rollback_hook() +** for the same database connection is overridden. +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. +** +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. +** +** The commit and rollback hook callbacks are not reentrant. +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. +** +** ^Registering a NULL function disables the callback. +** +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook +** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. +** ^The rollback hook is invoked on a rollback that results from a commit +** hook returning non-zero, just as it would be with any other rollback. +** +** ^For the purposes of this API, a transaction is said to have been +** rolled back if an explicit "ROLLBACK" statement is executed, or +** an error or constraint causes an implicit rollback to occur. +** ^The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. +** +** See also the [sqlite3_update_hook()] interface. +*/ +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted in +** a [rowid table]. +** ^Any callback set by a previous call to this function +** for the same database connection is overridden. +** +** ^The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted in a rowid table. +** ^The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** ^The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. +** +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. +** +** ^In the current implementation, the update hook +** is not invoked when conflicting rows are deleted because of an +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook +** invoked when rows are deleted using the [truncate optimization]. +** The exceptions defined in this paragraph might change in a future +** release of SQLite. +** +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. +** +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3*, + void(*)(void *,int ,char const *,char const *,sqlite3_int64), + void* +); + +/* +** CAPI3REF: Enable Or Disable Shared Pager Cache +** +** ^(This routine enables or disables the sharing of the database cache +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false.)^ +** +** ^Cache sharing is enabled and disabled for an entire process. +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. +** +** ^(The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue to use the sharing mode +** that was in effect at the time they were opened.)^ +** +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ +** +** ^Shared cache is disabled by default. It is recommended that it stay +** that way. In other words, do not use this routine. This interface +** continues to be provided for historical compatibility, but its use is +** discouraged. Any use of shared cache is discouraged. If shared cache +** must be used, it is recommended that shared cache only be enabled for +** individual database connections using the [sqlite3_open_v2()] interface +** with the [SQLITE_OPEN_SHAREDCACHE] flag. +** +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. +** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** +** See Also: [SQLite Shared-Cache Mode] +*/ +SQLITE_API int sqlite3_enable_shared_cache(int); + +/* +** CAPI3REF: Attempt To Free Heap Memory +** +** ^The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** ^sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] +*/ +SQLITE_API int sqlite3_release_memory(int); + +/* +** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + +/* +** CAPI3REF: Impose A Limit On Heap Size +** +** These interfaces impose limits on the amount of heap memory that will be +** by all database connections within a single process. +** +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of +** N bytes on the amount of memory that will be allocated. ^The +** sqlite3_hard_heap_limit64(N) interface is similar to +** sqlite3_soft_heap_limit64(N) except that memory allocations will fail +** when the hard heap limit is reached. +** +** ^The return value from both sqlite3_soft_heap_limit64() and +** sqlite3_hard_heap_limit64() is the size of +** the heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the heap limit. Hence, the current +** size of heap limits can be determined by invoking +** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). +** +** ^Setting the heap limits to zero disables the heap limiter mechanism. +** +** ^The soft heap limit may not be greater than the hard heap limit. +** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) +** is invoked with a value of N that is greater than the hard heap limit, +** the the soft heap limit is set to the value of the hard heap limit. +** ^The soft heap limit is automatically enabled whenever the hard heap +** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and +** the soft heap limit is outside the range of 1..N, then the soft heap +** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the +** hard heap limit is enabled makes the soft heap limit equal to the +** hard heap limit. +** +** The memory allocation limits can also be adjusted using +** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. +** +** ^(The heap limits are not enforced in the current implementation +** if one or more of following conditions are true: +** +**
    +**
  • The limit value is set to zero. +**
  • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
  • An alternative page cache implementation is specified using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). +**
  • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
)^ +** +** The circumstances under which SQLite will enforce the heap limits may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + + +/* +** CAPI3REF: Extract Metadata About A Column Of A Table +** METHOD: sqlite3 +** +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existence of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. If the table name parameter T in a call to +** sqlite3_table_column_metadata(X,D,T,C,...) is NULL then the result is +** undefined behavior. +** +** ^The column is identified by the second, third and fourth parameters to +** this function. ^(The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL.)^ ^If it is NULL, then all attached databases are searched +** for the table using the same algorithm used by the database engine to +** resolve unqualified table references. +** +** ^The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. +** +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. +** +** ^(
+** +**
Parameter Output
Type 		Description +** +**
5th const char* Data type +**
6th const char* Name of default collation sequence +**
7th int True if column has a NOT NULL constraint +**
8th int True if column is part of the PRIMARY KEY +**
9th int True if column is [AUTOINCREMENT] +**
+**
)^ +** +** ^The memory pointed to by the character pointers returned for the +** declaration type and collation sequence is valid until the next +** call to any SQLite API function. +** +** ^If the specified table is actually a view, an [error code] is returned. +** +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an +** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output +** parameters are set for the explicitly declared column. ^(If there is no +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: +** +** 
+**     data type: "INTEGER"
+**     collation sequence: "BINARY"
+**     not null: 0
+**     primary key: 1
+**     auto increment: 0
+**
)^ +** +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. +*/ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +); + +/* +** CAPI3REF: Load An Extension +** METHOD: sqlite3 +** +** ^This interface loads an SQLite extension library from the named file. +** +** ^The sqlite3_load_extension() interface attempts to load an +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. +** +** ^The entry point is zProc. +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. +** +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, +** otherwise an error will be returned. +** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** +** See also the [load_extension() SQL function]. +*/ +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Derived from zFile if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +); + +/* +** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 +** +** ^So as not to open security holes in older applications that are +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API +** is provided to turn the [sqlite3_load_extension()] mechanism on and off. +** +** ^Extension loading is off by default. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); + +/* +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked [SQLite extension] +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects an integer result as if the signature of the +** entry point where as follows: +** +** 
+**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
)^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] +*/ +SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void)); + +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); + +/* +** CAPI3REF: Reset Automatic Extension Loading +** +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void); + +/* +** The interface to the virtual-table mechanism is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** Structures used by the virtual table interface +*/ +typedef struct sqlite3_vtab sqlite3_vtab; +typedef struct sqlite3_index_info sqlite3_index_info; +typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; +typedef struct sqlite3_module sqlite3_module; + +/* +** CAPI3REF: Virtual Table Object +** KEYWORDS: sqlite3_module {virtual table module} +** +** This structure, sometimes called a "virtual table module", +** defines the implementation of a [virtual table]. +** This structure consists mostly of methods for the module. +** +** ^A virtual table module is created by filling in a persistent +** instance of this structure and passing a pointer to that instance +** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. +** ^The registration remains valid until it is replaced by a different +** module or until the [database connection] closes. The content +** of this structure must not change while it is registered with +** any database connection. +*/ +struct sqlite3_module { + int iVersion; + int (*xCreate)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xConnect)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xBestIndex)(sqlite3_vtab *pVTab, sqlite3_index_info*); + int (*xDisconnect)(sqlite3_vtab *pVTab); + int (*xDestroy)(sqlite3_vtab *pVTab); + int (*xOpen)(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor); + int (*xClose)(sqlite3_vtab_cursor*); + int (*xFilter)(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); + int (*xNext)(sqlite3_vtab_cursor*); + int (*xEof)(sqlite3_vtab_cursor*); + int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int); + int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid); + int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *); + int (*xBegin)(sqlite3_vtab *pVTab); + int (*xSync)(sqlite3_vtab *pVTab); + int (*xCommit)(sqlite3_vtab *pVTab); + int (*xRollback)(sqlite3_vtab *pVTab); + int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg); + int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); + /* The methods above are in versions 1 and 2 of the sqlite_module object. + ** Those below are for version 3 and greater. */ + int (*xShadowName)(const char*); +}; + +/* +** CAPI3REF: Virtual Table Indexing Information +** KEYWORDS: sqlite3_index_info +** +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to +** pass information into and receive the reply from the [xBestIndex] +** method of a [virtual table module]. The fields under **Inputs** are the +** inputs to xBestIndex and are read-only. xBestIndex inserts its +** results into the **Outputs** fields. +** +** ^(The aConstraint[] array records WHERE clause constraints of the form: +** +**
column OP expr
+** +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the +** expr on the right-hand side can be evaluated (and thus the constraint +** is usable) and false if it cannot.)^ +** +** ^The optimizer automatically inverts terms of the form "expr OP column" +** and makes other simplifications to the WHERE clause in an attempt to +** get as many WHERE clause terms into the form shown above as possible. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. +** +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. +** +** The colUsed field indicates which columns of the virtual table may be +** required by the current scan. Virtual table columns are numbered from +** zero in the order in which they appear within the CREATE TABLE statement +** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62), +** the corresponding bit is set within the colUsed mask if the column may be +** required by SQLite. If the table has at least 64 columns and any column +** to the right of the first 63 is required, then bit 63 of colUsed is also +** set. In other words, column iCol may be required if the expression +** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to +** non-zero. +** +** The [xBestIndex] method must fill aConstraintUsage[] with information +** about what parameters to pass to xFilter. ^If argvIndex>0 then +** the right-hand side of the corresponding aConstraint[] is evaluated +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit +** is true, then the constraint is assumed to be fully handled by the +** virtual table and might not be checked again by the byte code.)^ ^(The +** aConstraintUsage[].omit flag is an optimization hint. When the omit flag +** is left in its default setting of false, the constraint will always be +** checked separately in byte code. If the omit flag is change to true, then +** the constraint may or may not be checked in byte code. In other words, +** when the omit flag is true there is no guarantee that the constraint will +** not be checked again using byte code.)^ +** +** ^The idxNum and idxPtr values are recorded and passed into the +** [xFilter] method. +** ^[sqlite3_free()] is used to free idxPtr if and only if +** needToFreeIdxPtr is true. +** +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in +** the correct order to satisfy the ORDER BY clause so that no separate +** sorting step is required. +** +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). +** If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to include crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for [version 3.9.0] ([dateof:3.9.0]). +** It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. +*/ +struct sqlite3_index_info { + /* Inputs */ + int nConstraint; /* Number of entries in aConstraint */ + struct sqlite3_index_constraint { + int iColumn; /* Column constrained. -1 for ROWID */ + unsigned char op; /* Constraint operator */ + unsigned char usable; /* True if this constraint is usable */ + int iTermOffset; /* Used internally - xBestIndex should ignore */ + } *aConstraint; /* Table of WHERE clause constraints */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + struct sqlite3_index_orderby { + int iColumn; /* Column number */ + unsigned char desc; /* True for DESC. False for ASC. */ + } *aOrderBy; /* The ORDER BY clause */ + /* Outputs */ + struct sqlite3_index_constraint_usage { + int argvIndex; /* if >0, constraint is part of argv to xFilter */ + unsigned char omit; /* Do not code a test for this constraint */ + } *aConstraintUsage; + int idxNum; /* Number used to identify the index */ + char *idxStr; /* String, possibly obtained from sqlite3_malloc */ + int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ + int orderByConsumed; /* True if output is already ordered */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ + /* Fields below are only available in SQLite 3.10.0 and later */ + sqlite3_uint64 colUsed; /* Input: Mask of columns used by statement */ +}; + +/* +** CAPI3REF: Virtual Table Scan Flags +** +** Virtual table implementations are allowed to set the +** [sqlite3_index_info].idxFlags field to some combination of +** these bits. +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros define the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 +#define SQLITE_INDEX_CONSTRAINT_NE 68 +#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 +#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 +#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 +#define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 + +/* +** CAPI3REF: Register A Virtual Table Implementation +** METHOD: sqlite3 +** +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a +** preexisting [virtual table] for the module. +** +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth +** parameter is an arbitrary client data pointer that is passed through +** into the [xCreate] and [xConnect] methods of the virtual table module +** when a new virtual table is be being created or reinitialized. +** +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is create and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData /* Client data for xCreate/xConnect */ +); +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ +); + +/* +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 +** +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + +/* +** CAPI3REF: Virtual Table Instance Object +** KEYWORDS: sqlite3_vtab +** +** Every [virtual table module] implementation uses a subclass +** of this object to describe a particular instance +** of the [virtual table]. Each subclass will +** be tailored to the specific needs of the module implementation. +** The purpose of this superclass is to define certain fields that are +** common to all module implementations. +** +** ^Virtual tables methods can set an error message by assigning a +** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should +** take care that any prior string is freed by a call to [sqlite3_free()] +** prior to assigning a new string to zErrMsg. ^After the error message +** is delivered up to the client application, the string will be automatically +** freed by sqlite3_free() and the zErrMsg field will be zeroed. +*/ +struct sqlite3_vtab { + const sqlite3_module *pModule; /* The module for this virtual table */ + int nRef; /* Number of open cursors */ + char *zErrMsg; /* Error message from sqlite3_mprintf() */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Virtual Table Cursor Object +** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} +** +** Every [virtual table module] implementation uses a subclass of the +** following structure to describe cursors that point into the +** [virtual table] and are used +** to loop through the virtual table. Cursors are created using the +** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed +** by the [sqlite3_module.xClose | xClose] method. Cursors are used +** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods +** of the module. Each module implementation will define +** the content of a cursor structure to suit its own needs. +** +** This superclass exists in order to define fields of the cursor that +** are common to all implementations. +*/ +struct sqlite3_vtab_cursor { + sqlite3_vtab *pVtab; /* Virtual table of this cursor */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Declare The Schema Of A Virtual Table +** +** ^The [xCreate] and [xConnect] methods of a +** [virtual table module] call this interface +** to declare the format (the names and datatypes of the columns) of +** the virtual tables they implement. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); + +/* +** CAPI3REF: Overload A Function For A Virtual Table +** METHOD: sqlite3 +** +** ^(Virtual tables can provide alternative implementations of functions +** using the [xFindFunction] method of the [virtual table module]. +** But global versions of those functions +** must exist in order to be overloaded.)^ +** +** ^(This API makes sure a global version of a function with a particular +** name and number of parameters exists. If no such function exists +** before this API is called, a new function is created.)^ ^The implementation +** of the new function always causes an exception to be thrown. So +** the new function is not good for anything by itself. Its only +** purpose is to be a placeholder function that can be overloaded +** by a [virtual table]. +*/ +SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); + +/* +** The interface to the virtual-table mechanism defined above (back up +** to a comment remarkably similar to this one) is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** CAPI3REF: A Handle To An Open BLOB +** KEYWORDS: {BLOB handle} {BLOB handles} +** +** An instance of this object represents an open BLOB on which +** [sqlite3_blob_open | incremental BLOB I/O] can be performed. +** ^Objects of this type are created by [sqlite3_blob_open()] +** and destroyed by [sqlite3_blob_close()]. +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** can be used to read or write small subsections of the BLOB. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. +*/ +typedef struct sqlite3_blob sqlite3_blob; + +/* +** CAPI3REF: Open A BLOB For Incremental I/O +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_blob +** +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located +** in row iRow, column zColumn, table zTable in database zDb; +** in other words, the same BLOB that would be selected by: +** +** 
+**     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
+**
)^ +** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
    +**
  • ^(Database zDb does not exist)^, +**
  • ^(Table zTable does not exist within database zDb)^, +**
  • ^(Table zTable is a WITHOUT ROWID table)^, +**
  • ^(Column zColumn does not exist)^, +**
  • ^(Row iRow is not present in the table)^, +**
  • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
  • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
  • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
+** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** A BLOB referenced by sqlite3_blob_open() may be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed after the [BLOB handle] is opened. +** +** ^(If the row that a BLOB handle points to is modified by an +** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects +** then the BLOB handle is marked as "expired". +** This is true if any column of the row is changed, even a column +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ +** +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this +** interface. Use the [UPDATE] SQL command to change the size of a +** blob. +** +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. +** +** To avoid a resource leak, every open [BLOB handle] should eventually +** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3*, + const char *zDb, + const char *zTable, + const char *zColumn, + sqlite3_int64 iRow, + int flags, + sqlite3_blob **ppBlob +); + +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob +** +** ^This function is used to move an existing [BLOB handle] so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing [BLOB handle] to a new row is +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + +/* +** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob +** +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ +** +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. +** +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); + +/* +** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob +** +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The +** incremental blob I/O routines can only read or overwriting existing +** blob content; they cannot change the size of a blob. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); + +/* +** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to read data from an open [BLOB handle] into a +** caller-supplied buffer. N bytes of data are copied into buffer Z +** from the open BLOB, starting at offset iOffset.)^ +** +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is +** less than zero, [SQLITE_ERROR] is returned and no data is read. +** ^The size of the blob (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to read from an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. +** +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); + +/* +** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** ^If the [BLOB handle] passed as the first argument was not opened for +** writing (the flags parameter to [sqlite3_blob_open()] was zero), +** this function returns [SQLITE_READONLY]. +** +** This function may only modify the contents of the BLOB; it is +** not possible to increase the size of a BLOB using this API. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. +** +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred +** before the [BLOB handle] expired are not rolled back by the +** expiration of the handle, though of course those changes might +** have been overwritten by the statement that expired the BLOB handle +** or by other independent statements. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_read()]. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + +/* +** CAPI3REF: Virtual File System Objects +** +** A virtual filesystem (VFS) is an [sqlite3_vfs] object +** that SQLite uses to interact +** with the underlying operating system. Most SQLite builds come with a +** single default VFS that is appropriate for the host computer. +** New VFSes can be registered and existing VFSes can be unregistered. +** The following interfaces are provided. +** +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. +** +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again +** with the makeDflt flag set. If two different VFSes with the +** same name are registered, the behavior is undefined. If a +** VFS is registered with a name that is NULL or an empty string, +** then the behavior is undefined. +** +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); + +/* +** CAPI3REF: Mutexes +** +** The SQLite core uses these routines for thread +** synchronization. Though they are intended for internal +** use by SQLite, code that links against SQLite is +** permitted to use any of these routines. +** +** The SQLite source code contains multiple implementations +** of these mutex routines. An appropriate implementation +** is selected automatically at compile-time. The following +** implementations are available in the SQLite core: +** +**
    +**
  • SQLITE_MUTEX_PTHREADS +**
  • SQLITE_MUTEX_W32 +**
  • SQLITE_MUTEX_NOOP +**
+** +** The SQLITE_MUTEX_NOOP implementation is a set of routines +** that does no real locking and is appropriate for use in +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. +** +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex +** implementation is included with the library. In this case the +** application must supply a custom mutex implementation using the +** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function +** before calling sqlite3_initialize() or any other public sqlite3_ +** function that calls sqlite3_initialize(). +** +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MAIN +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +**
+** +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Nine static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. ^For the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +** +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. +** +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using +** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. +** In such cases, the +** mutex must be exited an equal number of times before another thread +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. +** +** ^(Some systems (for example, Windows 95) do not support the operation +** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ +** +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered by the +** calling thread or is not currently allocated. +** +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or +** sqlite3_mutex_leave() is a NULL pointer, then all three routines +** behave as no-ops. +** +** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); + +/* +** CAPI3REF: Mutex Methods Object +** +** An instance of this structure defines the low-level routines +** used to allocate and use mutexes. +** +** Usually, the default mutex implementations provided by SQLite are +** sufficient, however the application has the option of substituting a custom +** implementation for specialized deployments or systems for which SQLite +** does not provide a suitable implementation. In this case, the application +** creates and populates an instance of this structure to pass +** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. +** Additionally, an instance of this structure can be used as an +** output variable when querying the system for the current mutex +** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. +** +** ^The xMutexInit method defined by this structure is invoked as +** part of system initialization by the sqlite3_initialize() function. +** ^The xMutexInit routine is called by SQLite exactly once for each +** effective call to [sqlite3_initialize()]. +** +** ^The xMutexEnd method defined by this structure is invoked as +** part of system shutdown by the sqlite3_shutdown() function. The +** implementation of this method is expected to release all outstanding +** resources obtained by the mutex methods implementation, especially +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. +** +** ^(The remaining seven methods defined by this structure (xMutexAlloc, +** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and +** xMutexNotheld) implement the following interfaces (respectively): +** +**
    +**
  • [sqlite3_mutex_alloc()]
    • +**
  • [sqlite3_mutex_free()]
    • +**
  • [sqlite3_mutex_enter()]
    • +**
  • [sqlite3_mutex_try()]
    • +**
  • [sqlite3_mutex_leave()]
    • +**
  • [sqlite3_mutex_held()]
    • +**
  • [sqlite3_mutex_notheld()]
    • +**
)^ +** +** The only difference is that the public sqlite3_XXX functions enumerated +** above silently ignore any invocations that pass a NULL pointer instead +** of a valid mutex handle. The implementations of the methods defined +** by this structure are not required to handle this case. The results +** of passing a NULL pointer instead of a valid mutex handle are undefined +** (i.e. it is acceptable to provide an implementation that segfaults if +** it is passed a NULL pointer). +** +** The xMutexInit() method must be threadsafe. It must be harmless to +** invoke xMutexInit() multiple times within the same process and without +** intervening calls to xMutexEnd(). Second and subsequent calls to +** xMutexInit() must be no-ops. +** +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite +** memory allocation for a fast or recursive mutex. +** +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** called, but only if the prior call to xMutexInit returned SQLITE_OK. +** If xMutexInit fails in any way, it is expected to clean up after itself +** prior to returning. +*/ +typedef struct sqlite3_mutex_methods sqlite3_mutex_methods; +struct sqlite3_mutex_methods { + int (*xMutexInit)(void); + int (*xMutexEnd)(void); + sqlite3_mutex *(*xMutexAlloc)(int); + void (*xMutexFree)(sqlite3_mutex *); + void (*xMutexEnter)(sqlite3_mutex *); + int (*xMutexTry)(sqlite3_mutex *); + void (*xMutexLeave)(sqlite3_mutex *); + int (*xMutexHeld)(sqlite3_mutex *); + int (*xMutexNotheld)(sqlite3_mutex *); +}; + +/* +** CAPI3REF: Mutex Verification Routines +** +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines +** are intended for use inside assert() statements. The SQLite core +** never uses these routines except inside an assert() and applications +** are advised to follow the lead of the core. The SQLite core only +** provides implementations for these routines when it is compiled +** with the SQLITE_DEBUG flag. External mutex implementations +** are only required to provide these routines if SQLITE_DEBUG is +** defined and if NDEBUG is not defined. +** +** These routines should return true if the mutex in their argument +** is held or not held, respectively, by the calling thread. +** +** The implementation is not required to provide versions of these +** routines that actually work. If the implementation does not provide working +** versions of these routines, it should at least provide stubs that always +** return true so that one does not get spurious assertion failures. +** +** If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since +** clearly the mutex cannot be held if it does not exist. But +** the reason the mutex does not exist is because the build is not +** using mutexes. And we do not want the assert() containing the +** call to sqlite3_mutex_held() to fail, so a non-zero return is +** the appropriate thing to do. The sqlite3_mutex_notheld() +** interface should also return 1 when given a NULL pointer. +*/ +#ifndef NDEBUG +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif + +/* +** CAPI3REF: Mutex Types +** +** The [sqlite3_mutex_alloc()] interface takes a single argument +** which is one of these integer constants. +** +** The set of static mutexes may change from one SQLite release to the +** next. Applications that override the built-in mutex logic must be +** prepared to accommodate additional static mutexes. +*/ +#define SQLITE_MUTEX_FAST 0 +#define SQLITE_MUTEX_RECURSIVE 1 +#define SQLITE_MUTEX_STATIC_MAIN 2 +#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ +#define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ +#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ + +/* Legacy compatibility: */ +#define SQLITE_MUTEX_STATIC_MASTER 2 + + +/* +** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 +** +** ^This interface returns a pointer the [sqlite3_mutex] object that +** serializes access to the [database connection] given in the argument +** when the [threading mode] is Serialized. +** ^If the [threading mode] is Single-thread or Multi-thread then this +** routine returns a NULL pointer. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); + +/* +** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 +** KEYWORDS: {file control} +** +** ^The [sqlite3_file_control()] interface makes a direct call to the +** xFileControl method for the [sqlite3_io_methods] object associated +** with a particular database identified by the second argument. ^The +** name of the database is "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine +** are passed directly through to the second and third parameters of +** the xFileControl method. ^The return value of the xFileControl +** method becomes the return value of this routine. +** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. +** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. +** +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error +** code is not remembered and will not be recalled by [sqlite3_errcode()] +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between +** an incorrect zDbName and an SQLITE_ERROR return from the underlying +** xFileControl method. +** +** See also: [file control opcodes] +*/ +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); + +/* +** CAPI3REF: Testing Interface +** +** ^The sqlite3_test_control() interface is used to read out internal +** state of SQLite and to inject faults into SQLite for testing +** purposes. ^The first parameter is an operation code that determines +** the number, meaning, and operation of all subsequent parameters. +** +** This interface is not for use by applications. It exists solely +** for verifying the correct operation of the SQLite library. Depending +** on how the SQLite library is compiled, this interface might not exist. +** +** The details of the operation codes, their meanings, the parameters +** they take, and what they do are all subject to change without notice. +** Unlike most of the SQLite API, this function is not guaranteed to +** operate consistently from one release to the next. +*/ +SQLITE_API int sqlite3_test_control(int op, ...); + +/* +** CAPI3REF: Testing Interface Operation Codes +** +** These constants are the valid operation code parameters used +** as the first argument to [sqlite3_test_control()]. +** +** These parameters and their meanings are subject to change +** without notice. These values are for testing purposes only. +** Applications should not use any of these parameters or the +** [sqlite3_test_control()] interface. +*/ +#define SQLITE_TESTCTRL_FIRST 5 +#define SQLITE_TESTCTRL_PRNG_SAVE 5 +#define SQLITE_TESTCTRL_PRNG_RESTORE 6 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_BITVEC_TEST 8 +#define SQLITE_TESTCTRL_FAULT_INSTALL 9 +#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 +#define SQLITE_TESTCTRL_PENDING_BYTE 11 +#define SQLITE_TESTCTRL_ASSERT 12 +#define SQLITE_TESTCTRL_ALWAYS 13 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_SEEK_COUNT 30 +#define SQLITE_TESTCTRL_TRACEFLAGS 31 +#define SQLITE_TESTCTRL_LAST 31 /* Largest TESTCTRL */ + +/* +** CAPI3REF: SQL Keyword Checking +** +** These routines provide access to the set of SQL language keywords +** recognized by SQLite. Applications can uses these routines to determine +** whether or not a specific identifier needs to be escaped (for example, +** by enclosing in double-quotes) so as not to confuse the parser. +** +** The sqlite3_keyword_count() interface returns the number of distinct +** keywords understood by SQLite. +** +** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** makes *Z point to that keyword expressed as UTF8 and writes the number +** of bytes in the keyword into *L. The string that *Z points to is not +** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns +** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z +** or L are NULL or invalid pointers then calls to +** sqlite3_keyword_name(N,Z,L) result in undefined behavior. +** +** The sqlite3_keyword_check(Z,L) interface checks to see whether or not +** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero +** if it is and zero if not. +** +** The parser used by SQLite is forgiving. It is often possible to use +** a keyword as an identifier as long as such use does not result in a +** parsing ambiguity. For example, the statement +** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and +** creates a new table named "BEGIN" with three columns named +** "REPLACE", "PRAGMA", and "END". Nevertheless, best practice is to avoid +** using keywords as identifiers. Common techniques used to avoid keyword +** name collisions include: +**
    +**
  • Put all identifier names inside double-quotes. This is the official +** SQL way to escape identifier names. +**
  • Put identifier names inside [...]. This is not standard SQL, +** but it is what SQL Server does and so lots of programmers use this +** technique. +**
  • Begin every identifier with the letter "Z" as no SQL keywords start +** with "Z". +**
  • Include a digit somewhere in every identifier name. +**
+** +** Note that the number of keywords understood by SQLite can depend on +** compile-time options. For example, "VACUUM" is not a keyword if +** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option. Also, +** new keywords may be added to future releases of SQLite. +*/ +SQLITE_API int sqlite3_keyword_count(void); +SQLITE_API int sqlite3_keyword_name(int,const char**,int*); +SQLITE_API int sqlite3_keyword_check(const char*,int); + +/* +** CAPI3REF: Dynamic String Object +** KEYWORDS: {dynamic string} +** +** An instance of the sqlite3_str object contains a dynamically-sized +** string under construction. +** +** The lifecycle of an sqlite3_str object is as follows: +**
    +**
  1. ^The sqlite3_str object is created using [sqlite3_str_new()]. +**
  2. ^Text is appended to the sqlite3_str object using various +** methods, such as [sqlite3_str_appendf()]. +**
  3. ^The sqlite3_str object is destroyed and the string it created +** is returned using the [sqlite3_str_finish()] interface. +**
+*/ +typedef struct sqlite3_str sqlite3_str; + +/* +** CAPI3REF: Create A New Dynamic String Object +** CONSTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_new(D)] interface allocates and initializes +** a new [sqlite3_str] object. To avoid memory leaks, the object returned by +** [sqlite3_str_new()] must be freed by a subsequent call to +** [sqlite3_str_finish(X)]. +** +** ^The [sqlite3_str_new(D)] interface always returns a pointer to a +** valid [sqlite3_str] object, though in the event of an out-of-memory +** error the returned object might be a special singleton that will +** silently reject new text, always return SQLITE_NOMEM from +** [sqlite3_str_errcode()], always return 0 for +** [sqlite3_str_length()], and always return NULL from +** [sqlite3_str_finish(X)]. It is always safe to use the value +** returned by [sqlite3_str_new(D)] as the sqlite3_str parameter +** to any of the other [sqlite3_str] methods. +** +** The D parameter to [sqlite3_str_new(D)] may be NULL. If the +** D parameter in [sqlite3_str_new(D)] is not NULL, then the maximum +** length of the string contained in the [sqlite3_str] object will be +** the value set for [sqlite3_limit](D,[SQLITE_LIMIT_LENGTH]) instead +** of [SQLITE_MAX_LENGTH]. +*/ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); + +/* +** CAPI3REF: Finalize A Dynamic String +** DESTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_finish(X)] interface destroys the sqlite3_str object X +** and returns a pointer to a memory buffer obtained from [sqlite3_malloc64()] +** that contains the constructed string. The calling application should +** pass the returned value to [sqlite3_free()] to avoid a memory leak. +** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any +** errors were encountered during construction of the string. ^The +** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** string in [sqlite3_str] object X is zero bytes long. +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str*); + +/* +** CAPI3REF: Add Content To A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces add content to an sqlite3_str object previously obtained +** from [sqlite3_str_new()]. +** +** ^The [sqlite3_str_appendf(X,F,...)] and +** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] +** functionality of SQLite to append formatted text onto the end of +** [sqlite3_str] object X. +** +** ^The [sqlite3_str_append(X,S,N)] method appends exactly N bytes from string S +** onto the end of the [sqlite3_str] object X. N must be non-negative. +** S must contain at least N non-zero bytes of content. To append a +** zero-terminated string in its entirety, use the [sqlite3_str_appendall()] +** method instead. +** +** ^The [sqlite3_str_appendall(X,S)] method appends the complete content of +** zero-terminated string S onto the end of [sqlite3_str] object X. +** +** ^The [sqlite3_str_appendchar(X,N,C)] method appends N copies of the +** single-byte character C onto the end of [sqlite3_str] object X. +** ^This method can be used, for example, to add whitespace indentation. +** +** ^The [sqlite3_str_reset(X)] method resets the string under construction +** inside [sqlite3_str] object X back to zero bytes in length. +** +** These methods do not return a result code. ^If an error occurs, that fact +** is recorded in the [sqlite3_str] object and can be recovered by a +** subsequent call to [sqlite3_str_errcode(X)]. +*/ +SQLITE_API void sqlite3_str_appendf(sqlite3_str*, const char *zFormat, ...); +SQLITE_API void sqlite3_str_vappendf(sqlite3_str*, const char *zFormat, va_list); +SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); +SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); +SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); +SQLITE_API void sqlite3_str_reset(sqlite3_str*); + +/* +** CAPI3REF: Status Of A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces return the current status of an [sqlite3_str] object. +** +** ^If any prior errors have occurred while constructing the dynamic string +** in sqlite3_str X, then the [sqlite3_str_errcode(X)] method will return +** an appropriate error code. ^The [sqlite3_str_errcode(X)] method returns +** [SQLITE_NOMEM] following any out-of-memory error, or +** [SQLITE_TOOBIG] if the size of the dynamic string exceeds +** [SQLITE_MAX_LENGTH], or [SQLITE_OK] if there have been no errors. +** +** ^The [sqlite3_str_length(X)] method returns the current length, in bytes, +** of the dynamic string under construction in [sqlite3_str] object X. +** ^The length returned by [sqlite3_str_length(X)] does not include the +** zero-termination byte. +** +** ^The [sqlite3_str_value(X)] method returns a pointer to the current +** content of the dynamic string under construction in X. The value +** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X +** and might be freed or altered by any subsequent method on the same +** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str_value(X)] after any subsequent method call on the same +** object. ^Applications may change the content of the string returned +** by [sqlite3_str_value(X)] as long as they do not write into any bytes +** outside the range of 0 to [sqlite3_str_length(X)] and do not read or +** write any byte after any subsequent sqlite3_str method call. +*/ +SQLITE_API int sqlite3_str_errcode(sqlite3_str*); +SQLITE_API int sqlite3_str_length(sqlite3_str*); +SQLITE_API char *sqlite3_str_value(sqlite3_str*); + +/* +** CAPI3REF: SQLite Runtime Status +** +** ^These interfaces are used to retrieve runtime status information +** about the performance of SQLite, and optionally to reset various +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [status parameters | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. ^(Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ +** +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. +** +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. +** +** See also: [sqlite3_db_status()] +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); + + +/* +** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
+** [[SQLITE_STATUS_MEMORY_USED]] ^(
SQLITE_STATUS_MEMORY_USED
+**
This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as reported by the xSize method in [sqlite3_mem_methods].
)^ +** +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
SQLITE_STATUS_MALLOC_SIZE
+**
This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
)^ +** +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
SQLITE_STATUS_MALLOC_COUNT
+**
This parameter records the number of separate memory allocations +** currently checked out.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
SQLITE_STATUS_PAGECACHE_USED
+**
This parameter returns the number of pages used out of the +** [pagecache memory allocator] that was configured using +** [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** ^(
SQLITE_STATUS_PAGECACHE_OVERFLOW
+**
This parameter returns the number of bytes of page cache +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()]. The +** returned value includes allocations that overflowed because they +** where too large (they were larger than the "sz" parameter to +** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because +** no space was left in the page cache.
)^ +** +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
SQLITE_STATUS_PAGECACHE_SIZE
+**
This parameter records the largest memory allocation request +** handed to the [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
)^ +** +** [[SQLITE_STATUS_SCRATCH_USED]]
SQLITE_STATUS_SCRATCH_USED
+**
No longer used.
+** +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
+**
No longer used.
+** +** [[SQLITE_STATUS_SCRATCH_SIZE]]
SQLITE_STATUS_SCRATCH_SIZE
+**
No longer used.
+** +** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
+**
The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ +**
+** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 /* NOT USED */ +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_SIZE 5 +#define SQLITE_STATUS_PARSER_STACK 6 +#define SQLITE_STATUS_PAGECACHE_SIZE 7 +#define SQLITE_STATUS_SCRATCH_SIZE 8 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_COUNT 9 + +/* +** CAPI3REF: Database Connection Status +** METHOD: sqlite3 +** +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is an integer constant, taken from the set of +** [SQLITE_DBSTATUS options], that +** determines the parameter to interrogate. The set of +** [SQLITE_DBSTATUS options] is likely +** to grow in future releases of SQLite. +** +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If +** the resetFlg is true, then the highest instantaneous value is +** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. +*/ +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); + +/* +** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} +** +** These constants are the available integer "verbs" that can be passed as +** the second argument to the [sqlite3_db_status()] interface. +** +** New verbs may be added in future releases of SQLite. Existing verbs +** might be discontinued. Applications should check the return code from +** [sqlite3_db_status()] to make sure that the call worked. +** The [sqlite3_db_status()] interface will return a non-zero error code +** if a discontinued or unsupported verb is invoked. +** +**
+** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
+**
This parameter returns the number of lookaside memory slots currently +** checked out.
)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
+**
This parameter returns the number of malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
+**
This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
+**
This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
+**
This parameter returns the approximate number of bytes of heap +** memory used by all pager caches associated with the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] +** ^(
SQLITE_DBSTATUS_CACHE_USED_SHARED
+**
This parameter is similar to DBSTATUS_CACHE_USED, except that if a +** pager cache is shared between two or more connections the bytes of heap +** memory used by that pager cache is divided evenly between the attached +** connections.)^ In other words, if none of the pager caches associated +** with the database connection are shared, this request returns the same +** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** shared, the value returned by this call will be smaller than that returned +** by DBSTATUS_CACHE_USED. ^The highwater mark associated with +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
+**
This parameter returns the approximate number of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
+**
This parameter returns the approximate number of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
SQLITE_DBSTATUS_CACHE_HIT
+**
This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
SQLITE_DBSTATUS_CACHE_MISS
+**
This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
SQLITE_DBSTATUS_CACHE_SPILL
+**
This parameter returns the number of dirty cache entries that have +** been written to disk in the middle of a transaction due to the page +** cache overflowing. Transactions are more efficient if they are written +** to disk all at once. When pages spill mid-transaction, that introduces +** additional overhead. This parameter can be used help identify +** inefficiencies that can be resolved by increasing the cache size. +**
+** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
SQLITE_DBSTATUS_DEFERRED_FKS
+**
This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
+**
+*/ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 +#define SQLITE_DBSTATUS_CACHE_SPILL 12 +#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ + + +/* +** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt +** +** ^(Each prepared statement maintains various +** [SQLITE_STMTSTATUS counters] that measure the number +** of times it has performed specific operations.)^ These counters can +** be used to monitor the performance characteristics of the prepared +** statements. For example, if the number of table steps greatly exceeds +** the number of table searches or result rows, that would tend to indicate +** that the prepared statement is using a full table scan rather than +** an index. +** +** ^(This interface is used to retrieve and reset counter values from +** a [prepared statement]. The first argument is the prepared statement +** object to be interrogated. The second argument +** is an integer code for a specific [SQLITE_STMTSTATUS counter] +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this +** interface call returns. +** +** See also: [sqlite3_status()] and [sqlite3_db_status()]. +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); + +/* +** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} +** +** These preprocessor macros define integer codes that name counter +** values associated with the [sqlite3_stmt_status()] interface. +** The meanings of the various counters are as follows: +** +**
+** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
SQLITE_STMTSTATUS_FULLSCAN_STEP
+**
^This is the number of times that SQLite has stepped forward in +** a table as part of a full table scan. Large numbers for this counter +** may indicate opportunities for performance improvement through +** careful use of indices.
+** +** [[SQLITE_STMTSTATUS_SORT]]
SQLITE_STMTSTATUS_SORT
+**
^This is the number of sort operations that have occurred. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance through careful use of indices.
+** +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
SQLITE_STMTSTATUS_AUTOINDEX
+**
^This is the number of rows inserted into transient indices that +** were created automatically in order to help joins run faster. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance by adding permanent indices that do not +** need to be reinitialized each time the statement is run.
+** +** [[SQLITE_STMTSTATUS_VM_STEP]]
SQLITE_STMTSTATUS_VM_STEP
+**
^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +** +** [[SQLITE_STMTSTATUS_REPREPARE]]
SQLITE_STMTSTATUS_REPREPARE
+**
^This is the number of times that the prepare statement has been +** automatically regenerated due to schema changes or changes to +** [bound parameters] that might affect the query plan. +** +** [[SQLITE_STMTSTATUS_RUN]]
SQLITE_STMTSTATUS_RUN
+**
^This is the number of times that the prepared statement has +** been run. A single "run" for the purposes of this counter is one +** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. +** The counter is incremented on the first [sqlite3_step()] call of each +** cycle. +** +** [[SQLITE_STMTSTATUS_MEMUSED]]
SQLITE_STMTSTATUS_MEMUSED
+**
^This is the approximate number of bytes of heap memory +** used to store the prepared statement. ^This value is not actually +** a counter, and so the resetFlg parameter to sqlite3_stmt_status() +** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. +**
+**
+*/ +#define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 +#define SQLITE_STMTSTATUS_SORT 2 +#define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 +#define SQLITE_STMTSTATUS_REPREPARE 5 +#define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_MEMUSED 99 + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache type is opaque. It is implemented by +** the pluggable module. The SQLite core has no knowledge of +** its size or internal structure and never deals with the +** sqlite3_pcache object except by holding and passing pointers +** to the object. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache sqlite3_pcache; + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* +** CAPI3REF: Application Defined Page Cache. +** KEYWORDS: {page cache} +** +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can +** register an alternative page cache implementation by passing in an +** instance of the sqlite3_pcache_methods2 structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for +** how long. +** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an +** internal buffer by SQLite within the call to [sqlite3_config]. Hence +** the application may discard the parameter after the call to +** [sqlite3_config()] returns.)^ +** +** [[the xInit() page cache method]] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ +** +** [[the xShutdown() page cache method]] +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up +** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. +** +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. All other methods must be threadsafe +** in multithreaded applications. +** +** ^SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +** +** [[the xCreate() page cache methods]] +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, +** though this is not guaranteed. ^The +** first parameter, szPage, is the size in bytes of the pages that must +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends +** on the SQLite version, the target platform, and how SQLite was compiled. +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or +** false if it is used for an in-memory database. The cache implementation +** does not have to do anything special based with the value of bPurgeable; +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will +** never invoke xUnpin() except to deliberately delete a page. +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will +** never contain any unpinned pages. +** +** [[the xCachesize() page cache method]] +** ^(The xCachesize() method may be called at any time by SQLite to set the +** suggested maximum cache-size (number of pages stored by) the cache +** instance passed as the first argument. This is the value configured using +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable +** parameter, the implementation is not required to do anything with this +** value; it is advisory only. +** +** [[the xPagecount() page cache methods]] +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. +** +** [[the xFetch() page cache methods]] +** The xFetch() method locates a page in the cache and returns a pointer to +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". +** +** If the requested page is already in the page cache, then the page cache +** implementation must return a pointer to the page buffer with its content +** intact. If the requested page is not already in the cache, then the +** cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: +** +** +**
createFlag Behavior when page is not already in cache +**
0 Do not allocate a new page. Return NULL. +**
1 Allocate a new page if it easy and convenient to do so. +** Otherwise return NULL. +**
2 Make every effort to allocate a new page. Only return +** NULL if allocating a new page is effectively impossible. +**
+** +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the xFetch() calls, SQLite may +** attempt to unpin one or more cache pages by spilling the content of +** pinned pages to disk and synching the operating system disk cache. +** +** [[the xUnpin() page cache method]] +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation +** may choose to evict unpinned pages at any time. +** +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls +** to xFetch(). +** +** [[the xRekey() page cache methods]] +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be +** discarded. ^Any prior cache entry associated with newKey is guaranteed not +** to be pinned. +** +** When SQLite calls the xTruncate() method, the cache must discard all +** existing cache entries with page numbers (keys) greater than or equal +** to the value of the iLimit parameter passed to xTruncate(). If any +** of these pages are pinned, they are implicitly unpinned, meaning that +** they can be safely discarded. +** +** [[the xDestroy() page cache method]] +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After +** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 +** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. +*/ +typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; +struct sqlite3_pcache_methods { + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, void*, int discard); + void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); +}; + + +/* +** CAPI3REF: Online Backup Object +** +** The sqlite3_backup object records state information about an ongoing +** online backup operation. ^The sqlite3_backup object is created by +** a call to [sqlite3_backup_init()] and is destroyed by a call to +** [sqlite3_backup_finish()]. +** +** See Also: [Using the SQLite Online Backup API] +*/ +typedef struct sqlite3_backup sqlite3_backup; + +/* +** CAPI3REF: Online Backup API. +** +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** See Also: [Using the SQLite Online Backup API] +** +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from +** reading or writing to the source database while the backup is underway. +** +** ^(To perform a backup operation: +**
    +**
  1. sqlite3_backup_init() is called once to initialize the +** backup, +**
  2. sqlite3_backup_step() is called one or more times to transfer +** the data between the two databases, and finally +**
  3. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. +**
)^ +** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** [[sqlite3_backup_init()]] sqlite3_backup_init() +** +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with +** an error. +** +** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** there is already a read or read-write transaction open on the +** destination database. +** +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are stored in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** [[sqlite3_backup_step()]] sqlite3_backup_step() +** +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function returns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and +** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], +** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. +** +** ^(The sqlite3_backup_step() might return [SQLITE_READONLY] if +**
    +**
  1. the destination database was opened read-only, or +**
  2. the destination database is using write-ahead-log journaling +** and the destination and source page sizes differ, or +**
  3. the destination database is an in-memory database and the +** destination and source page sizes differ. +**
)^ +** +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the [sqlite3_busy_handler | busy-handler function] +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source +** [database connection] +** is being used to write to the source database when sqlite3_backup_step() +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is automatically +** updated at the same time. +** +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() +** +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. +** +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of +** sqlite3_backup_finish(). +** +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ +** +** Concurrent Usage of Database Handles +** +** ^The source [database connection] may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** ^If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also also cause a mutex deadlock. +** +** If running in [shared cache mode], the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the disk file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); + +/* +** CAPI3REF: Unlock Notification +** METHOD: sqlite3 +** +** ^When running in shared-cache mode, a database operation may fail with +** an [SQLITE_LOCKED] error if the required locks on the shared-cache or +** individual tables within the shared-cache cannot be obtained. See +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke +** when the connection currently holding the required lock relinquishes it. +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. +** +** See Also: [Using the SQLite Unlock Notification Feature]. +** +** ^Shared-cache locks are released when a database connection concludes +** its current transaction, either by committing it or rolling it back. +** +** ^When a connection (known as the blocked connection) fails to obtain a +** shared-cache lock and SQLITE_LOCKED is returned to the caller, the +** identity of the database connection (the blocking connection) that +** has locked the required resource is stored internally. ^After an +** application receives an SQLITE_LOCKED error, it may call the +** sqlite3_unlock_notify() method with the blocked connection handle as +** the first argument to register for a callback that will be invoked +** when the blocking connections current transaction is concluded. ^The +** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +** call that concludes the blocking connection's transaction. +** +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, +** there is a chance that the blocking connection will have already +** concluded its transaction by the time sqlite3_unlock_notify() is invoked. +** If this happens, then the specified callback is invoked immediately, +** from within the call to sqlite3_unlock_notify().)^ +** +** ^If the blocked connection is attempting to obtain a write-lock on a +** shared-cache table, and more than one other connection currently holds +** a read-lock on the same table, then SQLite arbitrarily selects one of +** the other connections to use as the blocking connection. +** +** ^(There may be at most one unlock-notify callback registered by a +** blocked connection. If sqlite3_unlock_notify() is called when the +** blocked connection already has a registered unlock-notify callback, +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is +** called with a NULL pointer as its second argument, then any existing +** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback may also be canceled by closing the blocked +** connection using [sqlite3_close()]. +** +** The unlock-notify callback is not reentrant. If an application invokes +** any sqlite3_xxx API functions from within an unlock-notify callback, a +** crash or deadlock may be the result. +** +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** returns SQLITE_OK. +** +** Callback Invocation Details +** +** When an unlock-notify callback is registered, the application provides a +** single void* pointer that is passed to the callback when it is invoked. +** However, the signature of the callback function allows SQLite to pass +** it an array of void* context pointers. The first argument passed to +** an unlock-notify callback is a pointer to an array of void* pointers, +** and the second is the number of entries in the array. +** +** When a blocking connection's transaction is concluded, there may be +** more than one blocked connection that has registered for an unlock-notify +** callback. ^If two or more such blocked connections have specified the +** same callback function, then instead of invoking the callback function +** multiple times, it is invoked once with the set of void* context pointers +** specified by the blocked connections bundled together into an array. +** This gives the application an opportunity to prioritize any actions +** related to the set of unblocked database connections. +** +** Deadlock Detection +** +** Assuming that after registering for an unlock-notify callback a +** database waits for the callback to be issued before taking any further +** action (a reasonable assumption), then using this API may cause the +** application to deadlock. For example, if connection X is waiting for +** connection Y's transaction to be concluded, and similarly connection +** Y is waiting on connection X's transaction, then neither connection +** will proceed and the system may remain deadlocked indefinitely. +** +** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock +** detection. ^If a given call to sqlite3_unlock_notify() would put the +** system in a deadlocked state, then SQLITE_LOCKED is returned and no +** unlock-notify callback is registered. The system is said to be in +** a deadlocked state if connection A has registered for an unlock-notify +** callback on the conclusion of connection B's transaction, and connection +** B has itself registered for an unlock-notify callback when connection +** A's transaction is concluded. ^Indirect deadlock is also detected, so +** the system is also considered to be deadlocked if connection B has +** registered for an unlock-notify callback on the conclusion of connection +** C's transaction, where connection C is waiting on connection A. ^Any +** number of levels of indirection are allowed. +** +** The "DROP TABLE" Exception +** +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** always appropriate to call sqlite3_unlock_notify(). There is however, +** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, +** SQLite checks if there are any currently executing SELECT statements +** that belong to the same connection. If there are, SQLITE_LOCKED is +** returned. In this case there is no "blocking connection", so invoking +** sqlite3_unlock_notify() results in the unlock-notify callback being +** invoked immediately. If the application then re-attempts the "DROP TABLE" +** or "DROP INDEX" query, an infinite loop might be the result. +** +** One way around this problem is to check the extended error code returned +** by an sqlite3_step() call. ^(If there is a blocking connection, then the +** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in +** the special "DROP TABLE/INDEX" case, the extended error code is just +** SQLITE_LOCKED.)^ +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *pBlocked, /* Waiting connection */ + void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ + void *pNotifyArg /* Argument to pass to xNotify */ +); + + +/* +** CAPI3REF: String Comparison +** +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); + +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if +** string X matches the [GLOB] pattern P. +** ^The definition of [GLOB] pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function +** is case sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strlike()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* +** CAPI3REF: String LIKE Matching +* +** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if +** string X matches the [LIKE] pattern P with escape character E. +** ^The definition of [LIKE] pattern matching used in +** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" +** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without +** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. +** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case +** insensitive - equivalent upper and lower case ASCII characters match +** one another. +** +** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though +** only ASCII characters are case folded. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strglob()]. +*/ +SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); + +/* +** CAPI3REF: Error Logging Interface +** +** ^The [sqlite3_log()] interface writes a message into the [error log] +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** used with [sqlite3_snprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); + +/* +** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_hook()] function is used to register a callback that +** is invoked each time data is committed to a database in wal mode. +** +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation +** may read, write or [checkpoint] the database as required. +** +** ^The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_wal_hook() when +** registering the callback. ^The second is a copy of the database handle. +** ^The third parameter is the name of the database that was written to - +** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter +** is the number of pages currently in the write-ahead log file, +** including those that were just committed. +** +** The callback function should normally return [SQLITE_OK]. ^If an error +** code is returned, that error will propagate back up through the +** SQLite code base to cause the statement that provoked the callback +** to report an error, though the commit will have still occurred. If the +** callback returns [SQLITE_ROW] or [SQLITE_DONE], or if it returns a value +** that does not correspond to any valid SQLite error code, the results +** are undefined. +** +** A single database handle may have at most a single write-ahead log callback +** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any +** previously registered write-ahead log callback. ^Note that the +** [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will +** overwrite any prior [sqlite3_wal_hook()] settings. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + +/* +** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around +** [sqlite3_wal_hook()] that causes any database on [database connection] D +** to automatically [checkpoint] +** after committing a transaction if there are N or +** more frames in the [write-ahead log] file. ^Passing zero or +** a negative value as the nFrame parameter disables automatic +** checkpoints entirely. +** +** ^The callback registered by this function replaces any existing callback +** registered using [sqlite3_wal_hook()]. ^Likewise, registering a callback +** using [sqlite3_wal_hook()] disables the automatic checkpoint mechanism +** configured by this function. +** +** ^The [wal_autocheckpoint pragma] can be used to invoke this interface +** from SQL. +** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** +** ^Every new [database connection] defaults to having the auto-checkpoint +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface +** is only necessary if the default setting is found to be suboptimal +** for a particular application. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ +** +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. +** +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ +** +**
+**
SQLITE_CHECKPOINT_PASSIVE
+** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. +** +**
SQLITE_CHECKPOINT_FULL
+** ^This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. +** +**
SQLITE_CHECKPOINT_RESTART
+** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
SQLITE_CHECKPOINT_TRUNCATE
+** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. +**
+** +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. ^SQLITE_BUSY is returned in this case. +** +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned at the end. ^If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code is returned to the caller immediately. ^If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} +** +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** In the call sqlite3_vtab_config(D,C,...) the D parameter is the +** [database connection] in which the virtual table is being created and +** which is passed in as the first argument to the [xConnect] or [xCreate] +** method that is invoking sqlite3_vtab_config(). The C parameter is one +** of the [virtual table configuration options]. The presence and meaning +** of parameters after C depend on which [virtual table configuration option] +** is used. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** KEYWORDS: {virtual table configuration options} +** KEYWORDS: {virtual table configuration option} +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
+** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +**
SQLITE_VTAB_CONSTRAINT_SUPPORT
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+** +** [[SQLITE_VTAB_DIRECTONLY]]
SQLITE_VTAB_DIRECTONLY
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** prohibits that virtual table from being used from within triggers and +** views. +**
+** +** [[SQLITE_VTAB_INNOCUOUS]]
SQLITE_VTAB_INNOCUOUS
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** identify that virtual table as being safe to use from within triggers +** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the +** virtual table can do no serious harm even if it is controlled by a +** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS +** flag unless absolutely necessary. +**
+**
+*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +#define SQLITE_VTAB_INNOCUOUS 2 +#define SQLITE_VTAB_DIRECTONLY 3 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE +** +** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn] +** method of a [virtual table], then it might return true if the +** column is being fetched as part of an UPDATE operation during which the +** column value will not change. The virtual table implementation can use +** this hint as permission to substitute a return value that is less +** expensive to compute and that the corresponding +** [xUpdate] method understands as a "no-change" value. +** +** If the [xColumn] method calls sqlite3_vtab_nochange() and finds that +** the column is not changed by the UPDATE statement, then the xColumn +** method can optionally return without setting a result, without calling +** any of the [sqlite3_result_int|sqlite3_result_xxxxx() interfaces]. +** In that case, [sqlite3_value_nochange(X)] will return true for the +** same column in the [xUpdate] method. +** +** The sqlite3_vtab_nochange() routine is an optimization. Virtual table +** implementations should continue to give a correct answer even if the +** sqlite3_vtab_nochange() interface were to always return false. In the +** current implementation, the sqlite3_vtab_nochange() interface does always +** returns false for the enhanced [UPDATE FROM] statement. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); + +/* +** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** +** This function may only be called from within a call to the [xBestIndex] +** method of a [virtual table]. +** +** The first argument must be the sqlite3_index_info object that is the +** first parameter to the xBestIndex() method. The second argument must be +** an index into the aConstraint[] array belonging to the sqlite3_index_info +** structure passed to xBestIndex. This function returns a pointer to a buffer +** containing the name of the collation sequence for the corresponding +** constraint. +*/ +SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + +/* +** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +**
+** [[SQLITE_SCANSTAT_NLOOP]]
SQLITE_SCANSTAT_NLOOP
+**
^The [sqlite3_int64] variable pointed to by the V parameter will be +** set to the total number of times that the X-th loop has run.
+** +** [[SQLITE_SCANSTAT_NVISIT]]
SQLITE_SCANSTAT_NVISIT
+**
^The [sqlite3_int64] variable pointed to by the V parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
+** +** [[SQLITE_SCANSTAT_EST]]
SQLITE_SCANSTAT_EST
+**
^The "double" variable pointed to by the V parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME
+**
^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
SQLITE_SCANSTAT_EXPLAIN
+**
^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
SQLITE_SCANSTAT_SELECT
+**
^The "int" variable pointed to by the V parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
+*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 + +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** This interface returns information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + +/* +** CAPI3REF: Flush caches to disk mid-transaction +** METHOD: sqlite3 +** +** ^If a write-transaction is open on [database connection] D when the +** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] +** interface flushes caches for all schemas - "main", "temp", and +** any [attached] databases. +** +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. ^If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. ^If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** ^If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite [error code] is returned to the caller immediately. +** +** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. +** +** ^This function does not set the database handle error code or message +** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3*); + +/* +** CAPI3REF: The pre-update hook. +** METHOD: sqlite3 +** +** ^These interfaces are only available if SQLite is compiled using the +** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. +** +** ^The [sqlite3_preupdate_hook()] interface registers a callback function +** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation +** on a database table. +** ^At most one preupdate hook may be registered at a time on a single +** [database connection]; each call to [sqlite3_preupdate_hook()] overrides +** the previous setting. +** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()] +** with a NULL pointer as the second parameter. +** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as +** the first parameter to callbacks. +** +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_sequence or sqlite_stat1. +** +** ^The second parameter to the preupdate callback is a pointer to +** the [database connection] that registered the preupdate hook. +** ^The third parameter to the preupdate callback is one of the constants +** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the +** kind of update operation that is about to occur. +** ^(The fourth parameter to the preupdate callback is the name of the +** database within the database connection that is being modified. This +** will be "main" for the main database or "temp" for TEMP tables or +** the name given after the AS keyword in the [ATTACH] statement for attached +** databases.)^ +** ^The fifth parameter to the preupdate callback is the name of the +** table that is being modified. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** DELETE operations on rowid tables. +** +** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], +** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces +** provide additional information about a preupdate event. These routines +** may only be called from within a preupdate callback. Invoking any of +** these routines from outside of a preupdate callback or with a +** [database connection] pointer that is different from the one supplied +** to the preupdate callback results in undefined and probably undesirable +** behavior. +** +** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns +** in the row that is being inserted, updated, or deleted. +** +** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row before it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE +** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row after it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE +** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate +** callback was invoked as a result of a direct insert, update, or delete +** operation; or 1 for inserts, updates, or deletes invoked by top-level +** triggers; or 2 for changes resulting from triggers called by top-level +** triggers; and so forth. +** +** See also: [sqlite3_update_hook()] +*/ +#if defined(SQLITE_ENABLE_PREUPDATE_HOOK) +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, + void(*xPreUpdate)( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ + ), + void* +); +SQLITE_API int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_count(sqlite3 *); +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *); +SQLITE_API int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +#endif + +/* +** CAPI3REF: Low-level system error code +** METHOD: sqlite3 +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most recent I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +SQLITE_API int sqlite3_system_errno(sqlite3*); + +/* +** CAPI3REF: Database Snapshot +** KEYWORDS: {snapshot} {sqlite3_snapshot} +** +** An instance of the snapshot object records the state of a [WAL mode] +** database for some specific point in history. +** +** In [WAL mode], multiple [database connections] that are open on the +** same database file can each be reading a different historical version +** of the database file. When a [database connection] begins a read +** transaction, that connection sees an unchanging copy of the database +** as it existed for the point in time when the transaction first started. +** Subsequent changes to the database from other connections are not seen +** by the reader until a new read transaction is started. +** +** The sqlite3_snapshot object records state information about an historical +** version of the database file so that it is possible to later open a new read +** transaction that sees that historical version of the database rather than +** the most recent version. +*/ +typedef struct sqlite3_snapshot { + unsigned char dummyhidden[48]; +} sqlite3_snapshot; + +/* +** CAPI3REF: Record A Database Snapshot +** CONSTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a +** new [sqlite3_snapshot] object that records the current state of +** schema S in database connection D. ^On success, the +** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly +** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. +** If there is not already a read-transaction open on schema S when +** this function is called, one is opened automatically. +** +** The following must be true for this function to succeed. If any of +** the following statements are false when sqlite3_snapshot_get() is +** called, SQLITE_ERROR is returned. The final value of *P is undefined +** in this case. +** +**
    +**
  • The database handle must not be in [autocommit mode]. +** +**
  • Schema S of [database connection] D must be a [WAL mode] database. +** +**
  • There must not be a write transaction open on schema S of database +** connection D. +** +**
  • One or more transactions must have been written to the current wal +** file since it was created on disk (by any connection). This means +** that a snapshot cannot be taken on a wal mode database with no wal +** file immediately after it is first opened. At least one transaction +** must be written to it first. +**
+** +** This function may also return SQLITE_NOMEM. If it is called with the +** database handle in autocommit mode but fails for some other reason, +** whether or not a read transaction is opened on schema S is undefined. +** +** The [sqlite3_snapshot] object returned from a successful call to +** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] +** to avoid a memory leak. +** +** The [sqlite3_snapshot_get()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot **ppSnapshot +); + +/* +** CAPI3REF: Start a read transaction on an historical snapshot +** METHOD: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. +** +** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the +** database connection D does not know that the database file for +** schema S is in [WAL mode]. A database connection might not know +** that the database file is in [WAL mode] if there has been no prior +** I/O on that database connection, or if the database entered [WAL mode] +** after the most recent I/O on the database connection.)^ +** (Hint: Run "[PRAGMA application_id]" against a newly opened +** database connection in order to make it ready to use snapshots.) +** +** The [sqlite3_snapshot_open()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot *pSnapshot +); + +/* +** CAPI3REF: Destroy a snapshot +** DESTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. +** The application must eventually free every [sqlite3_snapshot] object +** using this routine to avoid a memory leak. +** +** The [sqlite3_snapshot_free()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); + +/* +** CAPI3REF: Compare the ages of two snapshot handles. +** METHOD: sqlite3_snapshot +** +** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages +** of two valid snapshot handles. +** +** If the two snapshot handles are not associated with the same database +** file, the result of the comparison is undefined. +** +** Additionally, the result of the comparison is only valid if both of the +** snapshot handles were obtained by calling sqlite3_snapshot_get() since the +** last time the wal file was deleted. The wal file is deleted when the +** database is changed back to rollback mode or when the number of database +** clients drops to zero. If either snapshot handle was obtained before the +** wal file was last deleted, the value returned by this function +** is undefined. +** +** Otherwise, this API returns a negative value if P1 refers to an older +** snapshot than P2, zero if the two handles refer to the same database +** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( + sqlite3_snapshot *p1, + sqlite3_snapshot *p2 +); + +/* +** CAPI3REF: Recover snapshots from a wal file +** METHOD: sqlite3_snapshot +** +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. +** +** This function attempts to scan the WAL file associated with database zDb +** of database handle db and make all valid snapshots available to +** sqlite3_snapshot_open(). It is an error if there is already a read +** transaction open on the database, or if the database is not a WAL mode +** database. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Serialize a database +** +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory +** that is a serialization of the S database on [database connection] D. +** If P is not a NULL pointer, then the size of the database in bytes +** is written into *P. +** +** For an ordinary on-disk database file, the serialization is just a +** copy of the disk file. For an in-memory database or a "TEMP" database, +** the serialization is the same sequence of bytes which would be written +** to disk if that database where backed up to disk. +** +** The usual case is that sqlite3_serialize() copies the serialization of +** the database into memory obtained from [sqlite3_malloc64()] and returns +** a pointer to that memory. The caller is responsible for freeing the +** returned value to avoid a memory leak. However, if the F argument +** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations +** are made, and the sqlite3_serialize() function will return a pointer +** to the contiguous memory representation of the database that SQLite +** is currently using for that database, or NULL if the no such contiguous +** memory representation of the database exists. A contiguous memory +** representation of the database will usually only exist if there has +** been a prior call to [sqlite3_deserialize(D,S,...)] with the same +** values of D and S. +** The size of the database is written into *P even if the +** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy +** of the database exists. +** +** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the +** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory +** allocation error occurs. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_DESERIALIZE] option. +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to serialize. ex: "main", "temp", ... */ + sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */ + unsigned int mFlags /* Zero or more SQLITE_SERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_serialize +** +** Zero or more of the following constants can be OR-ed together for +** the F argument to [sqlite3_serialize(D,S,P,F)]. +** +** SQLITE_SERIALIZE_NOCOPY means that [sqlite3_serialize()] will return +** a pointer to contiguous in-memory database that it is currently using, +** without making a copy of the database. If SQLite is not currently using +** a contiguous in-memory database, then this option causes +** [sqlite3_serialize()] to return a NULL pointer. SQLite will only be +** using a contiguous in-memory database if it has been initialized by a +** prior call to [sqlite3_deserialize()]. +*/ +#define SQLITE_SERIALIZE_NOCOPY 0x001 /* Do no memory allocations */ + +/* +** CAPI3REF: Deserialize a database +** +** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the +** [database connection] D to disconnect from database S and then +** reopen S as an in-memory database based on the serialization contained +** in P. The serialized database P is N bytes in size. M is the size of +** the buffer P, which might be larger than N. If M is larger than N, and +** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is +** permitted to add content to the in-memory database as long as the total +** size does not exceed M bytes. +** +** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will +** invoke sqlite3_free() on the serialization buffer when the database +** connection closes. If the SQLITE_DESERIALIZE_RESIZEABLE bit is set, then +** SQLite will try to increase the buffer size using sqlite3_realloc64() +** if writes on the database cause it to grow larger than M bytes. +** +** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the +** database is currently in a read transaction or is involved in a backup +** operation. +** +** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the +** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then +** [sqlite3_free()] is invoked on argument P prior to returning. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_DESERIALIZE] option. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_deserialize() +** +** The following are allowed values for 6th argument (the F argument) to +** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. +** +** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization +** in the P argument is held in memory obtained from [sqlite3_malloc64()] +** and that SQLite should take ownership of this memory and automatically +** free it when it has finished using it. Without this flag, the caller +** is responsible for freeing any dynamically allocated memory. +** +** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to +** grow the size of the database using calls to [sqlite3_realloc64()]. This +** flag should only be used if SQLITE_DESERIALIZE_FREEONCLOSE is also used. +** Without this flag, the deserialized database cannot increase in size beyond +** the number of bytes specified by the M parameter. +** +** The SQLITE_DESERIALIZE_READONLY flag means that the deserialized database +** should be treated as read-only. +*/ +#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */ +#define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ +#define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ + +/* +** Undo the hack that converts floating point types to integer for +** builds on processors without floating point support. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# undef double +#endif + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif +#endif /* SQLITE3_H */ + +/******** Begin file sqlite3rtree.h *********/ +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visibility */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + +/******** End of sqlite3rtree.h *********/ +/******** Begin file sqlite3session.h *********/ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + + +/* +** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object acts as a cursor for iterating +** over the elements of a [changeset] or [patchset]. +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +SQLITE_API int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); + + +/* +** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +SQLITE_API int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
    +**
  • The session object "indirect" flag is set when the change is +** made, or +**
  • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
+** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +SQLITE_API int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +** +**

Special sqlite_stat1 Handling

+** +** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to +** some of the rules above. In SQLite, the schema of sqlite_stat1 is: +** 
+**        CREATE TABLE sqlite_stat1(tbl,idx,stat)
+**
+** +** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are +** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes +** are recorded for rows for which (idx IS NULL) is true. However, for such +** rows a zero-length blob (SQL value X'') is stored in the changeset or +** patchset instead of a NULL value. This allows such changesets to be +** manipulated by legacy implementations of sqlite3changeset_invert(), +** concat() and similar. +** +** The sqlite3changeset_apply() function automatically converts the +** zero-length blob back to a NULL value when updating the sqlite_stat1 +** table. However, if the application calls sqlite3changeset_new(), +** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset +** iterator directly (including on a changeset iterator passed to a +** conflict-handler callback) then the X'' value is returned. The application +** must translate X'' to NULL itself if required. +** +** Legacy (older than 3.22.0) versions of the sessions module cannot capture +** changes made to the sqlite_stat1 table. Legacy versions of the +** sqlite3changeset_apply() function silently ignore any modifications to the +** sqlite_stat1 table that are part of a changeset or patchset. +*/ +SQLITE_API int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes are not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +SQLITE_API void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

Changeset Generation

+** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
    +**
  • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
  • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
+** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +SQLITE_API int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
    +**
  • Has the same name, +**
  • Has the same set of columns declared in the same order, and +**
  • Has the same PRIMARY KEY definition. +**
+** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
    +**
  • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
  • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
  • For each row (primary key) that exists in both tables, but features +** different non-PK values in each, an UPDATE record is added to the +** session. +**
+** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +SQLITE_API int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session +** +** The differences between a patchset and a changeset are that: +** +**
    +**
  • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
  • The original values of any modified fields are omitted from +** UPDATE records. +**
+** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +SQLITE_API int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppPatchset */ + void **ppPatchset /* OUT: Buffer containing patchset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Query for the amount of heap memory used by a session object. +** +** This API returns the total amount of heap memory in bytes currently +** used by the session object passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3session_memory_used(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
    +**
  • [sqlite3changeset_next()] +**
  • [sqlite3changeset_op()] +**
  • [sqlite3changeset_new()] +**
  • [sqlite3changeset_old()] +**
+** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +**
SQLITE_CHANGESETAPPLY_INVERT
+** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 + + +/* +** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function may only be used with iterators created by the function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** Arguments pOp, pnCol and pzTab may not be NULL. Upon return, three +** outputs are set through these pointers: +** +** *pOp is set to one of [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], +** depending on the type of change that the iterator currently points to; +** +** *pnCol is set to the number of columns in the table affected by the change; and +** +** *pzTab is set to point to a nul-terminated utf-8 encoded string containing +** the name of the table affected by the current change. The buffer remains +** valid until either sqlite3changeset_next() is called on the iterator +** or until the conflict-handler function returns. +** +** If pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +SQLITE_API int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter +** +** For each modified table, a changeset includes the following: +** +**
    +**
  • The number of columns in the table, and +**
  • Which of those columns make up the tables PRIMARY KEY. +**
+** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +SQLITE_API int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +SQLITE_API int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** 
+**   sqlite3changeset_start();
+**   while( SQLITE_ROW==sqlite3changeset_next() ){
+**     // Do something with change.
+**   }
+**   rc = sqlite3changeset_finalize();
+**   if( rc!=SQLITE_OK ){
+**     // An error has occurred
+**   }
+**
+*/ +SQLITE_API int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
    +**
  • Each DELETE change is changed to an INSERT, and +**
  • Each INSERT change is changed to a DELETE, and +**
  • For each UPDATE change, the old.* and new.* values are exchanged. +**
+** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +SQLITE_API int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** 
+**   sqlite3_changegroup *pGrp;
+**   rc = sqlite3_changegroup_new(&pGrp);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB);
+**   if( rc==SQLITE_OK ){
+**     rc = sqlite3changegroup_output(pGrp, pnOut, ppOut);
+**   }else{
+**     *ppOut = 0;
+**     *pnOut = 0;
+**   }
+**
+** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +SQLITE_API int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
    +**
  • It is created using a call to sqlite3changegroup_new(). +** +**
  • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
  • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
  • The object is deleted using a call to sqlite3changegroup_delete(). +**
+** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
Existing Change New Change Output Change +**
INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
+** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. If the input changeset +** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is +** returned. Or, if an out-of-memory condition occurs during processing, this +** function returns SQLITE_NOMEM. In all cases, if an error occurs the state +** of the final contents of the changegroup is undefined. +** +** If no error occurs, SQLITE_OK is returned. +*/ +SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup +*/ +SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset or patchset to a database. These functions attempt to +** update the "main" database attached to handle db with the changes found in +** the changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to these functions is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument as the first. If the "filter callback" +** returns zero, then no attempt is made to apply any changes to the table. +** Otherwise, if the return value is non-zero or the xFilter argument to +** is NULL, all changes related to the table are attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
    +**
  • The table has the same name as the name recorded in the +** changeset, and +**
  • The table has at least as many columns as recorded in the +** changeset, and +**
  • The table has primary key columns in the same position as +** recorded in the changeset. +**
+** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
+**
DELETE Changes
+** For each DELETE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
INSERT Changes
+** For each INSERT change, an attempt is made to insert the new row into +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
UPDATE Changes
+** For each UPDATE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
+** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the application's conflict +** resolution strategy. +** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +** +** If the output parameters (ppRebase) and (pnRebase) are non-NULL and +** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() +** may set (*ppRebase) to point to a "rebase" that may be used with the +** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase) +** is set to the size of the buffer in bytes. It is the responsibility of the +** caller to eventually free any such buffer using sqlite3_free(). The buffer +** is only allocated and populated if one or more conflicts were encountered +** while applying the patchset. See comments surrounding the sqlite3_rebaser +** APIs for further details. +** +** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter. +** +** Note that the sqlite3changeset_apply_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_apply_v2 +** +** The following flags may passed via the 9th parameter to +** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]: +** +**
+**
SQLITE_CHANGESETAPPLY_NOSAVEPOINT
+** Usually, the sessions module encloses all operations performed by +** a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The +** SAVEPOINT is committed if the changeset or patchset is successfully +** applied, or rolled back if an error occurs. Specifying this flag +** causes the sessions module to omit this savepoint. In this case, if the +** caller has an open transaction or savepoint when apply_v2() is called, +** it may revert the partially applied changeset by rolling it back. +** +**
SQLITE_CHANGESETAPPLY_INVERT
+** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
+**
SQLITE_CHANGESET_DATA
+** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
SQLITE_CHANGESET_NOTFOUND
+** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
SQLITE_CHANGESET_CONFLICT
+** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
SQLITE_CHANGESET_FOREIGN_KEY
+** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
SQLITE_CHANGESET_CONSTRAINT
+** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
+*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
+**
SQLITE_CHANGESET_OMIT
+** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
SQLITE_CHANGESET_REPLACE
+** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
SQLITE_CHANGESET_ABORT
+** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
+*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Rebasing changesets +** EXPERIMENTAL +** +** Suppose there is a site hosting a database in state S0. And that +** modifications are made that move that database to state S1 and a +** changeset recorded (the "local" changeset). Then, a changeset based +** on S0 is received from another site (the "remote" changeset) and +** applied to the database. The database is then in state +** (S1+"remote"), where the exact state depends on any conflict +** resolution decisions (OMIT or REPLACE) made while applying "remote". +** Rebasing a changeset is to update it to take those conflict +** resolution decisions into account, so that the same conflicts +** do not have to be resolved elsewhere in the network. +** +** For example, if both the local and remote changesets contain an +** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)": +** +** local: INSERT INTO t1 VALUES(1, 'v1'); +** remote: INSERT INTO t1 VALUES(1, 'v2'); +** +** and the conflict resolution is REPLACE, then the INSERT change is +** removed from the local changeset (it was overridden). Or, if the +** conflict resolution was "OMIT", then the local changeset is modified +** to instead contain: +** +** UPDATE t1 SET b = 'v2' WHERE a=1; +** +** Changes within the local changeset are rebased as follows: +** +**
+**
Local INSERT
+** This may only conflict with a remote INSERT. If the conflict +** resolution was OMIT, then add an UPDATE change to the rebased +** changeset. Or, if the conflict resolution was REPLACE, add +** nothing to the rebased changeset. +** +**
Local DELETE
+** This may conflict with a remote UPDATE or DELETE. In both cases the +** only possible resolution is OMIT. If the remote operation was a +** DELETE, then add no change to the rebased changeset. If the remote +** operation was an UPDATE, then the old.* fields of change are updated +** to reflect the new.* values in the UPDATE. +** +**
Local UPDATE
+** This may conflict with a remote UPDATE or DELETE. If it conflicts +** with a DELETE, and the conflict resolution was OMIT, then the update +** is changed into an INSERT. Any undefined values in the new.* record +** from the update change are filled in using the old.* values from +** the conflicting DELETE. Or, if the conflict resolution was REPLACE, +** the UPDATE change is simply omitted from the rebased changeset. +** +** If conflict is with a remote UPDATE and the resolution is OMIT, then +** the old.* values are rebased using the new.* values in the remote +** change. Or, if the resolution is REPLACE, then the change is copied +** into the rebased changeset with updates to columns also updated by +** the conflicting remote UPDATE removed. If this means no columns would +** be updated, the change is omitted. +**
+** +** A local change may be rebased against multiple remote changes +** simultaneously. If a single key is modified by multiple remote +** changesets, they are combined as follows before the local changeset +** is rebased: +** +**
    +**
  • If there has been one or more REPLACE resolutions on a +** key, it is rebased according to a REPLACE. +** +**
  • If there have been no REPLACE resolutions on a key, then +** the local changeset is rebased according to the most recent +** of the OMIT resolutions. +**
+** +** Note that conflict resolutions from multiple remote changesets are +** combined on a per-field basis, not per-row. This means that in the +** case of multiple remote UPDATE operations, some fields of a single +** local change may be rebased for REPLACE while others are rebased for +** OMIT. +** +** In order to rebase a local changeset, the remote changeset must first +** be applied to the local database using sqlite3changeset_apply_v2() and +** the buffer of rebase information captured. Then: +** +**
    +**
  1. An sqlite3_rebaser object is created by calling +** sqlite3rebaser_create(). +**
  2. The new object is configured with the rebase buffer obtained from +** sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure(). +** If the local changeset is to be rebased against multiple remote +** changesets, then sqlite3rebaser_configure() should be called +** multiple times, in the same order that the multiple +** sqlite3changeset_apply_v2() calls were made. +**
  3. Each local changeset is rebased by calling sqlite3rebaser_rebase(). +**
  4. The sqlite3_rebaser object is deleted by calling +** sqlite3rebaser_delete(). +**
+*/ +typedef struct sqlite3_rebaser sqlite3_rebaser; + +/* +** CAPI3REF: Create a changeset rebaser object. +** EXPERIMENTAL +** +** Allocate a new changeset rebaser object. If successful, set (*ppNew) to +** point to the new object and return SQLITE_OK. Otherwise, if an error +** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) +** to NULL. +*/ +SQLITE_API int sqlite3rebaser_create(sqlite3_rebaser **ppNew); + +/* +** CAPI3REF: Configure a changeset rebaser object. +** EXPERIMENTAL +** +** Configure the changeset rebaser object to rebase changesets according +** to the conflict resolutions described by buffer pRebase (size nRebase +** bytes), which must have been obtained from a previous call to +** sqlite3changeset_apply_v2(). +*/ +SQLITE_API int sqlite3rebaser_configure( + sqlite3_rebaser*, + int nRebase, const void *pRebase +); + +/* +** CAPI3REF: Rebase a changeset +** EXPERIMENTAL +** +** Argument pIn must point to a buffer containing a changeset nIn bytes +** in size. This function allocates and populates a buffer with a copy +** of the changeset rebased according to the configuration of the +** rebaser object passed as the first argument. If successful, (*ppOut) +** is set to point to the new buffer containing the rebased changeset and +** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the +** responsibility of the caller to eventually free the new buffer using +** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) +** are set to zero and an SQLite error code returned. +*/ +SQLITE_API int sqlite3rebaser_rebase( + sqlite3_rebaser*, + int nIn, const void *pIn, + int *pnOut, void **ppOut +); + +/* +** CAPI3REF: Delete a changeset rebaser object. +** EXPERIMENTAL +** +** Delete the changeset rebaser object and all associated resources. There +** should be one call to this function for each successful invocation +** of sqlite3rebaser_create(). +*/ +SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p); + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
Streaming functionNon-streaming equivalent
sqlite3changeset_apply_strm[sqlite3changeset_apply] +**
sqlite3changeset_apply_strm_v2[sqlite3changeset_apply_v2] +**
sqlite3changeset_concat_strm[sqlite3changeset_concat] +**
sqlite3changeset_invert_strm[sqlite3changeset_invert] +**
sqlite3changeset_start_strm[sqlite3changeset_start] +**
sqlite3session_changeset_strm[sqlite3session_changeset] +**
sqlite3session_patchset_strm[sqlite3session_patchset] +**
+** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
+**        int nChangeset,
+**        void *pChangeset,
+**
+** +** Is replaced by: +** +** 
+**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
+** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
+**        int *pnChangeset,
+**        void **ppChangeset,
+**
+** +** Is replaced by: +** +** 
+**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
+** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +SQLITE_API int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); +SQLITE_API int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); +SQLITE_API int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3rebaser_rebase_strm( + sqlite3_rebaser *pRebaser, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
+**
SQLITE_SESSION_CONFIG_STRMSIZE
+** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
+** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ + +/******** End of sqlite3session.h *********/ +/******** Begin file fts5.h *********/ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
+** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + +/******** End of fts5.h *********/ diff --git a/third_party/sqlite3/sqlite3.mk b/third_party/sqlite3/sqlite3.mk new file mode 100644 index 000000000..2c702ee8d --- /dev/null +++ b/third_party/sqlite3/sqlite3.mk @@ -0,0 +1,173 @@ +#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ +#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ +# +# OVERVIEW +# +# SQLite Embedded Database +# +# NOTES +# +# Please be warned that locks currently do nothing on Windows since +# figuring out how to polyfill them correctly is a work in progress +# Further note we currently don't do that thing SQLite does for Mac +# file locks so your dbase will only be as reliable as Apple wanted +# it to be when they wrote their POSIX file locking implementation. + +PKGS += THIRD_PARTY_SQLITE3 + +THIRD_PARTY_SQLITE3_ARTIFACTS += THIRD_PARTY_SQLITE3_A +THIRD_PARTY_SQLITE3 = $(THIRD_PARTY_SQLITE3_A_DEPS) $(THIRD_PARTY_SQLITE3_A) +THIRD_PARTY_SQLITE3_A = o/$(MODE)/third_party/sqlite3/libsqlite3.a +THIRD_PARTY_SQLITE3_A_FILES := $(wildcard third_party/sqlite3/*) +THIRD_PARTY_SQLITE3_A_HDRS = $(filter %.h,$(THIRD_PARTY_SQLITE3_A_FILES)) +THIRD_PARTY_SQLITE3_A_SRCS_C = $(filter %.c,$(THIRD_PARTY_SQLITE3_A_FILES)) +THIRD_PARTY_SQLITE3_A_SRCS_T = $(filter %.inc,$(THIRD_PARTY_SQLITE3_A_FILES)) +THIRD_PARTY_SQLITE3_BINS = $(THIRD_PARTY_SQLITE3_COMS) $(THIRD_PARTY_SQLITE3_COMS:%=%.dbg) + +THIRD_PARTY_SQLITE3_A_SRCS = \ + $(THIRD_PARTY_SQLITE3_A_SRCS_C) \ + $(THIRD_PARTY_SQLITE3_A_SRCS_T) + +THIRD_PARTY_SQLITE3_A_OBJS = \ + $(filter-out %/shell.o,$(THIRD_PARTY_SQLITE3_A_SRCS_C:%.c=o/$(MODE)/%.o)) + +THIRD_PARTY_SQLITE3_SHELL_OBJS = \ + $(THIRD_PARTY_SQLITE3_A_SRCS_C:%.c=o/$(MODE)/%.shell.o) + +THIRD_PARTY_SQLITE3_COMS = \ + o/$(MODE)/third_party/sqlite3/sqlite3.com + +THIRD_PARTY_SQLITE3_A_CHECKS = \ + $(THIRD_PARTY_SQLITE3_A).pkg \ + $(THIRD_PARTY_SQLITE3_A_HDRS:%=o/$(MODE)/%.ok) + +THIRD_PARTY_SQLITE3_A_DIRECTDEPS = \ + LIBC_ALG \ + LIBC_BITS \ + LIBC_CALLS \ + LIBC_FMT \ + LIBC_INTRIN \ + LIBC_MEM \ + LIBC_NEXGEN32E \ + LIBC_RUNTIME \ + LIBC_STDIO \ + LIBC_STR \ + LIBC_STUBS \ + LIBC_SYSV \ + LIBC_SYSV_CALLS \ + LIBC_TIME \ + LIBC_TINYMATH \ + LIBC_UNICODE \ + THIRD_PARTY_GDTOA \ + THIRD_PARTY_MUSL \ + THIRD_PARTY_ZLIB + +THIRD_PARTY_SQLITE3_A_DEPS := \ + $(call uniq,$(foreach x,$(THIRD_PARTY_SQLITE3_A_DIRECTDEPS),$($(x)))) + +o/$(MODE)/third_party/sqlite3/sqlite3.com.dbg: \ + $(THIRD_PARTY_SQLITE3_A_DEPS) \ + $(THIRD_PARTY_SQLITE3_SHELL_OBJS) \ + o/$(MODE)/third_party/sqlite3/shell.shell.o \ + o/$(MODE)/third_party/sqlite3/shell.pkg \ + $(CRT) \ + $(APE) + -@$(APELINK) + +$(THIRD_PARTY_SQLITE3_A): \ + third_party/sqlite3/ \ + $(THIRD_PARTY_SQLITE3_A).pkg \ + $(THIRD_PARTY_SQLITE3_A_OBJS) + +$(THIRD_PARTY_SQLITE3_A).pkg: \ + $(THIRD_PARTY_SQLITE3_A_OBJS) \ + $(foreach x,$(THIRD_PARTY_SQLITE3_A_DIRECTDEPS),$($(x)_A).pkg) + +o/$(MODE)/third_party/sqlite3/shell.pkg: \ + $(THIRD_PARTY_SQLITE3_SHELL_OBJS) \ + $(foreach x,$(THIRD_PARTY_SQLITE3_A_DIRECTDEPS),$($(x)_A).pkg) + +# https://www.sqlite.org/compile.html +THIRD_PARTY_SQLITE3_FLAGS = \ + -DNDEBUG \ + -DSQLITE_CORE \ + -DSQLITE_OS_UNIX \ + -DBUILD_sqlite \ + -DHAVE_USLEEP \ + -DHAVE_GMTIME_R \ + -DHAVE_FDATASYNC \ + -DHAVE_STRCHRNUL \ + -DHAVE_LOCALTIME_R \ + -DHAVE_MALLOC_USABLE_SIZE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_MAX_EXPR_DEPTH=0 \ + -DSQLITE_DEFAULT_MEMSTATUS=0 \ + -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \ + -DSQLITE_LIKE_DOESNT_MATCH_BLOBS \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_TCL_VARIABLE \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_OMIT_AUTOINIT \ + -DSQLITE_OMIT_GET_TABLE \ + -DSQLITE_HAVE_C99_MATH_FUNCS \ + -DSQLITE_ENABLE_MATH_FUNCTIONS + +$(THIRD_PARTY_SQLITE3_A_OBJS): \ + OVERRIDE_CFLAGS += \ + $(THIRD_PARTY_SQLITE3_FLAGS) \ + -DSQLITE_OMIT_TRACE \ + -DSQLITE_OMIT_VACUUM \ + -DSQLITE_OMIT_EXPLAIN \ + -DSQLITE_OMIT_ANALYZE \ + -DSQLITE_OMIT_COMPLETE \ + -DSQLITE_OMIT_ALTERTABLE \ + -DSQLITE_OMIT_UPDATE_HOOK \ + -DSQLITE_OMIT_AUTHORIZATION \ + -DSQLITE_OMIT_PROGRESS_CALLBACK + +$(THIRD_PARTY_SQLITE3_SHELL_OBJS): \ + OVERRIDE_CFLAGS += \ + $(THIRD_PARTY_SQLITE3_FLAGS) \ + -DHAVE_READLINE=0 \ + -DHAVE_EDITLINE=0 \ + -DSQLITE_HAVE_ZLIB \ + -DSQLITE_ENABLE_IOTRACE \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_COLUMN_METADATA \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_JSON1 + +o/$(MODE)/third_party/sqlite3/shell.shell.o: \ + OVERRIDE_CFLAGS += \ + -DSTACK_FRAME_UNLIMITED + +o/$(MODE)/%.shell.o: %.c + @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) $< + +THIRD_PARTY_SQLITE3_LIBS = $(foreach x,$(THIRD_PARTY_SQLITE3_ARTIFACTS),$($(x))) +THIRD_PARTY_SQLITE3_SRCS = $(foreach x,$(THIRD_PARTY_SQLITE3_ARTIFACTS),$($(x)_SRCS)) +THIRD_PARTY_SQLITE3_HDRS = $(foreach x,$(THIRD_PARTY_SQLITE3_ARTIFACTS),$($(x)_HDRS)) +THIRD_PARTY_SQLITE3_CHECKS = $(foreach x,$(THIRD_PARTY_SQLITE3_ARTIFACTS),$($(x)_CHECKS)) +THIRD_PARTY_SQLITE3_OBJS = $(foreach x,$(THIRD_PARTY_SQLITE3_ARTIFACTS),$($(x)_OBJS)) +$(THIRD_PARTY_SQLITE3_OBJS): third_party/sqlite3/sqlite3.mk +$(THIRD_PARTY_SQLITE3_SHELL_OBJS): third_party/sqlite3/sqlite3.mk + +.PHONY: o/$(MODE)/third_party/sqlite3 +o/$(MODE)/third_party/sqlite3: \ + $(THIRD_PARTY_SQLITE3_BINS) \ + $(THIRD_PARTY_SQLITE3_CHECKS) diff --git a/third_party/sqlite3/sqlite3ext.h b/third_party/sqlite3/sqlite3ext.h new file mode 100644 index 000000000..c836347fc --- /dev/null +++ b/third_party/sqlite3/sqlite3ext.h @@ -0,0 +1,655 @@ +#ifndef SQLITE3EXT_H +#define SQLITE3EXT_H +#include "third_party/sqlite3/sqlite3.h" +/* clang-format off */ + +/* +** This header file defines the SQLite interface for use by +** shared libraries that want to be imported as extensions into +** an SQLite instance. Shared libraries that intend to be loaded +** as extensions by SQLite should #include this file instead of +** sqlite3.h. +*/ + +/* +** The following structure holds pointers to all of the SQLite API +** routines. +** +** WARNING: In order to maintain backwards compatibility, add new +** interfaces to the end of this structure only. If you insert new +** interfaces in the middle of this structure, then older different +** versions of SQLite will not be able to load each other's shared +** libraries! +*/ +struct sqlite3_api_routines { + void * (*aggregate_context)(sqlite3_context*,int nBytes); + int (*aggregate_count)(sqlite3_context*); + int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); + int (*bind_double)(sqlite3_stmt*,int,double); + int (*bind_int)(sqlite3_stmt*,int,int); + int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); + int (*bind_null)(sqlite3_stmt*,int); + int (*bind_parameter_count)(sqlite3_stmt*); + int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); + const char * (*bind_parameter_name)(sqlite3_stmt*,int); + int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); + int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); + int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); + int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); + int (*busy_timeout)(sqlite3*,int ms); + int (*changes)(sqlite3*); + int (*close)(sqlite3*); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); + const void * (*column_blob)(sqlite3_stmt*,int iCol); + int (*column_bytes)(sqlite3_stmt*,int iCol); + int (*column_bytes16)(sqlite3_stmt*,int iCol); + int (*column_count)(sqlite3_stmt*pStmt); + const char * (*column_database_name)(sqlite3_stmt*,int); + const void * (*column_database_name16)(sqlite3_stmt*,int); + const char * (*column_decltype)(sqlite3_stmt*,int i); + const void * (*column_decltype16)(sqlite3_stmt*,int); + double (*column_double)(sqlite3_stmt*,int iCol); + int (*column_int)(sqlite3_stmt*,int iCol); + sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); + const char * (*column_name)(sqlite3_stmt*,int); + const void * (*column_name16)(sqlite3_stmt*,int); + const char * (*column_origin_name)(sqlite3_stmt*,int); + const void * (*column_origin_name16)(sqlite3_stmt*,int); + const char * (*column_table_name)(sqlite3_stmt*,int); + const void * (*column_table_name16)(sqlite3_stmt*,int); + const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); + const void * (*column_text16)(sqlite3_stmt*,int iCol); + int (*column_type)(sqlite3_stmt*,int iCol); + sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); + void * (*commit_hook)(sqlite3*,int(*)(void*),void*); + int (*complete)(const char*sql); + int (*complete16)(const void*sql); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); + int (*data_count)(sqlite3_stmt*pStmt); + sqlite3 * (*db_handle)(sqlite3_stmt*); + int (*declare_vtab)(sqlite3*,const char*); + int (*enable_shared_cache)(int); + int (*errcode)(sqlite3*db); + const char * (*errmsg)(sqlite3*); + const void * (*errmsg16)(sqlite3*); + int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); + int (*expired)(sqlite3_stmt*); + int (*finalize)(sqlite3_stmt*pStmt); + void (*free)(void*); + void (*free_table)(char**result); + int (*get_autocommit)(sqlite3*); + void * (*get_auxdata)(sqlite3_context*,int); + int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); + int (*global_recover)(void); + void (*interruptx)(sqlite3*); + sqlite_int64 (*last_insert_rowid)(sqlite3*); + const char * (*libversion)(void); + int (*libversion_number)(void); + void *(*malloc)(int); + char * (*mprintf)(const char*,...); + int (*open)(const char*,sqlite3**); + int (*open16)(const void*,sqlite3**); + int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); + void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); + void *(*realloc)(void*,int); + int (*reset)(sqlite3_stmt*pStmt); + void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_double)(sqlite3_context*,double); + void (*result_error)(sqlite3_context*,const char*,int); + void (*result_error16)(sqlite3_context*,const void*,int); + void (*result_int)(sqlite3_context*,int); + void (*result_int64)(sqlite3_context*,sqlite_int64); + void (*result_null)(sqlite3_context*); + void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); + void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_value)(sqlite3_context*,sqlite3_value*); + void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); + void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); + char * (*xsnprintf)(int,char*,const char*,...); + int (*step)(sqlite3_stmt*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); + void (*thread_cleanup)(void); + int (*total_changes)(sqlite3*); + void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); + int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); + void * (*user_data)(sqlite3_context*); + const void * (*value_blob)(sqlite3_value*); + int (*value_bytes)(sqlite3_value*); + int (*value_bytes16)(sqlite3_value*); + double (*value_double)(sqlite3_value*); + int (*value_int)(sqlite3_value*); + sqlite_int64 (*value_int64)(sqlite3_value*); + int (*value_numeric_type)(sqlite3_value*); + const unsigned char * (*value_text)(sqlite3_value*); + const void * (*value_text16)(sqlite3_value*); + const void * (*value_text16be)(sqlite3_value*); + const void * (*value_text16le)(sqlite3_value*); + int (*value_type)(sqlite3_value*); + char *(*vmprintf)(const char*,va_list); + /* Added ??? */ + int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); + /* Added by 3.3.13 */ + int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + int (*clear_bindings)(sqlite3_stmt*); + /* Added by 3.4.1 */ + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); + /* Added by 3.5.0 */ + int (*bind_zeroblob)(sqlite3_stmt*,int,int); + int (*blob_bytes)(sqlite3_blob*); + int (*blob_close)(sqlite3_blob*); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); + int (*blob_read)(sqlite3_blob*,void*,int,int); + int (*blob_write)(sqlite3_blob*,const void*,int,int); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); + int (*file_control)(sqlite3*,const char*,int,void*); + sqlite3_int64 (*memory_highwater)(int); + sqlite3_int64 (*memory_used)(void); + sqlite3_mutex *(*mutex_alloc)(int); + void (*mutex_enter)(sqlite3_mutex*); + void (*mutex_free)(sqlite3_mutex*); + void (*mutex_leave)(sqlite3_mutex*); + int (*mutex_try)(sqlite3_mutex*); + int (*open_v2)(const char*,sqlite3**,int,const char*); + int (*release_memory)(int); + void (*result_error_nomem)(sqlite3_context*); + void (*result_error_toobig)(sqlite3_context*); + int (*sleep)(int); + void (*soft_heap_limit)(int); + sqlite3_vfs *(*vfs_find)(const char*); + int (*vfs_register)(sqlite3_vfs*,int); + int (*vfs_unregister)(sqlite3_vfs*); + int (*xthreadsafe)(void); + void (*result_zeroblob)(sqlite3_context*,int); + void (*result_error_code)(sqlite3_context*,int); + int (*test_control)(int, ...); + void (*randomness)(int,void*); + sqlite3 *(*context_db_handle)(sqlite3_context*); + int (*extended_result_codes)(sqlite3*,int); + int (*limit)(sqlite3*,int,int); + sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); + const char *(*sql)(sqlite3_stmt*); + int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*xvsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); + /* Version 3.10.0 and later */ + int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); + int (*strlike)(const char*,const char*,unsigned int); + int (*db_cacheflush)(sqlite3*); + /* Version 3.12.0 and later */ + int (*system_errno)(sqlite3*); + /* Version 3.14.0 and later */ + int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*); + char *(*expanded_sql)(sqlite3_stmt*); + /* Version 3.18.0 and later */ + void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64); + /* Version 3.20.0 and later */ + int (*prepare_v3)(sqlite3*,const char*,int,unsigned int, + sqlite3_stmt**,const char**); + int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int, + sqlite3_stmt**,const void**); + int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*)); + void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*)); + void *(*value_pointer)(sqlite3_value*,const char*); + int (*vtab_nochange)(sqlite3_context*); + int (*value_nochange)(sqlite3_value*); + const char *(*vtab_collation)(sqlite3_index_info*,int); + /* Version 3.24.0 and later */ + int (*keyword_count)(void); + int (*keyword_name)(int,const char**,int*); + int (*keyword_check)(const char*,int); + sqlite3_str *(*str_new)(sqlite3*); + char *(*str_finish)(sqlite3_str*); + void (*str_appendf)(sqlite3_str*, const char *zFormat, ...); + void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list); + void (*str_append)(sqlite3_str*, const char *zIn, int N); + void (*str_appendall)(sqlite3_str*, const char *zIn); + void (*str_appendchar)(sqlite3_str*, int N, char C); + void (*str_reset)(sqlite3_str*); + int (*str_errcode)(sqlite3_str*); + int (*str_length)(sqlite3_str*); + char *(*str_value)(sqlite3_str*); + /* Version 3.25.0 and later */ + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); + /* Version 3.26.0 and later */ + const char *(*normalized_sql)(sqlite3_stmt*); + /* Version 3.28.0 and later */ + int (*stmt_isexplain)(sqlite3_stmt*); + int (*value_frombind)(sqlite3_value*); + /* Version 3.30.0 and later */ + int (*drop_modules)(sqlite3*,const char**); + /* Version 3.31.0 and later */ + sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64); + const char *(*uri_key)(const char*,int); + const char *(*filename_database)(const char*); + const char *(*filename_journal)(const char*); + const char *(*filename_wal)(const char*); + /* Version 3.32.0 and later */ + char *(*create_filename)(const char*,const char*,const char*, + int,const char**); + void (*free_filename)(char*); + sqlite3_file *(*database_file_object)(const char*); + /* Version 3.34.0 and later */ + int (*txn_state)(sqlite3*,const char*); +}; + +/* +** This is the function signature used for all extension entry points. It +** is also defined in the file "loadext.c". +*/ +typedef int (*sqlite3_loadext_entry)( + sqlite3 *db, /* Handle to the database. */ + char **pzErrMsg, /* Used to set error string on failure. */ + const sqlite3_api_routines *pThunk /* Extension API function pointers. */ +); + +/* +** The following macros redefine the API routines so that they are +** redirected through the global sqlite3_api structure. +** +** This header file is also used by the loadext.c source file +** (part of the main SQLite library - not an extension) so that +** it can get access to the sqlite3_api_routines structure +** definition. But the main library does not want to redefine +** the API. So the redefinition macros are only valid if the +** SQLITE_CORE macros is undefined. +*/ +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +#define sqlite3_aggregate_context sqlite3_api->aggregate_context +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_aggregate_count sqlite3_api->aggregate_count +#endif +#define sqlite3_bind_blob sqlite3_api->bind_blob +#define sqlite3_bind_double sqlite3_api->bind_double +#define sqlite3_bind_int sqlite3_api->bind_int +#define sqlite3_bind_int64 sqlite3_api->bind_int64 +#define sqlite3_bind_null sqlite3_api->bind_null +#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count +#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index +#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name +#define sqlite3_bind_text sqlite3_api->bind_text +#define sqlite3_bind_text16 sqlite3_api->bind_text16 +#define sqlite3_bind_value sqlite3_api->bind_value +#define sqlite3_busy_handler sqlite3_api->busy_handler +#define sqlite3_busy_timeout sqlite3_api->busy_timeout +#define sqlite3_changes sqlite3_api->changes +#define sqlite3_close sqlite3_api->close +#define sqlite3_collation_needed sqlite3_api->collation_needed +#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 +#define sqlite3_column_blob sqlite3_api->column_blob +#define sqlite3_column_bytes sqlite3_api->column_bytes +#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 +#define sqlite3_column_count sqlite3_api->column_count +#define sqlite3_column_database_name sqlite3_api->column_database_name +#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 +#define sqlite3_column_decltype sqlite3_api->column_decltype +#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 +#define sqlite3_column_double sqlite3_api->column_double +#define sqlite3_column_int sqlite3_api->column_int +#define sqlite3_column_int64 sqlite3_api->column_int64 +#define sqlite3_column_name sqlite3_api->column_name +#define sqlite3_column_name16 sqlite3_api->column_name16 +#define sqlite3_column_origin_name sqlite3_api->column_origin_name +#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 +#define sqlite3_column_table_name sqlite3_api->column_table_name +#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 +#define sqlite3_column_text sqlite3_api->column_text +#define sqlite3_column_text16 sqlite3_api->column_text16 +#define sqlite3_column_type sqlite3_api->column_type +#define sqlite3_column_value sqlite3_api->column_value +#define sqlite3_commit_hook sqlite3_api->commit_hook +#define sqlite3_complete sqlite3_api->complete +#define sqlite3_complete16 sqlite3_api->complete16 +#define sqlite3_create_collation sqlite3_api->create_collation +#define sqlite3_create_collation16 sqlite3_api->create_collation16 +#define sqlite3_create_function sqlite3_api->create_function +#define sqlite3_create_function16 sqlite3_api->create_function16 +#define sqlite3_create_module sqlite3_api->create_module +#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 +#define sqlite3_data_count sqlite3_api->data_count +#define sqlite3_db_handle sqlite3_api->db_handle +#define sqlite3_declare_vtab sqlite3_api->declare_vtab +#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache +#define sqlite3_errcode sqlite3_api->errcode +#define sqlite3_errmsg sqlite3_api->errmsg +#define sqlite3_errmsg16 sqlite3_api->errmsg16 +#define sqlite3_exec sqlite3_api->exec +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_expired sqlite3_api->expired +#endif +#define sqlite3_finalize sqlite3_api->finalize +#define sqlite3_free sqlite3_api->free +#define sqlite3_free_table sqlite3_api->free_table +#define sqlite3_get_autocommit sqlite3_api->get_autocommit +#define sqlite3_get_auxdata sqlite3_api->get_auxdata +#define sqlite3_get_table sqlite3_api->get_table +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_global_recover sqlite3_api->global_recover +#endif +#define sqlite3_interrupt sqlite3_api->interruptx +#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid +#define sqlite3_libversion sqlite3_api->libversion +#define sqlite3_libversion_number sqlite3_api->libversion_number +#define sqlite3_malloc sqlite3_api->malloc +#define sqlite3_mprintf sqlite3_api->mprintf +#define sqlite3_open sqlite3_api->open +#define sqlite3_open16 sqlite3_api->open16 +#define sqlite3_prepare sqlite3_api->prepare +#define sqlite3_prepare16 sqlite3_api->prepare16 +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_profile sqlite3_api->profile +#define sqlite3_progress_handler sqlite3_api->progress_handler +#define sqlite3_realloc sqlite3_api->realloc +#define sqlite3_reset sqlite3_api->reset +#define sqlite3_result_blob sqlite3_api->result_blob +#define sqlite3_result_double sqlite3_api->result_double +#define sqlite3_result_error sqlite3_api->result_error +#define sqlite3_result_error16 sqlite3_api->result_error16 +#define sqlite3_result_int sqlite3_api->result_int +#define sqlite3_result_int64 sqlite3_api->result_int64 +#define sqlite3_result_null sqlite3_api->result_null +#define sqlite3_result_text sqlite3_api->result_text +#define sqlite3_result_text16 sqlite3_api->result_text16 +#define sqlite3_result_text16be sqlite3_api->result_text16be +#define sqlite3_result_text16le sqlite3_api->result_text16le +#define sqlite3_result_value sqlite3_api->result_value +#define sqlite3_rollback_hook sqlite3_api->rollback_hook +#define sqlite3_set_authorizer sqlite3_api->set_authorizer +#define sqlite3_set_auxdata sqlite3_api->set_auxdata +#define sqlite3_snprintf sqlite3_api->xsnprintf +#define sqlite3_step sqlite3_api->step +#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata +#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup +#define sqlite3_total_changes sqlite3_api->total_changes +#define sqlite3_trace sqlite3_api->trace +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings +#endif +#define sqlite3_update_hook sqlite3_api->update_hook +#define sqlite3_user_data sqlite3_api->user_data +#define sqlite3_value_blob sqlite3_api->value_blob +#define sqlite3_value_bytes sqlite3_api->value_bytes +#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 +#define sqlite3_value_double sqlite3_api->value_double +#define sqlite3_value_int sqlite3_api->value_int +#define sqlite3_value_int64 sqlite3_api->value_int64 +#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type +#define sqlite3_value_text sqlite3_api->value_text +#define sqlite3_value_text16 sqlite3_api->value_text16 +#define sqlite3_value_text16be sqlite3_api->value_text16be +#define sqlite3_value_text16le sqlite3_api->value_text16le +#define sqlite3_value_type sqlite3_api->value_type +#define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->xvsnprintf +#define sqlite3_overload_function sqlite3_api->overload_function +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_clear_bindings sqlite3_api->clear_bindings +#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob +#define sqlite3_blob_bytes sqlite3_api->blob_bytes +#define sqlite3_blob_close sqlite3_api->blob_close +#define sqlite3_blob_open sqlite3_api->blob_open +#define sqlite3_blob_read sqlite3_api->blob_read +#define sqlite3_blob_write sqlite3_api->blob_write +#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 +#define sqlite3_file_control sqlite3_api->file_control +#define sqlite3_memory_highwater sqlite3_api->memory_highwater +#define sqlite3_memory_used sqlite3_api->memory_used +#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc +#define sqlite3_mutex_enter sqlite3_api->mutex_enter +#define sqlite3_mutex_free sqlite3_api->mutex_free +#define sqlite3_mutex_leave sqlite3_api->mutex_leave +#define sqlite3_mutex_try sqlite3_api->mutex_try +#define sqlite3_open_v2 sqlite3_api->open_v2 +#define sqlite3_release_memory sqlite3_api->release_memory +#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem +#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig +#define sqlite3_sleep sqlite3_api->sleep +#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit +#define sqlite3_vfs_find sqlite3_api->vfs_find +#define sqlite3_vfs_register sqlite3_api->vfs_register +#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister +#define sqlite3_threadsafe sqlite3_api->xthreadsafe +#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob +#define sqlite3_result_error_code sqlite3_api->result_error_code +#define sqlite3_test_control sqlite3_api->test_control +#define sqlite3_randomness sqlite3_api->randomness +#define sqlite3_context_db_handle sqlite3_api->context_db_handle +#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes +#define sqlite3_limit sqlite3_api->limit +#define sqlite3_next_stmt sqlite3_api->next_stmt +#define sqlite3_sql sqlite3_api->sql +#define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_text64 sqlite3_api->result_text64 +#define sqlite3_strglob sqlite3_api->strglob +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +/* Version 3.10.0 and later */ +#define sqlite3_status64 sqlite3_api->status64 +#define sqlite3_strlike sqlite3_api->strlike +#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush +/* Version 3.12.0 and later */ +#define sqlite3_system_errno sqlite3_api->system_errno +/* Version 3.14.0 and later */ +#define sqlite3_trace_v2 sqlite3_api->trace_v2 +#define sqlite3_expanded_sql sqlite3_api->expanded_sql +/* Version 3.18.0 and later */ +#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid +/* Version 3.20.0 and later */ +#define sqlite3_prepare_v3 sqlite3_api->prepare_v3 +#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3 +#define sqlite3_bind_pointer sqlite3_api->bind_pointer +#define sqlite3_result_pointer sqlite3_api->result_pointer +#define sqlite3_value_pointer sqlite3_api->value_pointer +/* Version 3.22.0 and later */ +#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange +#define sqlite3_value_nochange sqlite3_api->value_nochange +#define sqlite3_vtab_collation sqlite3_api->vtab_collation +/* Version 3.24.0 and later */ +#define sqlite3_keyword_count sqlite3_api->keyword_count +#define sqlite3_keyword_name sqlite3_api->keyword_name +#define sqlite3_keyword_check sqlite3_api->keyword_check +#define sqlite3_str_new sqlite3_api->str_new +#define sqlite3_str_finish sqlite3_api->str_finish +#define sqlite3_str_appendf sqlite3_api->str_appendf +#define sqlite3_str_vappendf sqlite3_api->str_vappendf +#define sqlite3_str_append sqlite3_api->str_append +#define sqlite3_str_appendall sqlite3_api->str_appendall +#define sqlite3_str_appendchar sqlite3_api->str_appendchar +#define sqlite3_str_reset sqlite3_api->str_reset +#define sqlite3_str_errcode sqlite3_api->str_errcode +#define sqlite3_str_length sqlite3_api->str_length +#define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function +/* Version 3.26.0 and later */ +#define sqlite3_normalized_sql sqlite3_api->normalized_sql +/* Version 3.28.0 and later */ +#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain +#define sqlite3_value_frombind sqlite3_api->value_frombind +/* Version 3.30.0 and later */ +#define sqlite3_drop_modules sqlite3_api->drop_modules +/* Version 3.31.0 and later */ +#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64 +#define sqlite3_uri_key sqlite3_api->uri_key +#define sqlite3_filename_database sqlite3_api->filename_database +#define sqlite3_filename_journal sqlite3_api->filename_journal +#define sqlite3_filename_wal sqlite3_api->filename_wal +/* Version 3.32.0 and later */ +#define sqlite3_create_filename sqlite3_api->create_filename +#define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object +/* Version 3.34.0 and later */ +#define sqlite3_txn_state sqlite3_api->txn_state +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif + +#endif /* SQLITE3EXT_H */ diff --git a/third_party/sqlite3/sqlite3rbu.c b/third_party/sqlite3/sqlite3rbu.c new file mode 100644 index 000000000..7a90de084 --- /dev/null +++ b/third_party/sqlite3/sqlite3rbu.c @@ -0,0 +1,5303 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** +*/ +/* clang-format off */ + +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "third_party/sqlite3/sqlite3.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +#include "third_party/sqlite3/sqlite3rbu.inc" + +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 + +/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM +** to enable checksum verification. +*/ +#ifndef RBU_ENABLE_DELTA_CKSUM +# define RBU_ENABLE_DELTA_CKSUM 0 +#endif + +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +** +** RBU_STATE_DATATBL: +** Only valid if STAGE==1. The RBU database name of the table +** currently being read. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 +#define RBU_STATE_DATATBL 10 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif + +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +#define SQLITE_FCNTL_RBUCNT 5149216 + +/* +** A structure to store values read from the rbu_state table in memory. +*/ +struct RbuState { + int eStage; + char *zTbl; + char *zDataTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; + i64 nPhaseOneStep; +}; + +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; + +struct RbuSpan { + const char *zSpan; + int nSpan; +}; + +/* +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. +** +*/ +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; + +/* +** Values for RbuObjIter.eType +** +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. +*/ +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 + + +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ + +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ + +/* +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. +*/ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; + +/* +** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ + i64 iOalSz; + i64 nPhaseOneStep; + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ +}; + +/* +** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ + rbu_file *pMain; /* List of main db files */ + rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */ +}; + +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. +*/ +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ + + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ + + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ + rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */ +}; + +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + + +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +#if RBU_ENABLE_DELTA_CKSUM +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} +#endif + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#if RBU_ENABLE_DELTA_CKSUM + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#if RBU_ENABLE_DELTA_CKSUM + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } + return rc; +} + +/* +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). +*/ +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + return rc; +} + +/* +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +*/ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql +){ + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + return rc; +} + +/* +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). +*/ +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; + } + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; +} + +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); +} + +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } + + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } + } + } + } + } + + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; + } + return rc; +} + + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). +*/ +static void rbuTargetNameFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + const char *zIn; + assert( argc==1 || argc==2 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn ){ + if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } + } + } +} + +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); + + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_schema " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " + "ORDER BY name" + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_schema " + " WHERE type='index' AND tbl_name = ?" + ); + } + + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} + +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} + +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + } + } + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} + +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } +} + +/* +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. +*/ +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + } + + return zRet; +} + +/* +** Finalize the statement passed as the second argument. +** +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. +*/ +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_schema ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_schema ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_schema where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%'), rootpage" + " FROM sqlite_schema" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } + } + goto rbuTableType_end; + } + } + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } + +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. +*/ +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; + + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + } + + pIter->nIndex = 0; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + } + rbuFinalize(p, pXInfo); + bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; + } + + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; +} + + +/* +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. +*/ +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); + + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + assert( iPk>=0 ); + pIter->abTblPk[iOrder] = (u8)iPk; + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; + } + } + + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); + } + + return p->rc; +} + +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; +} + +/* +** Return a comma separated list of the quoted PRIMARY KEY column names, +** in order, for the current table. Before each column name, add the text +** zPre. After each column name, add the zPost text. Use zSeparator as +** the separator text (usually ", "). +*/ +static char *rbuObjIterGetPkList( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + const char *zPre, /* Before each quoted column name */ + const char *zSeparator, /* Separator to use between columns */ + const char *zPost /* After each quoted column name */ +){ + int iPk = 1; + char *zRet = 0; + const char *zSep = ""; + while( 1 ){ + int i; + for(i=0; inTblCol; i++){ + if( (int)pIter->abTblPk[i]==iPk ){ + const char *zCol = pIter->azTblCol[i]; + zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost); + zSep = zSeparator; + break; + } + } + if( i==pIter->nTblCol ) break; + iPk++; + } + return zRet; +} + +/* +** This function is called as part of restarting an RBU vacuum within +** stage 1 of the process (while the *-oal file is being built) while +** updating a table (not an index). The table may be a rowid table or +** a WITHOUT ROWID table. It queries the target database to find the +** largest key that has already been written to the target table and +** constructs a WHERE clause that can be used to extract the remaining +** rows from the source table. For a rowid table, the WHERE clause +** is of the form: +** +** "WHERE _rowid_ > ?" +** +** and for WITHOUT ROWID tables: +** +** "WHERE (key1, key2) > (?, ?)" +** +** Instead of "?" placeholders, the actual WHERE clauses created by +** this function contain literal SQL values. +*/ +static char *rbuVacuumTableStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* RBU iterator object */ + int bRowid, /* True for a rowid table */ + const char *zWrite /* Target table name prefix */ +){ + sqlite3_stmt *pMax = 0; + char *zRet = 0; + if( bRowid ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0); + zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax); + } + rbuFinalize(p, pMax); + }else{ + char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC"); + char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")"); + char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", + zSelect, zWrite, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + const char *zVal = (const char*)sqlite3_column_text(pMax, 0); + zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal); + } + rbuFinalize(p, pMax); + } + + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zList); + } + return zRet; +} + +/* +** This function is called as part of restating an RBU vacuum when the +** current operation is writing content to an index. If possible, it +** queries the target index b-tree for the largest key already written to +** it, then composes and returns an expression that can be used in a WHERE +** clause to select the remaining required rows from the source table. +** It is only possible to return such an expression if: +** +** * The index contains no DESC columns, and +** * The last key written to the index before the operation was +** suspended does not contain any NULL values. +** +** The expression is of the form: +** +** (index-field1, index-field2, ...) > (?, ?, ...) +** +** except that the "?" placeholders are replaced with literal values. +** +** If the expression cannot be created, NULL is returned. In this case, +** the caller has to use an OFFSET clause to extract only the required +** rows from the sourct table, just as it does for an RBU update operation. +*/ +char *rbuVacuumIndexStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter /* RBU iterator object */ +){ + char *zOrder = 0; + char *zLhs = 0; + char *zSelect = 0; + char *zVector = 0; + char *zRet = 0; + int bFailed = 0; + const char *zSep = ""; + int iCol = 0; + sqlite3_stmt *pXInfo = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + if( sqlite3_column_int(pXInfo, 3) ){ + bFailed = 1; + break; + } + + if( iCid<0 ){ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "_rowid_"; + } + }else{ + zCol = pIter->azTblCol[iCid]; + } + + zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q", + zLhs, zSep, zCol, zCollate + ); + zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC", + zOrder, zSep, iCol, zCol, zCollate + ); + zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")", + zSelect, zSep, iCol, zCol + ); + zSep = ", "; + iCol++; + } + rbuFinalize(p, pXInfo); + if( bFailed ) goto index_start_out; + + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pSel = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg, + sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1", + zSelect, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){ + zSep = ""; + for(iCol=0; iColnCol; iCol++){ + const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol); + if( zQuoted[0]=='N' ){ + bFailed = 1; + break; + } + zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted); + zSep = ", "; + } + + if( !bFailed ){ + zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector); + } + } + rbuFinalize(p, pSel); + } + + index_start_out: + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zVector); + sqlite3_free(zLhs); + return zRet; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. +** +** If the index is of the following form: +** +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... +*/ +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ +){ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol = 0; + const char *zType; + + if( iCid==-2 ){ + int iSeq = sqlite3_column_int(pXInfo, 0); + zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom, + pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate + ); + zType = ""; + }else { + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate); + } + + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); + } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol + ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; + } + + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } + + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} + +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); + } + } + return zList; +} + +/* +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** +** Return the string: +** +** "b = ?1 AND c = ?2" +*/ +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter +){ + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); + + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; + } + } + } + return zList; +} + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. +*/ +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); +} + + +/* +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; + + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } + } + return zList; +} + +/* +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; + + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; izIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; + } + } + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); + } + return z; +} + +/* +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. +** +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** +*/ +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_schema WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: +** +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. +*/ +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); + } +} + +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; + + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} + +static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pStmt = 0; + int rc = p->rc; + char *zRet = 0; + + assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zSql = (char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc); + } + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + int iIdxCol = 0; + int nIdxAlloc = 0; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + + /* If necessary, grow the pIter->aIdxCol[] array */ + if( iIdxCol==nIdxAlloc ){ + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + ); + if( aIdxCol==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->aIdxCol = aIdxCol; + nIdxAlloc += 16; + } + + if( c=='(' ){ + if( nParen==0 ){ + assert( iIdxCol==0 ); + pIter->aIdxCol[0].zSpan = &zSql[i+1]; + } + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + i++; + break; + } + }else if( c==',' && nParen==1 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + }else if( c=='-' && zSql[i+1]=='-' ){ + for(i=i+2; zSql[i] && zSql[i]!='\n'; i++); + if( zSql[i]=='\0' ) break; + }else if( c=='/' && zSql[i+1]=='*' ){ + for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++); + if( zSql[i]=='\0' ) break; + i++; + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + pIter->nIdxCol = iIdxCol; + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} + +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. +*/ +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +){ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; + + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } + + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + char *zPart = 0; + int nBind = 0; + + assert( pIter->eType!=RBU_PK_VTAB ); + zPart = rbuObjIterGetIndexWhere(p, pIter); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); + } + + /* And to delete index entries */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( rbuIsVacuum(p) ){ + char *zStart = 0; + if( nOffset ){ + zStart = rbuVacuumIndexStart(p, pIter); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zPart, + (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart, + zCollist, zLimit + ); + sqlite3_free(zStart); + }else + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zPart, zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " + "UNION ALL " + "SELECT %s, rbu_control FROM '%q' " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " + "ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, zPart, + zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), + zCollist, zLimit + ); + } + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql); + }else{ + sqlite3_free(zSql); + } + } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + sqlite3_free(zPart); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } + + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } + + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(4, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + char *zStart = 0; + char *zOrder = 0; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } + + if( rbuIsVacuum(p) ){ + if( nOffset ){ + zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + if( bRbuRowid ){ + zOrder = rbuMPrintf(p, "_rowid_"); + }else{ + zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, (zStart ? zStart : ""), + (zOrder ? "ORDER BY" : ""), zOrder, + zLimit + ) + ); + } + sqlite3_free(zStart); + sqlite3_free(zOrder); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} + +/* +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. +*/ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ +){ + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; + + /* In case an error occurs */ + *ppStmt = 0; + + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; + } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; + }else{ + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); + } + + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + + if( zSet ){ + const char *zPrefix = ""; + + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } + + return p->rc; +} + +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } + } + return db; +} + +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zDataTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_DATATBL: + pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + +/* +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. +*/ +static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + } + } + + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); + } + +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_schema" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>=RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema"); + + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + } +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = (int)strlen(z)&0xffffff; + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} + +/* +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. +*/ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ + + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0); + } + } + + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } + + if( p->rc==SQLITE_OK && p->nFrame>0 ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } + } +} + +/* +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. +*/ +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_INTERNAL; + } + + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame